diff --git a/.env.example b/.env.example index 3d2c618d..905e81d3 100644 --- a/.env.example +++ b/.env.example @@ -27,11 +27,13 @@ FRONTEND_URL= PUBLIC_POSTHOG_PROJECT_TOKEN= PUBLIC_POSTHOG_HOST=https://us.i.posthog.com -# ── Connection encryption (advanced) ── -# Encryption key for storing database connection strings at rest. -# If unset, a key is auto-generated and stored in ~/.bolodb/.secret. -# RECENT_CONNECTIONS_MASTER_KEY= # Master key for key rotation (encrypts the key file) -# RECENT_CONNECTIONS_KEY= # Direct encryption key (alternative to master key) +# ── Connection encryption (required for storing DB URLs at rest) ── +# Encrypts saved connection URLs. REQUIRED — the app will not start without it, +# and saved connections cannot be stored or reopened until it is set. It is read +# only from the environment (never written to disk), so keep it stable across +# deploys: change it and previously saved connections must be re-added. +# Generate with: python -c "import secrets; print(secrets.token_urlsafe(48))" +RECENT_CONNECTIONS_KEY= # ── Server (advanced) ── # Set to "true" when behind HTTPS termination (nginx with SSL) @@ -42,3 +44,9 @@ SKIP_DB_LOCK=false # CORS_ORIGINS= # Set automatically in Docker; controls SQLite path selection # RUNNING_IN_DOCKER=true + +# ── Maintenance & Storage ── +ACTIVITY_LOG_RETENTION_DAYS=30 +ACTIVITY_CLEANUP_ENABLED=true +ACTIVITY_CLEANUP_INTERVAL_HOURS=24 +BOLODB_DATA_DIR= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76d0a355..38f078af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ruff + # Pin to match .pre-commit-config.yaml so CI and local hooks lint + # identically; an unpinned install pulls whatever ruff released last, + # whose changing defaults have broken CI before. + pip install ruff==0.15.22 pip install -r backend/requirements.txt - name: Lint with Ruff diff --git a/.gitignore b/.gitignore index d74c3179..d4eed0cb 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ credentials.json *.key .credentials docker-compose.override.yml +venv/ + +.opencode/ +.agents/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c8a9f633..bce8b4f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ BoloDB is built using a modern containerized stack designed for ease of local de - **Backend:** FastAPI (Python 3.10+) utilizing SQLAlchemy for database connectivity, and `sqlglot` for secure AST-based read-only validation. - **Frontend:** A SvelteKit application utilizing TypeScript and Tailwind CSS for a polished, responsive user interface. - **Database:** PostgreSQL container storing user accounts, query history, conversations, and recent connections. -- **Knowledge Base:** Per-user SQLite database storing verified queries, glossary, and trust level. +- **Knowledge Base:** PostgreSQL persistence storing verified queries, glossary, and trust level. - **Reverse Proxy:** Nginx configured to route traffic seamlessly between the frontend and backend services while handling proxy configurations. - **LLM Connectors:** Support for OpenRouter API. diff --git a/DOCKERFILE.render b/DOCKERFILE.render index 77f8d4e5..51cfd5bc 100644 --- a/DOCKERFILE.render +++ b/DOCKERFILE.render @@ -45,9 +45,14 @@ COPY --from=frontend-builder /app/build /usr/share/nginx/html COPY nginx.conf.render /etc/nginx/conf.d/default.conf RUN addgroup --system bolodb && adduser --system --home /home/bolodb --ingroup bolodb bolodb \ - && mkdir -p /home/bolodb/.bolodb \ - && chown -R bolodb:bolodb /home/bolodb/.bolodb /app + && mkdir -p /app/data \ + && chmod +x /app/backend/docker-entrypoint.sh \ + && chown -R bolodb:bolodb /app EXPOSE 80 +# Same as the compose image: migrate to head before serving, so pointing +# DATABASE_URL at a new database needs no manual step. +ENTRYPOINT ["/app/backend/docker-entrypoint.sh"] + CMD sh -c "nginx -g 'daemon off;' & exec python -m uvicorn backend.app.server:create_app --factory --host 0.0.0.0 --port 4321 --log-level info" diff --git a/LICENSE b/LICENSE index 2c3cc8f1..739588a7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 HAAHIT +Copyright (c) 2026 BoloDB Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index bcd8b8a7..879b8cd2 100644 --- a/README.md +++ b/README.md @@ -4,102 +4,98 @@ **Ask your data. Trust the answer.** -A text-to-SQL product for non-technical users. Connect any SQL database, ask -questions in plain English, get answers with a plain-English explanation and a -confidence level. Every answer you confirm teaches it your database — accuracy -improves with use. +A multi-tenant text-to-SQL web application for non-technical users and teams. Connect your database, ask questions in plain English, and get instant answers with plain-English restatements, ECharts visualizations, and confidence levels. Save queries to interactive dashboards and manage multi-user workspaces with role-based access control (RBAC). -**📚 Full documentation lives in [`docs/`](docs/README.md)** — written for -non-technical readers, with code pointers for developers at every step. +**📚 Full documentation lives in [`docs/`](docs/README.md)** — written for non-technical readers, with code pointers for developers at every step. + +--- ## Quick Start (Docker) -The easiest and recommended way to run BoloDB is using Docker. This ensures all services (FastAPI Backend, SvelteKit Frontend, Nginx, and MongoDB) run seamlessly. +The easiest and recommended way to run BoloDB is using Docker Compose (FastAPI backend + SvelteKit frontend + Nginx + PostgreSQL). + +1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine. +2. Clone the repository and navigate into the root directory. +3. Copy `.env.example` to `.env` and fill in required secrets (`OPENROUTER_API_KEY`, `JWT_SECRET`, `DATABASE_URL`, `RECENT_CONNECTIONS_KEY`, `SUPABASE_URL`, `SUPABASE_ANON_KEY`, `SUPABASE_JWT_SECRET`). +4. Start the application stack: -1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) (Windows/Mac) or Docker Engine (Linux). -2. Open a terminal in the project directory. -3. Start the application: + **Production / Deployment**: ```bash - docker compose up -d + docker compose up --build -d ``` -4. Open [http://localhost:5173](http://localhost:5173) in your browser. -5. Set the `OPENROUTER_API_KEY` environment variable in your docker-compose or `.env` file, connect a database (or click "Try with sample data"), and start asking! + Open [http://localhost:8080](http://localhost:8080). -## The AI engine + **Local Development (Vite HMR)**: + ```bash + docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build + ``` + Open [http://localhost:8080](http://localhost:8080) (Nginx proxy) or [http://localhost:5174](http://localhost:5174) (Vite frontend). -BoloDB uses **OpenRouter** (specifically the `deepseek/deepseek-v4-flash` model) for all AI operations. You only need one thing: -an OpenRouter API key. +--- -1. Go to https://openrouter.ai/keys. -2. Click **Create Key** and copy it. -3. Set it as the `OPENROUTER_API_KEY` environment variable for your deployment. +## Key Features -| Model | Best for | -|---|---| -| `deepseek/deepseek-v4-flash` (default) | Best cost/accuracy balance for most uses | +- **Multi-Tenant Workspaces & RBAC**: Isolate connections, dashboards, and knowledge bases per workspace. Assign Owner, Admin, or Member roles with fine-grained capability checks. +- **OpenRouter AI Engine**: Powered by `deepseek/deepseek-v4-flash` via OpenRouter. +- **Interactive Dashboards & Charts**: Automated ECharts visual inference (Bar, Line, Area, Pie, Number, Table) with query panel customization. +- **Semantic Layer**: Define custom business metrics, explicit join paths, synonyms, and value mappings to guide AI SQL generation. +- **Safety & Defense in Depth**: Read-only AST validation, SSRF protection, host allowlisting, and 5s statement timeouts. +- **PostgreSQL Persistence**: All state (users, workspaces, connections encrypted at rest, query history, dashboards, verified Q&A) is persisted asynchronously in PostgreSQL. -**Privacy:** what's sent to the AI to generate SQL is your question plus the -context BoloDB builds around it: the database *structure* (table/column names, -keys), a few sample values and sample rows per table, your confirmed -business-term glossary, previously verified question→SQL examples, and the -last couple of conversation turns. **The prompt never includes** bulk table -contents, query results, or credentials — the OpenRouter API key travels only in -the request's authentication header. See -[docs/03-the-ai-layer-openrouter.md](docs/03-the-ai-layer-openrouter.md) for exactly -what's in every prompt. +--- -## Database connection strings +## Supported Database Connections -| Database | Format | +| Database | Connection URL Format | |---|---| -| SQLite | `sqlite:///C:/path/to/file.db` | | PostgreSQL | `postgresql://user:pass@host:5432/dbname` | | MySQL | `mysql+pymysql://user:pass@host:3306/dbname` | +| SQLite | `sqlite:///C:/path/to/file.db` | | SQL Server | `mssql+pyodbc://user:pass@server/db?driver=ODBC+Driver+17+for+SQL+Server` | +| DuckDB | `duckdb:///path/to/file.duckdb` | + +--- + +## Architecture & Code Map + +Full file and directory index is available in [`docs/07-file-map.md`](docs/07-file-map.md). + +```text + Browser (SvelteKit 5 Frontend, frontend/src) + │ HTTP / SSE streaming (JWT cookie auth, X-Workspace-Id and X-Db-Id headers) + ▼ + FastAPI Backend (backend/app) + ├── controllers/query.py ─── The query pipeline (knowledge → schema link → LLM → repair) + ├── llm.py ───────────────── OpenRouter provider (deepseek/deepseek-v4-flash) + ├── schema_link.py ───────── Schema linking, budget management & table scoring + ├── sqlvalidate.py ───────── AST static SQL validation + ├── repair.py ────────────── Self-repair loop for auto-correcting broken SQL + ├── semantic.py ──────────── Semantic layer catalog & inference + ├── database.py ──────────── DB introspection & read-only execution guard + └── pgdatabase/ ──────────── Async PostgreSQL persistence (KnowledgeService, Users, Workspaces, Dashboards) +``` -Tip: connect with a **read-only database account** for safety (BoloDB also -enforces read-only itself — see -[docs/05-safety-validation-and-self-repair.md](docs/05-safety-validation-and-self-repair.md)). - -## How it works - -1. **Connect** a database and ensure your OpenRouter API key is set in the environment. -2. **Onboard** (first time) — BoloDB profiles the tables, confirms business-term meanings with you, and runs a few starter questions for you to verify -3. **Ask** questions in plain English. Every answer includes: - - A plain-English restatement ("I summed completed orders for this month") - - A confidence level (High/Medium/Low) based on real signals - - The results table and SQL on a toggle - - Automatic self-repair: generated SQL is validated against your schema and fixed before you ever see an error -4. **Verify** — click "Yes, correct" to save the answer. Similar future questions reuse it and show higher confidence. The trust level climbs from Supervised to Assisted to Trusted - -The full pipeline, step by step with code pointers: -[docs/02-how-a-question-becomes-an-answer.md](docs/02-how-a-question-becomes-an-answer.md). +--- -## Useful Docker Commands +## Running Unit Tests ```bash -docker compose up -d # Start the application in the background -docker compose logs -f # View live logs from all services -docker compose down # Stop all services -docker compose build --no-cache # Rebuild all images from scratch +pip install -r backend/requirements.txt +pytest tests -v ``` -## Running tests +The test suite runs entirely offline using mock OpenRouter providers. -```bash -pip install -r backend/requirements.txt -pytest tests -``` +--- -The test suite needs no network and no API key — all AI calls are faked. +## Privacy & Security -## Privacy +- Connection credentials are **encrypted at rest** using Fernet symmetric encryption key (`RECENT_CONNECTIONS_KEY`). +- AI prompts include only database structure, compact column schemas, verified Q&A examples, and business glossary definitions. Bulk data rows, query results, and credentials are **never sent** to the AI model. +- All SQL execution runs in strict **read-only** mode with statement timeout protection. -- All learned knowledge and user settings are stored locally (`~/.bolodb/`) and in the local MongoDB container volume. The API key is read from the environment and never stored on disk. -- To generate SQL, the AI is sent your question plus context: the schema, a few sample values/rows per table, your confirmed glossary terms, verified question→SQL examples, and recent conversation turns. The prompt never includes bulk table data, query results, or credentials (the API key is used only as the request's authentication header). -- Queries run strictly read-only. -- No telemetry, no cloud sync. +--- ## License -MIT — see [LICENSE](LICENSE). +MIT — see [LICENSE](LICENSE). diff --git a/backend/DOCKERFILE b/backend/DOCKERFILE index 7407d702..69c78baf 100644 --- a/backend/DOCKERFILE +++ b/backend/DOCKERFILE @@ -20,9 +20,12 @@ RUN pip install --no-cache-dir -r /app/backend/requirements.txt COPY backend/ /app/backend/ # ── 3. Create non-root user ── +# /app/data holds the regenerable sample database; create it owned by the +# runtime user so it is writable whether or not a volume is mounted over it. RUN addgroup --system bolodb && adduser --system --home /home/bolodb --ingroup bolodb bolodb \ - && mkdir -p /home/bolodb/.bolodb \ - && chown -R bolodb:bolodb /home/bolodb/.bolodb /app + && mkdir -p /app/data \ + && chmod +x /app/backend/docker-entrypoint.sh \ + && chown -R bolodb:bolodb /app USER bolodb @@ -31,5 +34,9 @@ EXPOSE 4321 HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ CMD python -c "import httpx; httpx.get('http://localhost:4321/api/health', timeout=3).raise_for_status()" || exit 1 +# Migrations run first, so a deploy pointed at a new or stale database brings +# the schema to head before the app serves its first request. +ENTRYPOINT ["/app/backend/docker-entrypoint.sh"] + CMD ["python", "-m", "uvicorn", "backend.app.server:create_app", \ "--factory", "--host", "0.0.0.0", "--port", "4321", "--log-level", "warning"] diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 2c04e642..a8b8b30d 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -39,9 +39,23 @@ def run_migrations_offline() -> None: context.run_migrations() +# Any 64-bit constant works; it only has to be the same in every process that +# migrates this database. Derived from the project name so it cannot collide +# with an advisory lock another application takes on a shared cluster. +_MIGRATION_LOCK_ID = 8410231166942030001 + + def do_run_migrations(connection): context.configure(connection=connection, target_metadata=target_metadata) with context.begin_transaction(): + # Containers start migrations concurrently on a scaled deployment, and + # two Alembic runs against one database race on alembic_version. The + # transaction-scoped advisory lock makes the second wait for the first + # and then find nothing left to apply; it is released on commit or + # rollback, so a crashed migration cannot wedge later deploys. + connection.exec_driver_sql( + f"SELECT pg_advisory_xact_lock({_MIGRATION_LOCK_ID})" + ) context.run_migrations() diff --git a/backend/alembic/versions/0001_initial_schema.py b/backend/alembic/versions/0001_initial_schema.py deleted file mode 100644 index 7a7860a4..00000000 --- a/backend/alembic/versions/0001_initial_schema.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Create initial schema (users, conversations, query_history, recent_connections) - -Revision ID: 0001 -Revises: -Create Date: 2026-07-13 -""" - -from typing import Sequence, Union -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import UUID, JSONB - -revision: str = "0001" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto") - # Primary keys carry no server_default: the application is the sole writer - # and generates UUIDv7 ids in Python (pgdatabase._uuid7). A gen_random_uuid() - # default would emit UUIDv4 on any direct insert/backfill, producing a mix of - # UUID versions and losing the time-ordering UUIDv7 provides. - op.create_table( - "users", - sa.Column( - "id", - UUID(as_uuid=True), - primary_key=True, - ), - sa.Column("email", sa.String(), nullable=False, unique=True), - sa.Column("hashed_pass", sa.String(), nullable=False, server_default=""), - sa.Column("role", sa.String(), nullable=False, server_default="user"), - sa.Column("google_id", sa.String(), nullable=True, unique=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - ) - - op.create_table( - "conversations", - sa.Column( - "id", - UUID(as_uuid=True), - primary_key=True, - ), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("title", sa.String(), nullable=False, server_default=""), - sa.Column("database_id", sa.String(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - ) - - op.create_table( - "query_history", - sa.Column( - "id", - UUID(as_uuid=True), - primary_key=True, - ), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("question", sa.Text(), nullable=False), - sa.Column("sql", sa.Text(), nullable=False, server_default=""), - sa.Column("result", JSONB(), nullable=True), - sa.Column("confidence", sa.String(), nullable=False, server_default="low"), - sa.Column("restatement", sa.Text(), nullable=False, server_default=""), - sa.Column( - "conversation_id", - UUID(as_uuid=True), - sa.ForeignKey("conversations.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column( - "timestamp", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - ) - - op.create_table( - "recent_connections", - sa.Column( - "id", - UUID(as_uuid=True), - primary_key=True, - ), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("db_url", sa.Text(), nullable=False), - sa.Column("display_url", sa.String(), nullable=False), - sa.Column("dialect", sa.String(), nullable=False), - sa.Column("db_id", sa.String(), nullable=False), - sa.Column("table_count", sa.Integer(), nullable=False, server_default="0"), - sa.Column( - "connected_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.UniqueConstraint("user_id", "db_id"), - ) - - op.create_index("idx_query_history_user_id", "query_history", ["user_id"]) - op.create_index( - "idx_query_history_conversation_id", "query_history", ["conversation_id"] - ) - op.create_index( - "idx_query_history_timestamp", - "query_history", - ["user_id", sa.text("timestamp DESC")], - ) - op.create_index( - "idx_conversations_user_id", - "conversations", - ["user_id", sa.text("updated_at DESC")], - ) - op.create_index("idx_recent_connections_user_id", "recent_connections", ["user_id"]) - - -def downgrade() -> None: - op.drop_table("recent_connections") - op.drop_table("query_history") - op.drop_table("conversations") - op.drop_table("users") diff --git a/backend/alembic/versions/0002_add_supabase_id.py b/backend/alembic/versions/0002_add_supabase_id.py deleted file mode 100644 index 5c9fb884..00000000 --- a/backend/alembic/versions/0002_add_supabase_id.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Add supabase_id column to users table - -Revision ID: 0002 -Revises: 0001 -Create Date: 2026-07-15 -""" - -from typing import Sequence, Union -from alembic import op -import sqlalchemy as sa - -revision: str = "0002" -down_revision: Union[str, None] = "0001" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "users", - sa.Column("supabase_id", sa.String(), nullable=True, unique=True), - ) - op.create_index("ix_users_supabase_id", "users", ["supabase_id"], unique=True) - - -def downgrade() -> None: - op.drop_index("ix_users_supabase_id", table_name="users") - op.drop_column("users", "supabase_id") diff --git a/backend/alembic/versions/0003_add_email_verified_and_otp_codes.py b/backend/alembic/versions/0003_add_email_verified_and_otp_codes.py deleted file mode 100644 index 0c4fa9af..00000000 --- a/backend/alembic/versions/0003_add_email_verified_and_otp_codes.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Add email_verified column and otp_codes table - -Revision ID: 0003 -Revises: 0002 -Create Date: 2026-07-16 -""" - -from typing import Sequence, Union -from alembic import op -import sqlalchemy as sa - -revision: str = "0003" -down_revision: Union[str, None] = "0002" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "users", - sa.Column( - "email_verified", sa.Boolean(), nullable=False, server_default="false" - ), - ) - op.create_table( - "otp_codes", - sa.Column("id", sa.dialects.postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "user_id", - sa.dialects.postgresql.UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("code_hash", sa.String(), nullable=False), - sa.Column("purpose", sa.String(), nullable=False), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("used", sa.Boolean(), nullable=False, server_default="false"), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.UniqueConstraint("user_id", "purpose", name="uq_otp_user_purpose"), - ) - op.create_index("ix_otp_codes_user_id", "otp_codes", ["user_id"]) - - -def downgrade() -> None: - op.drop_index("ix_otp_codes_user_id", table_name="otp_codes") - op.drop_table("otp_codes") - op.drop_column("users", "email_verified") diff --git a/backend/alembic/versions/0004_add_password_reset_tokens.py b/backend/alembic/versions/0004_add_password_reset_tokens.py deleted file mode 100644 index 25204538..00000000 --- a/backend/alembic/versions/0004_add_password_reset_tokens.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Add password_reset_tokens table - -Revision ID: 0004 -Revises: 0003 -Create Date: 2026-07-16 -""" - -from typing import Sequence, Union -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import UUID - -revision: str = "0004" -down_revision: Union[str, None] = "0003" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Primary keys carry no server_default: the application generates UUIDv7 ids - # in Python (see pgdatabase.models._uuid7), matching the users table. - op.create_table( - "password_reset_tokens", - sa.Column("id", UUID(as_uuid=True), primary_key=True), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("jti_hash", sa.String(), nullable=False, unique=True), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), - sa.Column( - "consumed", - sa.Boolean(), - nullable=False, - server_default=sa.text("false"), - ), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - ) - op.create_index( - "ix_password_reset_tokens_user_id", - "password_reset_tokens", - ["user_id"], - ) - - -def downgrade() -> None: - op.drop_index( - "ix_password_reset_tokens_user_id", - table_name="password_reset_tokens", - ) - op.drop_table("password_reset_tokens") diff --git a/backend/alembic/versions/0005_add_knowledge_tables.py b/backend/alembic/versions/0005_add_knowledge_tables.py deleted file mode 100644 index bad09c0a..00000000 --- a/backend/alembic/versions/0005_add_knowledge_tables.py +++ /dev/null @@ -1,175 +0,0 @@ -"""add knowledge tables and tour_completed - -Revision ID: 0005 -Revises: 0004 -Create Date: 2026-07-17 19:00:00.000000 - -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import UUID - -revision: str = "0005" -down_revision: Union[str, None] = "0004" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """ - Add the tour completion flag and knowledge-related tables to the database. - - The migration creates tables for verified questions and answers, glossary terms, - catalog metadata, joins, synonyms, and value mappings, with indexes scoped by - user and database identifier. - """ - op.add_column( - "users", - sa.Column( - "tour_completed", - sa.Boolean(), - nullable=False, - server_default=sa.text("false"), - ), - ) - - op.create_table( - "verified_qas", - sa.Column("id", UUID(as_uuid=True), primary_key=True), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("db_id", sa.String(), nullable=False), - sa.Column("question", sa.Text(), nullable=False), - sa.Column("sql", sa.Text(), nullable=False, server_default=""), - sa.Column("restatement", sa.Text(), nullable=False, server_default=""), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), - ) - op.create_index("ix_verified_qa_user_db", "verified_qas", ["user_id", "db_id"]) - op.create_unique_constraint( - "uq_verified_qa_user_db_question", - "verified_qas", - ["user_id", "db_id", "question"], - ) - - op.create_table( - "glossary_terms", - sa.Column("id", UUID(as_uuid=True), primary_key=True), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("db_id", sa.String(), nullable=False), - sa.Column("term", sa.String(), nullable=False), - sa.Column("maps_to", sa.String(), nullable=False), - sa.Column("sql_hint", sa.String(), nullable=False, server_default=""), - ) - op.create_index("ix_glossary_user_db", "glossary_terms", ["user_id", "db_id"]) - - op.create_table( - "catalog_columns", - sa.Column("id", UUID(as_uuid=True), primary_key=True), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("db_id", sa.String(), nullable=False), - sa.Column("table_name", sa.String(), nullable=False), - sa.Column("column_name", sa.String(), nullable=False), - sa.Column("description", sa.Text(), nullable=False), - ) - op.create_index("ix_cat_col_user_db", "catalog_columns", ["user_id", "db_id"]) - - op.create_table( - "catalog_metrics", - sa.Column("id", UUID(as_uuid=True), primary_key=True), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("db_id", sa.String(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("description", sa.Text(), nullable=False, server_default=""), - sa.Column("sql_expression", sa.Text(), nullable=False), - ) - op.create_index("ix_cat_met_user_db", "catalog_metrics", ["user_id", "db_id"]) - - op.create_table( - "catalog_joins", - sa.Column("id", UUID(as_uuid=True), primary_key=True), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("db_id", sa.String(), nullable=False), - sa.Column("tables", sa.String(), nullable=False), - sa.Column("join_condition", sa.Text(), nullable=False), - sa.Column("description", sa.Text(), nullable=False, server_default=""), - ) - op.create_index("ix_cat_join_user_db", "catalog_joins", ["user_id", "db_id"]) - - op.create_table( - "catalog_synonyms", - sa.Column("id", UUID(as_uuid=True), primary_key=True), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("db_id", sa.String(), nullable=False), - sa.Column("term", sa.String(), nullable=False), - sa.Column("entity_type", sa.String(), nullable=False), - sa.Column("entity_name", sa.String(), nullable=False), - ) - op.create_index("ix_cat_syn_user_db", "catalog_synonyms", ["user_id", "db_id"]) - - op.create_table( - "catalog_value_mappings", - sa.Column("id", UUID(as_uuid=True), primary_key=True), - sa.Column( - "user_id", - UUID(as_uuid=True), - sa.ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("db_id", sa.String(), nullable=False), - sa.Column("table_name", sa.String(), nullable=False), - sa.Column("column_name", sa.String(), nullable=False), - sa.Column("db_value", sa.String(), nullable=False), - sa.Column("business_label", sa.String(), nullable=False), - ) - op.create_index( - "ix_cat_val_user_db", "catalog_value_mappings", ["user_id", "db_id"] - ) - - -def downgrade() -> None: - """ - Revert the schema changes introduced by this migration. - """ - op.drop_constraint( - "uq_verified_qa_user_db_question", "verified_qas", type_="unique" - ) - op.drop_table("catalog_value_mappings") - op.drop_table("catalog_synonyms") - op.drop_table("catalog_joins") - op.drop_table("catalog_metrics") - op.drop_table("catalog_columns") - op.drop_table("glossary_terms") - op.drop_table("verified_qas") - op.drop_column("users", "tour_completed") diff --git a/backend/alembic/versions/32e55b8f71d4_initial_schema_with_workspaces_and_.py b/backend/alembic/versions/32e55b8f71d4_initial_schema_with_workspaces_and_.py new file mode 100644 index 00000000..cef0eea1 --- /dev/null +++ b/backend/alembic/versions/32e55b8f71d4_initial_schema_with_workspaces_and_.py @@ -0,0 +1,339 @@ +"""Initial schema with workspaces and multi DBs + +Revision ID: 32e55b8f71d4 +Revises: +Create Date: 2026-07-22 17:57:49.765868 +""" + +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "32e55b8f71d4" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "users", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("email", sa.String(), nullable=False), + sa.Column("hashed_pass", sa.String(), nullable=False), + sa.Column("role", sa.String(), nullable=False), + sa.Column("google_id", sa.String(), nullable=True), + sa.Column("supabase_id", sa.String(), nullable=True), + sa.Column("email_verified", sa.Boolean(), nullable=False), + sa.Column("first_name", sa.String(), nullable=True), + sa.Column("last_name", sa.String(), nullable=True), + sa.Column("avatar_url", sa.String(), nullable=True), + sa.Column("tour_completed", sa.Boolean(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("email"), + sa.UniqueConstraint("google_id"), + sa.UniqueConstraint("supabase_id"), + ) + op.create_table( + "otp_codes", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("user_id", sa.UUID(), nullable=False), + sa.Column("code_hash", sa.String(), nullable=False), + sa.Column("purpose", sa.String(), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("used", sa.Boolean(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "purpose", name="uq_otp_user_purpose"), + ) + op.create_table( + "password_reset_tokens", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("user_id", sa.UUID(), nullable=False), + sa.Column("jti_hash", sa.String(), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("consumed", sa.Boolean(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("jti_hash"), + ) + op.create_table( + "workspaces", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("slug", sa.String(), nullable=False), + sa.Column("created_by", sa.UUID(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["created_by"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("slug"), + ) + op.create_table( + "catalog_columns", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("db_id", sa.String(), nullable=False), + sa.Column("table_name", sa.String(), nullable=False), + sa.Column("column_name", sa.String(), nullable=False), + sa.Column("description", sa.Text(), nullable=False), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_cat_col_user_db", "catalog_columns", ["workspace_id", "db_id"], unique=False + ) + op.create_table( + "catalog_joins", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("db_id", sa.String(), nullable=False), + sa.Column("tables", sa.String(), nullable=False), + sa.Column("join_condition", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=False), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_cat_join_user_db", "catalog_joins", ["workspace_id", "db_id"], unique=False + ) + op.create_table( + "catalog_metrics", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("db_id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.Text(), nullable=False), + sa.Column("sql_expression", sa.Text(), nullable=False), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_cat_met_user_db", "catalog_metrics", ["workspace_id", "db_id"], unique=False + ) + op.create_table( + "catalog_synonyms", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("db_id", sa.String(), nullable=False), + sa.Column("term", sa.String(), nullable=False), + sa.Column("entity_type", sa.String(), nullable=False), + sa.Column("entity_name", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_cat_syn_user_db", + "catalog_synonyms", + ["workspace_id", "db_id"], + unique=False, + ) + op.create_table( + "catalog_value_mappings", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("db_id", sa.String(), nullable=False), + sa.Column("table_name", sa.String(), nullable=False), + sa.Column("column_name", sa.String(), nullable=False), + sa.Column("db_value", sa.String(), nullable=False), + sa.Column("business_label", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_cat_val_user_db", + "catalog_value_mappings", + ["workspace_id", "db_id"], + unique=False, + ) + op.create_table( + "conversations", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("title", sa.String(), nullable=False), + sa.Column("database_id", sa.String(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_conversations_workspace_updated", + "conversations", + ["workspace_id", "updated_at"], + unique=False, + ) + op.create_table( + "glossary_terms", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("db_id", sa.String(), nullable=False), + sa.Column("term", sa.String(), nullable=False), + sa.Column("maps_to", sa.String(), nullable=False), + sa.Column("sql_hint", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_glossary_user_db", "glossary_terms", ["workspace_id", "db_id"], unique=False + ) + op.create_table( + "recent_connections", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("db_url", sa.Text(), nullable=False), + sa.Column("display_url", sa.String(), nullable=False), + sa.Column("alias_name", sa.String(), nullable=True), + sa.Column("dialect", sa.String(), nullable=False), + sa.Column("db_id", sa.String(), nullable=False), + sa.Column("table_count", sa.Integer(), nullable=False), + sa.Column("connected_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("workspace_id", "db_id"), + ) + op.create_table( + "verified_qas", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("db_id", sa.String(), nullable=False), + sa.Column("question", sa.Text(), nullable=False), + sa.Column("sql", sa.Text(), nullable=False), + sa.Column("restatement", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "workspace_id", "db_id", "question", name="uq_verified_qa_user_db_question" + ), + ) + op.create_index( + "ix_verified_qa_user_db", + "verified_qas", + ["workspace_id", "db_id"], + unique=False, + ) + op.create_table( + "workspace_invites", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("email", sa.String(), nullable=False), + sa.Column("role", sa.String(), nullable=False), + sa.Column("token", sa.String(), nullable=False), + sa.Column("invited_by", sa.UUID(), nullable=True), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("accepted_at", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint( + "role IN ('owner', 'admin', 'member')", name="ck_workspace_invite_role" + ), + sa.ForeignKeyConstraint(["invited_by"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("token"), + sa.UniqueConstraint("workspace_id", "email", name="uq_workspace_invite_email"), + ) + op.create_table( + "workspace_members", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("user_id", sa.UUID(), nullable=False), + sa.Column("role", sa.String(), nullable=False), + sa.Column("invited_by", sa.UUID(), nullable=True), + sa.Column("joined_at", sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint( + "role IN ('owner', 'admin', 'member')", name="ck_workspace_member_role" + ), + sa.ForeignKeyConstraint(["invited_by"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("workspace_id", "user_id", name="uq_workspace_member"), + ) + op.create_index( + "ix_workspace_member_user", "workspace_members", ["user_id"], unique=False + ) + op.create_table( + "query_history", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("user_id", sa.UUID(), nullable=False), + sa.Column("question", sa.Text(), nullable=False), + sa.Column("sql", sa.Text(), nullable=False), + sa.Column("result", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("confidence", sa.String(), nullable=False), + sa.Column("restatement", sa.Text(), nullable=False), + sa.Column("conversation_id", sa.UUID(), nullable=True), + sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["conversation_id"], ["conversations.id"], ondelete="SET NULL" + ), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_query_history_workspace_conv", + "query_history", + ["workspace_id", "conversation_id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_query_history_workspace_conv", table_name="query_history") + op.drop_table("query_history") + op.drop_index("ix_workspace_member_user", table_name="workspace_members") + op.drop_table("workspace_members") + op.drop_table("workspace_invites") + op.drop_index("ix_verified_qa_user_db", table_name="verified_qas") + op.drop_table("verified_qas") + op.drop_table("recent_connections") + op.drop_index("ix_glossary_user_db", table_name="glossary_terms") + op.drop_table("glossary_terms") + op.drop_index("ix_conversations_workspace_updated", table_name="conversations") + op.drop_table("conversations") + op.drop_index("ix_cat_val_user_db", table_name="catalog_value_mappings") + op.drop_table("catalog_value_mappings") + op.drop_index("ix_cat_syn_user_db", table_name="catalog_synonyms") + op.drop_table("catalog_synonyms") + op.drop_index("ix_cat_met_user_db", table_name="catalog_metrics") + op.drop_table("catalog_metrics") + op.drop_index("ix_cat_join_user_db", table_name="catalog_joins") + op.drop_table("catalog_joins") + op.drop_index("ix_cat_col_user_db", table_name="catalog_columns") + op.drop_table("catalog_columns") + op.drop_table("workspaces") + op.drop_table("password_reset_tokens") + op.drop_table("otp_codes") + op.drop_table("users") + # ### end Alembic commands ### diff --git a/backend/alembic/versions/4b1d269a3ad8_add_workspace_activity_log.py b/backend/alembic/versions/4b1d269a3ad8_add_workspace_activity_log.py new file mode 100644 index 00000000..2b4a8484 --- /dev/null +++ b/backend/alembic/versions/4b1d269a3ad8_add_workspace_activity_log.py @@ -0,0 +1,63 @@ +"""Add workspace_activity_log + +Revision ID: 4b1d269a3ad8 +Revises: 32e55b8f71d4 +Create Date: 2026-07-22 19:34:32.909343 +""" + +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "4b1d269a3ad8" +down_revision: Union[str, None] = "32e55b8f71d4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "workspace_activity_log", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("actor_id", sa.UUID(), nullable=True), + sa.Column("event_type", sa.String(), nullable=False), + sa.Column("resource_type", sa.String(), nullable=False), + sa.Column("resource_id", sa.String(), nullable=True), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["actor_id"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_activity_workspace_date", + "workspace_activity_log", + ["workspace_id", "created_at"], + unique=False, + ) + op.create_index( + "ix_activity_workspace_event", + "workspace_activity_log", + ["workspace_id", "event_type"], + unique=False, + ) + op.alter_column("query_history", "user_id", existing_type=sa.UUID(), nullable=True) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # Rows may have been written with a NULL user_id while the column was + # nullable (streamed queries without an acting user). Restoring the NOT NULL + # constraint would fail against those rows, so drop them before the alter. + op.execute("DELETE FROM query_history WHERE user_id IS NULL") + op.alter_column("query_history", "user_id", existing_type=sa.UUID(), nullable=False) + op.drop_index("ix_activity_workspace_event", table_name="workspace_activity_log") + op.drop_index("ix_activity_workspace_date", table_name="workspace_activity_log") + op.drop_table("workspace_activity_log") + # ### end Alembic commands ### diff --git a/backend/alembic/versions/__init__.py b/backend/alembic/versions/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/alembic/versions/a1f2c3d4e5b6_add_user_id_to_conversations.py b/backend/alembic/versions/a1f2c3d4e5b6_add_user_id_to_conversations.py new file mode 100644 index 00000000..0ff97845 --- /dev/null +++ b/backend/alembic/versions/a1f2c3d4e5b6_add_user_id_to_conversations.py @@ -0,0 +1,63 @@ +"""Add user_id to conversations (user + workspace scoping) + +Conversations become private to the user that created them, scoped within their +workspace. Existing conversations were workspace-shared and have no owner, so +they are cleared out (along with their query_history turns) before the NOT NULL +owner column is added. + +Revision ID: a1f2c3d4e5b6 +Revises: e8f901a23b4c +Create Date: 2026-07-24 00:00:00.000000 +""" + +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "a1f2c3d4e5b6" +down_revision: Union[str, None] = "e8f901a23b4c" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Legacy conversations were workspace-shared with no owner; there is no + # correct user to attribute them to, so clear them (and their turns) before + # adding the NOT NULL owner column. + op.execute("DELETE FROM query_history WHERE conversation_id IS NOT NULL") + op.execute("DELETE FROM conversations") + + op.add_column( + "conversations", + sa.Column("user_id", sa.UUID(), nullable=False), + ) + op.create_foreign_key( + "fk_conversations_user_id_users", + "conversations", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + + # Swap the list index to include the owner: the list query filters by + # (workspace_id, user_id) and orders by updated_at. Use IF EXISTS/IF NOT + # EXISTS so the swap is resilient to databases where the old index was + # never created (older initial-schema revisions did not have it). + op.execute("DROP INDEX IF EXISTS ix_conversations_workspace_updated") + op.execute( + "CREATE INDEX IF NOT EXISTS ix_conversations_workspace_user_updated " + "ON conversations (workspace_id, user_id, updated_at)" + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_conversations_workspace_user_updated") + op.execute( + "CREATE INDEX IF NOT EXISTS ix_conversations_workspace_updated " + "ON conversations (workspace_id, updated_at)" + ) + op.drop_constraint( + "fk_conversations_user_id_users", "conversations", type_="foreignkey" + ) + op.drop_column("conversations", "user_id") diff --git a/backend/alembic/versions/b83664841390_add_user_metadata.py b/backend/alembic/versions/b83664841390_add_user_metadata.py new file mode 100644 index 00000000..87566159 --- /dev/null +++ b/backend/alembic/versions/b83664841390_add_user_metadata.py @@ -0,0 +1,92 @@ +"""Add user metadata column and dashboards, saved_queries, dashboard_panels tables + +Revision ID: b83664841390 +Revises: 4b1d269a3ad8 +Create Date: 2026-07-23 00:42:52.421777 +""" + +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "b83664841390" +down_revision: Union[str, None] = "4b1d269a3ad8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "dashboards", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("created_by", sa.UUID(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["created_by"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "saved_queries", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column("created_by", sa.UUID(), nullable=True), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("question", sa.String(), nullable=True), + sa.Column("sql", sa.String(), nullable=True), + sa.Column("columns", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("database_id", sa.String(), nullable=True), + sa.Column("visualization_type", sa.String(), nullable=True), + sa.Column("viz_config", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column( + "last_result", postgresql.JSONB(astext_type=sa.Text()), nullable=True + ), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["created_by"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "dashboard_panels", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("dashboard_id", sa.UUID(), nullable=False), + sa.Column("saved_query_id", sa.UUID(), nullable=True), + sa.Column("title", sa.String(length=255), nullable=True), + sa.Column("visualization_type", sa.String(), nullable=True), + sa.Column("viz_config", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("position", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["dashboard_id"], ["dashboards.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["saved_query_id"], ["saved_queries.id"], ondelete="SET NULL" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.add_column( + "users", + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("users", "metadata") + op.drop_table("dashboard_panels") + op.drop_table("saved_queries") + op.drop_table("dashboards") + # ### end Alembic commands ### diff --git a/backend/alembic/versions/c1a4d7e29b58_add_query_history_chart.py b/backend/alembic/versions/c1a4d7e29b58_add_query_history_chart.py new file mode 100644 index 00000000..29c3a4ab --- /dev/null +++ b/backend/alembic/versions/c1a4d7e29b58_add_query_history_chart.py @@ -0,0 +1,29 @@ +"""Add chart spec to query history + +Revision ID: c1a4d7e29b58 +Revises: b83664841390 +Create Date: 2026-07-23 14:10:00.000000 +""" + +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "c1a4d7e29b58" +down_revision: Union[str, None] = "b83664841390" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Nullable: turns recorded before the model started choosing a chart have + # none, and the UI falls back to its own heuristic for those. + op.add_column( + "query_history", + sa.Column("chart", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("query_history", "chart") diff --git a/backend/alembic/versions/e8f901a23b4c_add_workspace_settings_table.py b/backend/alembic/versions/e8f901a23b4c_add_workspace_settings_table.py new file mode 100644 index 00000000..e881f14f --- /dev/null +++ b/backend/alembic/versions/e8f901a23b4c_add_workspace_settings_table.py @@ -0,0 +1,67 @@ +"""add_workspace_settings_table + +Revision ID: e8f901a23b4c +Revises: c1a4d7e29b58 +Create Date: 2026-07-23 18:30:00.000000 +""" + +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "e8f901a23b4c" +down_revision: Union[str, None] = "c1a4d7e29b58" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "workspace_settings", + sa.Column("workspace_id", sa.UUID(), nullable=False), + sa.Column( + "default_invite_role", + sa.String(length=20), + server_default="member", + nullable=False, + ), + sa.Column( + "invite_expiry_days", + sa.Integer(), + server_default="7", + nullable=False, + ), + sa.Column( + "activity_retention_days", + sa.Integer(), + server_default="30", + nullable=False, + ), + sa.Column( + "role_permissions", + postgresql.JSONB(astext_type=sa.Text()), + server_default="{}", + nullable=False, + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["workspace_id"], ["workspaces.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("workspace_id"), + ) + + +def downgrade() -> None: + op.drop_table("workspace_settings") diff --git a/backend/app/config.py b/backend/app/config.py index 9b6b3ac2..7ccad2b5 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,26 +1,55 @@ -"""Local config + path constants. +"""Runtime configuration, held in memory only. -BoloDB uses OpenRouter (deepseek-v4-flash) for every AI operation. The API key -is read from the OPENROUTER_API_KEY environment variable at startup and -shared across all users — no per-user key storage or encryption is needed. +BoloDB keeps no local state on disk. Secrets come from the environment, and +everything a user creates lives in Postgres. The only file the app writes is the +sample database, and that is a regenerable cache under the repo's ``data/`` +directory (see ``backend/sample_data.py``), not configuration. -Stored at ~/.bolodb/config.json. Only non-sensitive settings (last connected -database URL) are persisted; the API key is never written to disk. +``cfg`` is a small per-process dict: the OpenRouter key (always read fresh from +the environment) and the last connected database URL (a convenience value that +lives only for the process lifetime). Nothing here is persisted. """ -import json import logging import os -from pathlib import Path - -from cryptography.fernet import Fernet, InvalidToken log = logging.getLogger(__name__) -CONFIG_DIR = Path(os.path.expanduser("~")) / ".bolodb" -CONFIG_FILE = CONFIG_DIR / "config.json" -SECRET_FILE = CONFIG_DIR / ".secret" -_DB_URL_KEY_FILE = CONFIG_DIR / "db_url.key" + +def _env_number(name, default, cast, minimum=None): + """Read a numeric env var without ever crashing process startup. + + A malformed value (e.g. ``ACTIVITY_CLEANUP_INTERVAL_HOURS=daily``) must not + abort import and take every route down with it — fall back to the default + and warn. ``minimum`` clamps the result so a misconfigured 0/negative + interval can't turn the cleanup loop into a busy loop. + """ + raw = os.environ.get(name) + if raw is None: + value = cast(default) + else: + try: + value = cast(raw) + except (TypeError, ValueError): + log.warning("Invalid %s=%r; falling back to default %r", name, raw, default) + value = cast(default) + if minimum is not None: + value = max(minimum, value) + return value + + +ACTIVITY_LOG_RETENTION_DAYS = _env_number( + "ACTIVITY_LOG_RETENTION_DAYS", 30, int, minimum=1 +) +# Periodic pruning of activity rows past the retention window. Safe to run +# in-process because BoloDB is deployed single-process; set the flag to "false" +# if that ever stops being true and the job moves to a dedicated worker. +ACTIVITY_CLEANUP_ENABLED = os.environ.get( + "ACTIVITY_CLEANUP_ENABLED", "true" +).lower() not in ("false", "0", "no") +ACTIVITY_CLEANUP_INTERVAL_HOURS = _env_number( + "ACTIVITY_CLEANUP_INTERVAL_HOURS", 24, float, minimum=0.1 +) DEFAULTS = { "openrouter_key": "", @@ -32,85 +61,26 @@ def default_config(): return dict(DEFAULTS) -def ensure_dir(): - CONFIG_DIR.mkdir(parents=True, exist_ok=True) - try: - os.chmod(CONFIG_DIR, 0o700) - except OSError: - pass - - def load_config(): - ensure_dir() - d = {} - if CONFIG_FILE.exists(): - try: - d = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) - except Exception: - pass - if not isinstance(d, dict): - d = {} - - # Strip legacy fields that no longer apply - changed = False - for key in ("provider", "model", "api_keys"): - if key in d: - d.pop(key) - changed = True - - cfg = {**default_config(), **d} - # API key is always read fresh from env, never from disk + """Build the in-memory config from the environment. Reads no files.""" + cfg = default_config() + # The API key is always read fresh from the environment, never stored. cfg["openrouter_key"] = os.environ.get("OPENROUTER_API_KEY", "") - - # Persist sanitized config if legacy fields were removed - if changed: - save_config(cfg) - return cfg def save_config(cfg): - ensure_dir() - # Never persist the API key - to_save = {k: v for k, v in cfg.items() if k != "openrouter_key"} - CONFIG_FILE.write_text(json.dumps(to_save, indent=2), encoding="utf-8") - try: - os.chmod(CONFIG_FILE, 0o600) - except OSError: - pass - - -def _db_url_fernet(): - CONFIG_DIR.mkdir(parents=True, exist_ok=True) - import base64 - import hashlib - - if _DB_URL_KEY_FILE.exists(): - secret = _DB_URL_KEY_FILE.read_text().strip() - else: - secret = base64.urlsafe_b64encode(os.urandom(32)).decode() - _DB_URL_KEY_FILE.write_text(secret) - key = base64.urlsafe_b64encode(hashlib.sha256(secret.encode()).digest()) - return Fernet(key) - - -def encrypt_db_url(url): - if not url: - return url - return _db_url_fernet().encrypt(url.encode()).decode() - + """No-op: config is in-memory only. -def decrypt_db_url(value): - if not value: - return value - try: - return _db_url_fernet().decrypt(value.encode()).decode() - except (InvalidToken, ValueError, TypeError): - return value + Kept so callers that update `last_db_url` don't have to special-case + persistence — mutating the shared `cfg` dict is the whole effect, and it + lasts exactly as long as the process, which is all that value needs. + """ + return None def public_config(cfg): """Config as exposed to the frontend — never includes the API key.""" return { - "last_db_url": decrypt_db_url(cfg.get("last_db_url", "")), + "last_db_url": cfg.get("last_db_url", ""), } diff --git a/backend/app/controllers/activity.py b/backend/app/controllers/activity.py new file mode 100644 index 00000000..95d446c9 --- /dev/null +++ b/backend/app/controllers/activity.py @@ -0,0 +1,186 @@ +import asyncio +import logging +from sqlalchemy import select, desc, delete, func +from datetime import datetime, timedelta, timezone +from backend.app.pgdatabase.engine import async_session +from backend.app.models.workspace_settings import WorkspaceSettings +from backend.app.models.activity import WorkspaceActivityLog +from backend.app.models.orm_user import User +from backend.app.pgdatabase.serialization import _to_uuid +from backend.app.config import ( + ACTIVITY_CLEANUP_INTERVAL_HOURS, + ACTIVITY_LOG_RETENTION_DAYS, +) + +log = logging.getLogger(__name__) + + +async def log_activity( + workspace_id: str, + actor_id: str | None, + event_type: str, + resource_type: str, + resource_id: str = None, + metadata: dict = None, +): + try: + wid = _to_uuid(workspace_id) + aid = _to_uuid(actor_id) if actor_id else None + except ValueError: + return + + async with async_session() as session: + log = WorkspaceActivityLog( + workspace_id=wid, + actor_id=aid, + event_type=event_type, + resource_type=resource_type, + resource_id=resource_id, + metadata_=metadata or {}, + ) + session.add(log) + await session.commit() + + +async def get_workspace_activity( + workspace_id: str, limit: int = 50, offset: int = 0, event_type: str = None +): + try: + wid = _to_uuid(workspace_id) + except ValueError: + return [] + + async with async_session() as session: + settings_res = await session.execute( + select(WorkspaceSettings).where(WorkspaceSettings.workspace_id == wid) + ) + settings = settings_res.scalar_one_or_none() + retention_days = getattr( + settings, "activity_retention_days", ACTIVITY_LOG_RETENTION_DAYS + ) + + cutoff_date = datetime.now(timezone.utc) - timedelta(days=retention_days) + + stmt = ( + select(WorkspaceActivityLog, User.email) + .outerjoin(User, WorkspaceActivityLog.actor_id == User.id) + .where(WorkspaceActivityLog.workspace_id == wid) + .where(WorkspaceActivityLog.created_at >= cutoff_date) + ) + + if event_type: + stmt = stmt.where(WorkspaceActivityLog.event_type == event_type) + + stmt = ( + stmt.order_by(desc(WorkspaceActivityLog.created_at)) + .limit(limit) + .offset(offset) + ) + result = await session.execute(stmt) + rows = result.all() + + return [ + { + "id": str(log.id), + "workspace_id": str(log.workspace_id), + "actor_id": str(log.actor_id) if log.actor_id else None, + "actor_email": email, + "event_type": log.event_type, + "resource_type": log.resource_type, + "resource_id": log.resource_id, + "metadata_": log.metadata_, + "created_at": log.created_at, + } + for log, email in rows + ] + + +# The UI route caps a page at 100 rows; an export should cover the whole +# retention window, so it pages through instead of taking a single big limit. +EXPORT_PAGE_SIZE = 500 +EXPORT_MAX_ROWS = 50_000 + + +async def iter_workspace_activity(workspace_id: str, event_type: str = None): + """Yield every retained activity row for a workspace, newest first.""" + offset = 0 + while offset < EXPORT_MAX_ROWS: + batch = await get_workspace_activity( + workspace_id, + limit=EXPORT_PAGE_SIZE, + offset=offset, + event_type=event_type, + ) + if not batch: + return + for row in batch: + yield row + if len(batch) < EXPORT_PAGE_SIZE: + return + offset += EXPORT_PAGE_SIZE + + +async def delete_old_activity(retention_days: int | None = None) -> int: + """Prune activity rows past the retention window. + + Reads beyond the window are already filtered out by + ``get_workspace_activity``; this reclaims the storage behind them. + + When ``retention_days`` is given it overrides every workspace's window. + Otherwise each workspace is pruned by its own ``activity_retention_days`` + setting, falling back to ``ACTIVITY_LOG_RETENTION_DAYS`` for workspaces + without a settings row. Returns the number of rows removed. + """ + async with async_session() as session: + if retention_days is not None: + cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) + stmt = delete(WorkspaceActivityLog).where( + WorkspaceActivityLog.created_at < cutoff + ) + else: + retention = func.coalesce( + select(WorkspaceSettings.activity_retention_days) + .where( + WorkspaceSettings.workspace_id == WorkspaceActivityLog.workspace_id + ) + .scalar_subquery(), + ACTIVITY_LOG_RETENTION_DAYS, + ) + stmt = delete(WorkspaceActivityLog).where( + WorkspaceActivityLog.created_at + < func.now() - func.make_interval(0, 0, 0, retention) + ) + + result = await session.execute(stmt) + await session.commit() + return result.rowcount or 0 + + +async def cleanup_old_activity_logs(db=None, retention_days: int | None = None) -> int: + """Alias function delegating to ``delete_old_activity`` or returning deleted count.""" + if isinstance(db, int) and retention_days is None: + retention_days = db + return await delete_old_activity(retention_days=retention_days) + + +async def activity_cleanup_loop(interval_hours: float | None = None): + """Run ``delete_old_activity`` forever on a fixed interval. + + Started from the app lifespan. Every failure is swallowed and retried on the + next tick — a pruning job must never be able to take the server down. + """ + interval = ( + interval_hours + if interval_hours is not None + else ACTIVITY_CLEANUP_INTERVAL_HOURS + ) * 3600 + while True: + try: + removed = await delete_old_activity() + if removed: + log.info("activity cleanup removed %d expired rows", removed) + except asyncio.CancelledError: + raise + except Exception: + log.exception("activity cleanup failed; retrying next interval") + await asyncio.sleep(interval) diff --git a/backend/app/controllers/auth.py b/backend/app/controllers/auth.py index 7e2f81e8..4b168a87 100644 --- a/backend/app/controllers/auth.py +++ b/backend/app/controllers/auth.py @@ -6,6 +6,7 @@ from pydantic import EmailStr from fastapi import HTTPException from backend.app.models.user import ( + UpdateProfile, UserSignup, UserInDB, Role, @@ -19,7 +20,7 @@ update_user, UserAlreadyExistsError, ) -from backend.app.pgdatabase.models import PasswordResetToken +from backend.app.models.auth_token import PasswordResetToken from backend.app.pgdatabase.engine import get_engine import jwt from jwt import PyJWKClient @@ -56,6 +57,41 @@ async def get_me(user_id): return data +async def update_profile(user_id: str, req: UpdateProfile): + fields = req.model_dump(mode="json", exclude_unset=True) + new_email = fields.get("email") + + if new_email: + existing = await get_user_by_email(new_email) + if existing and str(existing["_id"]) != str(user_id): + raise HTTPException( + status_code=409, detail="An account with this email already exists" + ) + + current = await get_me(user_id) + if current and current.get("email") != new_email: + fields["email_verified"] = False + + if fields: + from sqlalchemy.exc import IntegrityError + + try: + await update_user(user_id, **fields) + except (IntegrityError, UserAlreadyExistsError): + raise HTTPException( + status_code=409, detail="An account with this email already exists" + ) + + if new_email and fields.get("email_verified") is False: + from backend.app.pgdatabase.otp import create_otp + from backend.app.services.email import send_verification_email + + otp_code = await create_otp(user_id, purpose="signup") + await send_verification_email(new_email, otp_code) + + return await get_me(user_id) + + async def login(email: EmailStr, password: str): user_details = await get_user_by_email(email) if user_details is None: diff --git a/backend/app/controllers/catalog.py b/backend/app/controllers/catalog.py index 8306783e..23a00ddb 100644 --- a/backend/app/controllers/catalog.py +++ b/backend/app/controllers/catalog.py @@ -11,31 +11,35 @@ log = logging.getLogger(__name__) -async def get_catalog(user_id, db, kb): +async def get_catalog(workspace_id, db, kb, db_id=None): """Return the stored catalog for the user's connected database.""" - if not db.connected(user_id): + if not db.connected(workspace_id, db_id): raise HTTPException(409, "No database connected") - return {"catalog": await kb.get_catalog(user_id, db.get_db_id(user_id))} + return { + "catalog": await kb.get_catalog(workspace_id, db.get_db_id(workspace_id, db_id)) + } -async def save_catalog(user_id, db, kb, payload): +async def save_catalog(workspace_id, db, kb, payload, db_id=None): """Persist ``payload`` as the connected database's catalog.""" - if not db.connected(user_id): + if not db.connected(workspace_id, db_id): raise HTTPException(409, "No database connected") - await kb.set_catalog(user_id, db.get_db_id(user_id), payload.model_dump()) + await kb.set_catalog( + workspace_id, db.get_db_id(workspace_id, db_id), payload.model_dump() + ) return {"ok": True} -async def suggest(user_id, db, providers): +async def suggest(workspace_id, db, providers, db_id=None): """Suggest a catalog: the deterministic backbone (joins + value maps from the schema) enriched by the LLM (descriptions, metrics, synonyms, labels). The LLM step degrades gracefully — a failure still returns the backbone.""" - if not db.connected(user_id): + if not db.connected(workspace_id, db_id): raise HTTPException(409, "No database connected") - base = suggest_from_schema(db.get_schema(user_id)) + base = suggest_from_schema(db.get_schema(workspace_id, db_id=db_id)) try: enriched = await llm_suggest_catalog( - providers.get(user_id), db.schema_as_text(user_id) + providers.get(workspace_id), db.schema_as_text(workspace_id, db_id) ) except Exception: # Graceful degradation: still return the schema-only backbone, but log diff --git a/backend/app/controllers/conversations.py b/backend/app/controllers/conversations.py index 87b3d24c..bd487d81 100644 --- a/backend/app/controllers/conversations.py +++ b/backend/app/controllers/conversations.py @@ -5,25 +5,27 @@ log = logging.getLogger(__name__) -async def list_conversations(user_id): - return await mdb.get_conversations(user_id) +async def list_conversations(workspace_id, user_id): + return await mdb.get_conversations(workspace_id, user_id) -async def create_conversation(user_id, title="", database_id=None): - return await mdb.create_conversation(user_id, title=title, database_id=database_id) +async def create_conversation(workspace_id, user_id, title="", database_id=None): + return await mdb.create_conversation( + workspace_id, user_id, title=title, database_id=database_id + ) -async def get_conversation(user_id, conversation_id): - return await mdb.get_conversation(user_id, conversation_id) +async def get_conversation(workspace_id, user_id, conversation_id): + return await mdb.get_conversation(workspace_id, user_id, conversation_id) -async def rename_conversation(user_id, conversation_id, title): - return await mdb.rename_conversation(user_id, conversation_id, title) +async def rename_conversation(workspace_id, user_id, conversation_id, title): + return await mdb.rename_conversation(workspace_id, user_id, conversation_id, title) -async def delete_conversation(user_id, conversation_id): - return await mdb.delete_conversation(user_id, conversation_id) +async def delete_conversation(workspace_id, user_id, conversation_id): + return await mdb.delete_conversation(workspace_id, user_id, conversation_id) -async def clear_conversations(user_id): - return await mdb.clear_conversations(user_id) +async def clear_conversations(workspace_id, user_id): + return await mdb.clear_conversations(workspace_id, user_id) diff --git a/backend/app/controllers/dashboards.py b/backend/app/controllers/dashboards.py new file mode 100644 index 00000000..1e33108e --- /dev/null +++ b/backend/app/controllers/dashboards.py @@ -0,0 +1,174 @@ +import logging +import asyncio +from backend.app.pgdatabase import dashboards as mdb_dash +from backend.app.pgdatabase import saved_queries as mdb_sq + +log = logging.getLogger(__name__) + +# --- Saved Queries --- + + +async def create_saved_query(workspace_id: str, created_by: str, data: dict): + return await mdb_sq.create_saved_query(workspace_id, created_by, **data) + + +async def list_saved_queries(workspace_id: str, limit: int = 50, offset: int = 0): + return await mdb_sq.list_saved_queries(workspace_id, limit, offset) + + +async def get_saved_query(workspace_id: str, query_id: str): + return await mdb_sq.get_saved_query(workspace_id, query_id) + + +async def update_saved_query(workspace_id: str, query_id: str, data: dict): + return await mdb_sq.update_saved_query(workspace_id, query_id, **data) + + +async def delete_saved_query(workspace_id: str, query_id: str): + return await mdb_sq.delete_saved_query(workspace_id, query_id) + + +# --- Dashboards --- + + +async def create_dashboard( + workspace_id: str, created_by: str, name: str, description: str = None +): + return await mdb_dash.create_dashboard(workspace_id, created_by, name, description) + + +async def list_dashboards(workspace_id: str): + return await mdb_dash.list_dashboards(workspace_id) + + +async def get_dashboard(workspace_id: str, dashboard_id: str): + return await mdb_dash.get_dashboard(workspace_id, dashboard_id) + + +async def update_dashboard(workspace_id: str, dashboard_id: str, data: dict): + return await mdb_dash.update_dashboard(workspace_id, dashboard_id, **data) + + +async def delete_dashboard(workspace_id: str, dashboard_id: str): + return await mdb_dash.delete_dashboard(workspace_id, dashboard_id) + + +# --- Panels --- + + +async def add_panel(workspace_id: str, dashboard_id: str, data: dict): + return await mdb_dash.add_panel(workspace_id, dashboard_id, **data) + + +async def update_panel(workspace_id: str, dashboard_id: str, panel_id: str, data: dict): + return await mdb_dash.update_panel(workspace_id, dashboard_id, panel_id, **data) + + +async def delete_panel(workspace_id: str, dashboard_id: str, panel_id: str): + return await mdb_dash.delete_panel(workspace_id, dashboard_id, panel_id) + + +async def update_panels_batch(workspace_id: str, dashboard_id: str, updates: list): + return await mdb_dash.update_panels_batch(workspace_id, dashboard_id, updates) + + +# --- Execution --- + + +def _normalize_columns(columns, rows): + """ + Return columns as [{"name", "type_name"}] whatever shape they were stored in. + + Saved queries persist `columns` as a plain list of names (that is what the + chat turn carries), while a fresh execution only knows the names too. The + dashboard renderer picks its value axis by type, so infer the type from the + values actually returned rather than handing back bare strings. + """ + out = [] + for col in columns or []: + if isinstance(col, dict): + name = col.get("name") + type_name = col.get("type_name") + else: + name = col + type_name = None + if name is None: + continue + if not type_name: + type_name = _infer_type(name, rows) + out.append({"name": name, "type_name": type_name}) + return out + + +def _infer_type(name, rows): + for row in (rows or [])[:20]: + val = row.get(name) if isinstance(row, dict) else None + if val is None or val == "": + continue + if isinstance(val, bool): + return "string" + if isinstance(val, int): + return "integer" + if isinstance(val, float): + return "float" + try: + float(str(val).replace(",", "").strip()) + return "float" + except (TypeError, ValueError): + return "string" + return "string" + + +async def execute_dashboard_queries(workspace_id: str, dashboard_id: str, db_manager): + dash = await mdb_dash.get_dashboard(workspace_id, dashboard_id) + if not dash: + return None + + panel_sq_ids = [] + for p in dash["panels"]: + if p.get("saved_query_id"): + panel_sq_ids.append(p["saved_query_id"]) + + results = {} + sem = asyncio.Semaphore(5) + + async def run_sq(sq_id): + sq = await mdb_sq.get_saved_query(workspace_id, str(sq_id)) + if not sq: + return None + sq_key = str(sq.get("id") or sq.get("_id") or sq_id) + try: + sql = sq.get("sql") + target_db_id = sq.get("database_id") + if sql: + async with sem: + data = await asyncio.to_thread( + db_manager.execute, workspace_id, sql, target_db_id + ) + + # data comes back as dict: {"columns": [], "rows": [{}], "error": ""} + if data.get("error"): + return {"id": sq_key, "error": data["error"]} + + # Charts assume array of objects keyed by column name + rows = data.get("rows", [])[:500] + # Trust the live result's column list — a saved query's stored + # columns can be stale if the underlying SQL was edited. + return { + "id": sq_key, + "rows": rows, + "columns": _normalize_columns(data.get("columns", []), rows), + } + except Exception as e: + log.warning(f"Dashboard panel query failed sq_id={sq_id}: {e}") + return {"id": sq_key, "error": str(e)} + return None + + tasks = [run_sq(sq_id) for sq_id in set(panel_sq_ids)] + fetched = await asyncio.gather(*tasks) + + for res in fetched: + if res: + results[res["id"]] = res + + return results diff --git a/backend/app/controllers/database.py b/backend/app/controllers/database.py index b8838d01..d3113a0a 100644 --- a/backend/app/controllers/database.py +++ b/backend/app/controllers/database.py @@ -1,101 +1,248 @@ from fastapi import HTTPException -from backend.app.database import sanitize_url -from backend.sample_data import ensure_sample_db +from backend.app.database import sanitize_url, db_id_for +from backend.sample_data import ensure_sample_db, sample_db_path import backend.app.pgdatabase as mdb +from backend.app.pgdatabase.connections import ConnectionKeyError import logging +from backend.app.controllers.activity import log_activity logger = logging.getLogger(__name__) +# A server-misconfiguration message, distinct from a transient save failure: +# every save fails the same way until an operator sets RECENT_CONNECTIONS_KEY, +# and reconnecting existing databases fails too — so "try again" would mislead. +_KEY_ERROR_MESSAGE = ( + "Connected, but saved connections aren't configured on the server " + "(RECENT_CONNECTIONS_KEY is missing). Ask an administrator to set it — " + "until then this database won't be remembered." +) -async def connect(db, kb, cfg, req_data, user_id=None): +# Shown wherever a database needs a name — header, switcher, connect screen. +SAMPLE_ALIAS = "Sample Webshop" + + +def _call_with_db_id(fn, workspace_id, db_id): + """Call a DatabaseManager accessor, tolerating single-database managers. + + Not every manager the app is handed takes a `db_id` (test doubles, and the + single-connection manager this grew out of), so fall back to the one-argument + form rather than requiring every implementation to grow the parameter. """ - Connects to a database and enriches the connection details with knowledge-base information. + if db_id is not None: + try: + return fn(workspace_id, db_id=db_id) + except TypeError: + pass + try: + return fn(workspace_id, db_id) + except TypeError: + pass + return fn(workspace_id) - Parameters: - req_data: Request data containing the database URL. - user_id: Optional identifier of the user establishing the connection. - Returns: - The connection result with trust, glossary, knowledge availability, and starter questions. +def _sample_db_id() -> str: + """The db_id the sample database connects under, without building it. - Raises: - HTTPException: If the database connection fails. + `db_id` is a hash of the connection URL, so this matches whatever + `db.connect(ensure_sample_db())` would produce — letting us recognise a + request for the sample before deciding whether to rebuild it. """ - result = db.connect(user_id, req_data.db_url) + return db_id_for(f"sqlite:///{sample_db_path().as_posix()}") + + +def _connect_sample_url(db, workspace_id): + """Build (if needed) and connect the sample database. Returns its db_id.""" + url = ensure_sample_db() + result = db.connect(workspace_id, url) + if not result["ok"]: + logger.warning("Could not connect the sample database: %s", result["error"]) + return None + logger.info("Restored the sample database for workspace") + return result["db_id"] + + +async def ensure_connection(db, workspace_id, db_id=None): + """Return the db_id this workspace is connected to, reconnecting if needed. + + Live engines live in process memory, so a restart, a redeploy or a request + landing on a different worker used to look exactly like the user having + disconnected — they were bounced back to the connect screen with their + database apparently gone. The credentials are already stored against the + workspace, so re-establish the connection from them instead. + + Returns None when the workspace has no usable stored connection. + """ + if not workspace_id: + return None + if _call_with_db_id(db.connected, workspace_id, db_id): + # An explicitly requested database that is connected *is* the answer — + # asking the manager to name it again would let a manager that ignores + # db_id hand back the workspace default instead. + return db_id or db.get_db_id(workspace_id) + + try: + if db_id: + conn = await mdb.get_recent_connection_by_db_id(workspace_id, db_id) + else: + conn = await mdb.get_latest_recent_connection(workspace_id) + except RuntimeError: + # Undecryptable stored credentials — the user has to re-add the + # connection, and /api/reconnect reports that explicitly. + logger.exception("Could not decrypt stored connection for workspace") + conn = None + + if not conn or not conn.get("db_url"): + # The sample database needs no stored credentials — it's regenerable + # from the vendored dump. So when the request explicitly targets the + # sample and there's nothing to restore, rebuild it rather than + # reporting no database. This keeps "try the sample" working with zero + # configuration, even if encryption isn't set up or the worker + # restarted and lost the live connection. A blank db_id is left alone so + # nothing silently auto-connects the sample. + if db_id and db_id == _sample_db_id(): + return _connect_sample_url(db, workspace_id) + return None + + result = db.connect(workspace_id, conn["db_url"]) + if not result["ok"]: + logger.warning( + "Could not restore connection %s: %s", conn["db_id"], result["error"] + ) + return None + logger.info("Restored stored connection %s from saved credentials", result["db_id"]) + return result["db_id"] + + +async def connect(db, kb, cfg, req_data, workspace_id=None, user_id=None): + result = db.connect(workspace_id, req_data.db_url) if not result["ok"]: raise HTTPException(400, result["error"]) db_id = result["db_id"] - result["trust"] = await kb.trust_level(user_id, db_id) - result["glossary"] = await kb.get_glossary(user_id, db_id) - result["has_knowledge"] = (await kb.count_verified(user_id, db_id)) > 0 + result["trust"] = await kb.trust_level(workspace_id, db_id) + result["glossary"] = await kb.get_glossary(workspace_id, db_id) + result["has_knowledge"] = (await kb.count_verified(workspace_id, db_id)) > 0 result["starters"] = [ - v["question"] for v in (await kb.get_verified(user_id, db_id))[:6] + v["question"] for v in (await kb.get_verified(workspace_id, db_id))[:6] ] - if user_id: + alias_name = getattr(req_data, "alias_name", None) or None + result["alias_name"] = alias_name + + if workspace_id: try: await mdb.save_recent_connection( - user_id=user_id, + workspace_id=workspace_id, db_url=req_data.db_url, display_url=sanitize_url(req_data.db_url), dialect=result["dialect"], db_id=result["db_id"], table_count=result["tables"], + alias_name=alias_name, + ) + if not alias_name: + # An existing row keeps its alias when we don't send one, so + # read it back — the header and switcher label from this. + saved = await mdb.get_recent_connection_by_db_id(workspace_id, db_id) + result["alias_name"] = saved.get("alias_name") if saved else None + except ConnectionKeyError: + logger.exception("Recent-connection encryption key is not configured") + result["save_error"] = _KEY_ERROR_MESSAGE + except Exception: + # The connection itself is live; only its persistence failed. Say so + # rather than letting it quietly vanish from the connect screen. + logger.exception("Failed to save recent connection") + result["save_error"] = ( + "Connected, but this database couldn't be saved to your workspace — " + "you may have to reconnect it next time." ) - except Exception as e: - logger.warning("Failed to save recent connection: %s", e) + await log_activity( + workspace_id, + user_id, + "db.connected", + "connection", + str(result.get("db_id", "")), + {"db_url": sanitize_url(req_data.db_url), "dialect": result.get("dialect")}, + ) return result -async def connect_sample(db, kb, cfg, user_id=None): +async def connect_sample(db, kb, cfg, workspace_id=None, user_id=None): """ Connect to the sample database and return its connection details and knowledge metadata. Parameters: - user_id: Optional identifier of the user establishing the connection. + workspace_id: Identifier of the workspace establishing the connection. Returns: dict: Connection details enriched with trust, glossary, knowledge availability, starter questions, and a sample-database flag. """ url = ensure_sample_db() - result = db.connect(user_id, url) + result = db.connect(workspace_id, url) if not result["ok"]: raise HTTPException(500, result["error"]) - dbid = db.get_db_id(user_id) - await kb.seed_sample(user_id, dbid) + dbid = db.get_db_id(workspace_id) + await kb.seed_sample(workspace_id, dbid) - result["trust"] = await kb.trust_level(user_id, dbid) - result["glossary"] = await kb.get_glossary(user_id, dbid) - result["has_knowledge"] = (await kb.count_verified(user_id, dbid)) > 0 + result["trust"] = await kb.trust_level(workspace_id, dbid) + result["glossary"] = await kb.get_glossary(workspace_id, dbid) + result["has_knowledge"] = (await kb.count_verified(workspace_id, dbid)) > 0 result["starters"] = [ - v["question"] for v in (await kb.get_verified(user_id, dbid))[:6] + v["question"] for v in (await kb.get_verified(workspace_id, dbid))[:6] ] result["is_sample"] = True - - if user_id: + result["alias_name"] = SAMPLE_ALIAS + + if workspace_id: + # Preserve a custom alias the user set on a previous connect; only + # brand-new sample connections get the default SAMPLE_ALIAS. Mirrors + # connect()'s alias handling so reconnecting never clobbers a rename. + existing = await mdb.get_recent_connection_by_db_id( + workspace_id, result["db_id"] + ) + alias_name = (existing.get("alias_name") if existing else None) or SAMPLE_ALIAS + result["alias_name"] = alias_name try: await mdb.save_recent_connection( - user_id=user_id, + workspace_id=workspace_id, db_url=url, display_url=sanitize_url(url), dialect=result["dialect"], db_id=result["db_id"], table_count=result["tables"], + alias_name=alias_name, + ) + except ConnectionKeyError: + logger.exception("Recent-connection encryption key is not configured") + result["save_error"] = _KEY_ERROR_MESSAGE + except Exception: + logger.exception("Failed to save recent connection") + result["save_error"] = ( + "Connected, but the sample database couldn't be saved to your workspace." ) - except Exception as e: - logger.warning("Failed to save recent connection: %s", e) + await log_activity( + workspace_id, + user_id, + "db.connected", + "connection", + str(result.get("db_id", "")), + {"is_sample": True, "dialect": result.get("dialect")}, + ) return result -async def disconnect(user_id, db, cfg): - db.disconnect(user_id) +async def disconnect(workspace_id, db, cfg, db_id=None, user_id=None): + db.disconnect(workspace_id, db_id) + await log_activity( + workspace_id, user_id, "db.disconnected", "connection", db_id, {} + ) return {"ok": True} -async def get_schema(user_id, db, refresh): - if not db.connected(user_id): +async def get_schema(workspace_id, db, refresh, db_id=None): + actual_db_id = await ensure_connection(db, workspace_id, db_id) + if not actual_db_id: raise HTTPException(409, "No database connected") - return db.get_schema(user_id, refresh=refresh) + return db.get_schema(workspace_id, db_id=actual_db_id, refresh=refresh) diff --git a/backend/app/controllers/onboard.py b/backend/app/controllers/onboard.py index 3f473bd7..dcd6de81 100644 --- a/backend/app/controllers/onboard.py +++ b/backend/app/controllers/onboard.py @@ -4,13 +4,13 @@ import logging from backend.app.semantic import suggest_from_schema from backend.app.llm import generate_starters -from backend.app.pgdatabase.models import VerifiedQA +from backend.app.models.catalog import VerifiedQA from sqlalchemy.dialects.postgresql import insert log = logging.getLogger(__name__) -async def save(user_id, db, kb, req_data): +async def save(workspace_id, db, kb, req_data): """ Persist onboarding results and initialize the semantic catalog when it is empty. @@ -20,16 +20,16 @@ async def save(user_id, db, kb, req_data): Returns: A dictionary containing `ok` set to `True` and the user's updated trust level. """ - if not db.connected(user_id): + if not db.connected(workspace_id): raise HTTPException(409, "No database connected") - db_id = db.get_db_id(user_id) + db_id = db.get_db_id(workspace_id) if req_data.glossary is not None: await kb.set_glossary( - user_id, db_id, [g.model_dump() for g in req_data.glossary] + workspace_id, db_id, [g.model_dump() for g in req_data.glossary] ) for s in req_data.starters: await kb.add_verified( - user_id, + workspace_id, db_id, s.question, s.sql, @@ -39,25 +39,27 @@ async def save(user_id, db, kb, req_data): # join paths and value-map scaffolding derived straight from the schema — # so linking and the prompt benefit immediately. The user can enrich it # with AI suggestions and edits from Settings afterwards. - if await kb.catalog_is_empty(user_id, db_id): + if await kb.catalog_is_empty(workspace_id, db_id): await kb.set_catalog( - user_id, db_id, suggest_from_schema(db.get_schema(user_id)) + workspace_id, db_id, suggest_from_schema(db.get_schema(workspace_id)) ) - return {"ok": True, "trust": await kb.trust_level(user_id, db_id)} + return {"ok": True, "trust": await kb.trust_level(workspace_id, db_id)} -async def generate_starters_async(user_id, db, kb, providers): - if not db.connected(user_id): +async def generate_starters_async(workspace_id, db, kb, providers): + if not db.connected(workspace_id): raise HTTPException(409, "No database connected") try: - db_id = db.get_db_id(user_id) - schema_text = db.schema_as_text(user_id) - dialect = db.get_dialect(user_id) + db_id = db.get_db_id(workspace_id) + schema_text = db.schema_as_text(workspace_id) + dialect = db.get_dialect(workspace_id) - starters = await generate_starters(providers.get(user_id), schema_text, dialect) + starters = await generate_starters( + providers.get(workspace_id), schema_text, dialect + ) async def _run_starter(s): - res = await run_in_threadpool(db.execute, user_id, s.get("sql", "")) + res = await run_in_threadpool(db.execute, workspace_id, s.get("sql", "")) s["error"] = res.get("error") return s @@ -79,7 +81,7 @@ async def _run_starter(s): .values( [ { - "user_id": user_id, + "workspace_id": workspace_id, "db_id": db_id, "question": s["question"], "sql": s["sql"], @@ -89,7 +91,7 @@ async def _run_starter(s): ] ) .on_conflict_do_nothing( - index_elements=["user_id", "db_id", "question"] + index_elements=["workspace_id", "db_id", "question"] ) ) await session.execute(stmt) diff --git a/backend/app/controllers/query.py b/backend/app/controllers/query.py index c2496a04..4ad218b2 100644 --- a/backend/app/controllers/query.py +++ b/backend/app/controllers/query.py @@ -17,7 +17,14 @@ from fastapi import HTTPException from fastapi.concurrency import run_in_threadpool -from backend.app.llm import LLMError, explain_sql, generate_sql, shortlist_tables +from backend.app.llm import ( + DEFAULT_CHART, + LLMError, + explain_sql, + generate_sql, + parse_chart_spec, + shortlist_tables, +) from backend.app.repair import run_repair_loop, schema_validator from backend.app.schema_link import ( compact_schema, @@ -52,6 +59,7 @@ def _failure_payload(message, tables=None): "sql": "", "restatement": "", "assumptions": [], + "chart": dict(DEFAULT_CHART), "confidence": "low", "confidence_reason": message, "based_on_verified": False, @@ -65,12 +73,81 @@ def _failure_payload(message, tables=None): } -async def run_query(user_id, db, kb, cfg, providers, session_log, req_data): +async def _resolve_db_id(db, workspace_id, db_id=None): + """The database this request runs against, or None if there isn't one. + + Two jobs, both of which every entry point below needs: + + - Restore the connection if this process doesn't hold one. Engines live in + process memory, so a restart or a request served by a different worker + leaves none; `ensure_connection` re-establishes it from the credentials + stored against the workspace rather than failing the request. + - Resolve `X-Db-Id` to a concrete id *once*, so everything downstream — + schema, execution, knowledge, logging — agrees on which database this is. + Reading `db.get_db_id(workspace_id)` separately further down is what let + knowledge and query logs drift onto the workspace default while the SQL + ran against the selected database. + """ + from backend.app.controllers.database import ensure_connection + + return await ensure_connection(db, workspace_id, db_id) + + +def _db_get_schema(db, workspace_id, db_id=None): + if db_id is not None: + try: + return db.get_schema(workspace_id, db_id=db_id) + except TypeError: + pass + try: + return db.get_schema(workspace_id, db_id) + except TypeError: + pass + return db.get_schema(workspace_id) + + +def _db_get_dialect(db, workspace_id, db_id=None): + if db_id is not None: + try: + return db.get_dialect(workspace_id, db_id=db_id) + except TypeError: + pass + try: + return db.get_dialect(workspace_id, db_id) + except TypeError: + pass + return db.get_dialect(workspace_id) + + +def _db_execute(db, workspace_id, sql, db_id=None): + if db_id is not None: + try: + return db.execute(workspace_id, sql, db_id=db_id) + except TypeError: + pass + try: + return db.execute(workspace_id, sql, db_id) + except TypeError: + pass + return db.execute(workspace_id, sql) + + +async def run_query( + workspace_id, + db, + kb, + cfg, + providers, + session_log, + req_data, + db_id=None, + user_id=None, +): """ Generate, validate, and execute SQL for a user's question, with confidence scoring and repair attempts. Parameters: - user_id: Identifier of the user whose database and knowledge base are used. + workspace_id: Workspace scoping unit whose database and knowledge base are used. db: Database connection and schema provider. kb: Knowledge base used for glossary, catalog, and verified-answer retrieval. cfg: Application configuration. @@ -84,7 +161,8 @@ async def run_query(user_id, db, kb, cfg, providers, session_log, req_data): Raises: HTTPException: If no database is connected or the question is empty. """ - if not db.connected(user_id): + db_id = await _resolve_db_id(db, workspace_id, db_id) + if not db_id: raise HTTPException(409, "No database connected") q = req_data.question.strip() if not q: @@ -92,22 +170,21 @@ async def run_query(user_id, db, kb, cfg, providers, session_log, req_data): context = req_data.context # Step 1 — user knowledge: confirmed term meanings + similar verified answers. - db_id = db.get_db_id(user_id) - glossary = await kb.get_glossary(user_id, db_id) - catalog = await kb.get_catalog(user_id, db_id) - retrieved = await kb.retrieve_similar(user_id, db_id, q, k=3) + glossary = await kb.get_glossary(workspace_id, db_id) + catalog = await kb.get_catalog(workspace_id, db_id) + retrieved = await kb.retrieve_similar(workspace_id, db_id, q, k=3) # Step 2 — schema linking: budget for the configured model, then pick tables. budget = model_budget() - full_schema = db.get_schema(user_id) - dialect = db.get_dialect(user_id) + full_schema = _db_get_schema(db, workspace_id, db_id) + dialect = _db_get_dialect(db, workspace_id, db_id) context_tables = ( extract_table_names_from_prev_query(context[-1].sql, dialect) if context else set() ) try: - provider = providers.get(user_id) + provider = providers.get(workspace_id) except LLMError as e: raise HTTPException(503, e.user_message) @@ -158,7 +235,7 @@ async def _generate(feedback): ) async def _execute(sql): - return await run_in_threadpool(db.execute, user_id, sql) + return await run_in_threadpool(_db_execute, db, workspace_id, sql, db_id) def _on_failure(sql, errors): nonlocal linked @@ -203,6 +280,7 @@ def _on_failure(sql, errors): "sql": loop_result["sql"], "restatement": loop_result["restatement"], "assumptions": loop_result.get("assumptions", []), + "chart": parse_chart_spec(loop_result.get("chart")), "confidence": confidence, "confidence_reason": reason, "based_on_verified": based, @@ -221,25 +299,30 @@ def _on_failure(sql, errors): return out -async def explain(user_id, db, providers, req_data): +async def explain(workspace_id, db, providers, req_data, db_id=None): """Translate a SQL query into plain English (trust feature).""" - if not db.connected(user_id): + db_id = await _resolve_db_id(db, workspace_id, db_id) + if not db_id: raise HTTPException(409, "No database connected") sql = (req_data.sql or "").strip() if not sql: raise HTTPException(400, "Empty SQL") try: - return await explain_sql(providers.get(user_id), sql, db.get_dialect(user_id)) + return await explain_sql( + providers.get(workspace_id), + sql, + _db_get_dialect(db, workspace_id, db_id), + ) except LLMError as e: raise HTTPException(502, e.user_message) -async def feedback(user_id, db, kb, session_log, req_data): +async def feedback(workspace_id, db, kb, session_log, req_data, db_id=None): """ Record feedback for a query and update verified knowledge when the feedback is positive. Parameters: - user_id: Identifier of the user submitting the feedback. + workspace_id: Workspace scoping unit submitting the feedback. db: Database connection manager used to identify the connected database. kb: Knowledge base used to store verified queries and retrieve trust information. session_log: Session logger used to record the feedback. @@ -251,31 +334,37 @@ async def feedback(user_id, db, kb, session_log, req_data): Raises: HTTPException: If no database is connected for the user. """ - if not db.connected(user_id): + db_id = await _resolve_db_id(db, workspace_id, db_id) + if not db_id: raise HTTPException(409, "No database connected") session_log.log_feedback(req_data.query_id, req_data.verdict, req_data.reason) + # Verified knowledge belongs to the database the answer was checked against. + # Writing it under the workspace default instead would teach the wrong + # database this SQL is correct, and hand back that database's trust score. if req_data.verdict == "correct": await kb.add_verified( - user_id, - db.get_db_id(user_id), + workspace_id, + db_id, req_data.question, req_data.sql, req_data.restatement, ) - out = {"ok": True, "trust": await kb.trust_level(user_id, db.get_db_id(user_id))} + out = { + "ok": True, + "trust": await kb.trust_level(workspace_id, db_id), + } if req_data.verdict == "correct": out["starters"] = [ - v["question"] - for v in (await kb.get_verified(user_id, db.get_db_id(user_id)))[:6] + v["question"] for v in (await kb.get_verified(workspace_id, db_id))[:6] ] return out -async def verify(user_id, db, kb, req_data): +async def verify(workspace_id, db, kb, req_data, db_id=None): """Mark a question and its SQL explanation as verified. Parameters: - user_id: Identifier of the user verifying the query. + workspace_id: Workspace scoping unit verifying the query. db: Database connection used to determine the active database. kb: Knowledge base used to store the verified query and calculate trust. req_data: Request containing the question, SQL statement, and restatement. @@ -286,24 +375,28 @@ async def verify(user_id, db, kb, req_data): Raises: HTTPException: If no database is connected for the user. """ - if not db.connected(user_id): + db_id = await _resolve_db_id(db, workspace_id, db_id) + if not db_id: raise HTTPException(409, "No database connected") await kb.add_verified( - user_id, - db.get_db_id(user_id), + workspace_id, + db_id, req_data.question, req_data.sql, req_data.restatement, ) - return {"ok": True, "trust": await kb.trust_level(user_id, db.get_db_id(user_id))} + return { + "ok": True, + "trust": await kb.trust_level(workspace_id, db_id), + } -async def execute(user_id, db, req_data): +async def execute(workspace_id, db, req_data, db_id=None): """ Execute the requested SQL statement against the user's connected database. Parameters: - user_id: Identifier of the user whose database connection should be used. + workspace_id: Workspace scoping unit whose database connection should be used. db: Database service used to execute the statement. req_data: Request containing the SQL statement. @@ -313,9 +406,10 @@ async def execute(user_id, db, req_data): Raises: HTTPException: If no database is connected or the SQL execution returns an error. """ - if not db.connected(user_id): + db_id = await _resolve_db_id(db, workspace_id, db_id) + if not db_id: raise HTTPException(409, "No database connected") - res = await run_in_threadpool(db.execute, user_id, req_data.sql) + res = await run_in_threadpool(db.execute, workspace_id, req_data.sql, db_id) if "error" in res: raise HTTPException(400, res["error"]) return res @@ -399,12 +493,22 @@ def _build_checks(verdict, schema=None): # ── Streaming query controller ──────────────────────────────────── -async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_data): +async def run_query_stream( + workspace_id, + db, + kb, + cfg, + providers, + session_log, + req_data, + db_id=None, + user_id=None, +): """ Stream the SQL generation, validation, execution, confidence, and result events for a question. Parameters: - user_id: Identifier of the requesting user. + workspace_id: Workspace scoping unit. db: Database connection and schema provider. kb: Knowledge base used for glossary, catalog, and verified-answer retrieval. cfg: Application configuration. @@ -416,7 +520,8 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat dict: Progress, error, or final-result event. The final result includes generated SQL, execution data, confidence information, and the recorded query identifier. """ - if not db.connected(user_id): + db_id = await _resolve_db_id(db, workspace_id, db_id) + if not db_id: yield {"kind": "error", "message": "No database connected"} return q = req_data.question.strip() @@ -426,14 +531,18 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat context = req_data.context query_start = time.monotonic() - dbid = db.get_db_id(user_id) - glossary = await kb.get_glossary(user_id, dbid) - catalog = await kb.get_catalog(user_id, dbid) - retrieved = await kb.retrieve_similar(user_id, dbid, q, k=3) + # Knowledge is scoped to the database being queried. Reading it from the + # workspace default while the SQL runs against the selected database fed + # the model another database's glossary, catalog and verified examples. + glossary = await kb.get_glossary(workspace_id, db_id) + catalog = await kb.get_catalog(workspace_id, db_id) + retrieved = await kb.retrieve_similar(workspace_id, db_id, q, k=3) budget = model_budget() - full_schema = db.get_schema(user_id) + full_schema = _db_get_schema(db, workspace_id, db_id) context_tables = ( - extract_table_names_from_prev_query(context[-1].sql, db.get_dialect(user_id)) + extract_table_names_from_prev_query( + context[-1].sql, _db_get_dialect(db, workspace_id, db_id) + ) if context else set() ) @@ -450,7 +559,7 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat # Only the catalog entries for the linked tables go into the prompt. prompt_catalog = filter_catalog(catalog, tables) try: - provider_obj = providers.get(user_id) + provider_obj = providers.get(workspace_id) except LLMError as e: # Surface the configuration problem (e.g. missing API key) instead of # letting it escape the generator and get masked as an internal error. @@ -469,6 +578,7 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat max_iterations = _MAX_ATTEMPTS feedback = "" last_restatement = "" + last_chart = dict(DEFAULT_CHART) exec_result = None exec_elapsed = 0 @@ -477,7 +587,7 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat provider_obj, q, schema_text, - db.get_dialect(user_id), + _db_get_dialect(db, workspace_id, db_id), glossary, retrieved, budget["max_examples"], @@ -518,14 +628,18 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat sql = (gen_result.get("sql") or "").strip() restatement = (gen_result.get("restatement") or "").strip() last_restatement = restatement + last_chart = parse_chart_spec(gen_result.get("chart")) if not sql: yield {"kind": "error", "message": "Model returned empty SQL"} return yield {"kind": "sql", "attempt": attempt, "sql": sql} + yield {"kind": "chart", "attempt": attempt, "chart": last_chart} - verdict = validate_sql(sql, full_schema, db.get_dialect(user_id)) + verdict = validate_sql( + sql, full_schema, _db_get_dialect(db, workspace_id, db_id) + ) checks = _build_checks(verdict, full_schema) passed = verdict.get("ok", False) @@ -567,7 +681,9 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat # a query that only breaks at execution still gets another attempt. exec_start = time.monotonic() try: - exec_result = await run_in_threadpool(db.execute, user_id, sql) + exec_result = await run_in_threadpool( + _db_execute, db, workspace_id, sql, db_id + ) except Exception: log.warning("Query execution failed during streaming", exc_info=True) exec_result = {"error": "The query could not be run against the database."} @@ -627,6 +743,7 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat "answered": True, "sql": sql, "restatement": last_restatement, + "chart": last_chart, "confidence": confidence, "confidence_reason": reason, "based_on_verified": based, @@ -637,7 +754,7 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat } if "error" in exec_result: out["execution_error"] = exec_result["error"] - out["query_id"] = session_log.log_query(db.get_db_id(user_id), q, out) + out["query_id"] = session_log.log_query(db_id, q, out) # Persist to query history so the dashboard and history reflect streamed # queries too (the /api/query/stream route can't save after the response @@ -645,7 +762,7 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat # Only link the turn to a conversation the caller actually owns conversation_id = req_data.conversation_id if conversation_id and not await mdb.conversation_owned_by( - user_id, conversation_id + workspace_id, user_id, conversation_id ): conversation_id = None @@ -659,6 +776,7 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat ) try: await mdb.save_query( + workspace_id=workspace_id, user_id=user_id, question=q, sql=out["sql"], @@ -666,9 +784,10 @@ async def run_query_stream(user_id, db, kb, cfg, providers, session_log, req_dat confidence=conf_str, conversation_id=conversation_id, restatement=out.get("restatement", ""), + chart=out.get("chart"), ) if conversation_id: - await mdb.touch_conversation(conversation_id) + await mdb.touch_conversation(conversation_id, user_id) except Exception: log.warning("Failed to persist streamed query history", exc_info=True) diff --git a/backend/app/controllers/system.py b/backend/app/controllers/system.py index 3a0034ef..d7b43ba4 100644 --- a/backend/app/controllers/system.py +++ b/backend/app/controllers/system.py @@ -4,6 +4,7 @@ from fastapi import HTTPException from backend.app import config as cfgmod +import backend.app.controllers.database as dbctrl from backend.app.secrets import ( get_jwt_secret, get_supabase_url, @@ -14,7 +15,7 @@ log = logging.getLogger(__name__) -async def get_state(user_id, db, cfg, kb): +async def get_state(user_id, workspace_id, db_id, db, cfg, kb): """ Assemble the user's connection, configuration, onboarding, and knowledge state. @@ -29,25 +30,37 @@ async def get_state(user_id, db, cfg, kb): config = cfgmod.public_config(cfg) config.pop("last_db_url", None) user = await mdb.get_user_by_id(user_id) + # Restore the workspace's database from its stored credentials if this + # process doesn't hold a live engine for it — otherwise a restart reads to + # the user as their database having disconnected itself. + actual_db_id = await dbctrl.ensure_connection(db, workspace_id, db_id) s = { - "connected": db.connected(user_id), + "connected": bool(actual_db_id), "config": config, "openrouter_ready": bool(os.environ.get("OPENROUTER_API_KEY")), "tour_completed": user.get("tour_completed", False) if user else False, } - if db.connected(user_id): - dbid = db.get_db_id(user_id) + if actual_db_id: + try: + conn = await mdb.get_recent_connection_by_db_id(workspace_id, actual_db_id) + except RuntimeError: + # Only the alias is needed here; an unreadable stored URL must not + # take down the whole state response. + log.warning("Could not read stored connection for db_id=%s", actual_db_id) + conn = None s["database"] = { - "url": db.get_info(user_id)["url"], - "dialect": db.get_dialect(user_id), - "db_id": db.get_info(user_id)["db_id"], - "tables": db.get_info(user_id)["tables"], - "has_knowledge": (await kb.count_verified(user_id, dbid)) > 0, + "url": db.get_info(workspace_id, actual_db_id)["url"], + "dialect": db.get_dialect(workspace_id, actual_db_id), + "db_id": actual_db_id, + "tables": db.get_info(workspace_id, actual_db_id)["tables"], + "has_knowledge": (await kb.count_verified(workspace_id, actual_db_id)) > 0, + "alias_name": conn.get("alias_name") if conn else None, } - s["trust"] = await kb.trust_level(user_id, dbid) - s["glossary"] = await kb.get_glossary(user_id, dbid) + s["trust"] = await kb.trust_level(workspace_id, actual_db_id) + s["glossary"] = await kb.get_glossary(workspace_id, actual_db_id) s["starters"] = [ - v["question"] for v in (await kb.get_verified(user_id, dbid))[:6] + v["question"] + for v in (await kb.get_verified(workspace_id, actual_db_id))[:6] ] return s diff --git a/backend/app/controllers/workspaces.py b/backend/app/controllers/workspaces.py new file mode 100644 index 00000000..b8763625 --- /dev/null +++ b/backend/app/controllers/workspaces.py @@ -0,0 +1,883 @@ +import logging +import re +import secrets +from datetime import datetime, timedelta, timezone +from fastapi import HTTPException +from pydantic import EmailStr, TypeAdapter, ValidationError +import asyncio +from sqlalchemy import select, func +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import aliased +from backend.app.services.email import send_workspace_invite_email +from backend.app.pgdatabase.engine import async_session +from backend.app.models.workspace import Workspace, WorkspaceMember, WorkspaceInvite +from backend.app.models.workspace_settings import WorkspaceSettings +from backend.app.permissions import resolve_role_permissions +from backend.app.models.orm_user import User +from backend.app.pgdatabase.serialization import _to_uuid +from backend.app.controllers.activity import log_activity + + +log = logging.getLogger(__name__) + +INVITE_EXPIRY_DAYS = 7 + +_EMAIL_ADAPTER = TypeAdapter(EmailStr) + + +def slugify(text: str) -> str: + text = text.lower() + text = re.sub(r"[^a-z0-9]+", "-", text) + return text.strip("-") + + +# Crockford base32: no I, L, O or U, so a code survives being read aloud or +# copied by hand without the 1/I and 0/O confusions. +_CODE_ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + + +def generate_invite_code() -> str: + """A short, human-readable invite code in the form BOLO-XXXX-XXXX.""" + body = "".join(secrets.choice(_CODE_ALPHABET) for _ in range(8)) + return f"BOLO-{body[:4]}-{body[4:]}" + + +def normalize_invite_code(token: str) -> str: + """Canonicalise a pasted code so casing, spaces and stray hyphens still match. + + Only applied to codes in our own format — legacy UUID tokens are left as + typed, since they are lowercase and hyphen-significant. + """ + if not token: + return "" + stripped = token.strip() + compact = re.sub(r"[\s-]+", "", stripped).upper() + if not compact.startswith("BOLO") or len(compact) != 12: + return stripped + return f"BOLO-{compact[4:8]}-{compact[8:]}" + + +async def list_workspaces(user_id: str): + uid = _to_uuid(user_id) + async with async_session() as session: + # `member_count` powers the sidebar workspace pills. A correlated + # scalar subquery keeps this one round trip rather than one COUNT per + # workspace. + member_count = ( + select(func.count()) + .select_from(WorkspaceMember) + .where(WorkspaceMember.workspace_id == Workspace.id) + .scalar_subquery() + ) + membership = aliased(WorkspaceMember) + result = await session.execute( + select(Workspace, membership.role, member_count) + .join(membership, Workspace.id == membership.workspace_id) + .where(membership.user_id == uid) + .order_by(Workspace.created_at.desc()) + ) + rows = result.all() + return [ + { + "id": str(r[0].id), + "name": r[0].name, + "slug": r[0].slug, + "role": r[1], + "member_count": r[2], + "created_at": r[0].created_at, + } + for r in rows + ] + + +async def create_workspace(user_id: str, name: str): + uid = _to_uuid(user_id) + base_slug = slugify(name) or "workspace" + + max_attempts = 10 + slug = base_slug + + for attempt in range(max_attempts): + async with async_session() as session: + try: + if attempt > 0: + slug = f"{base_slug}-{attempt}" + + exists = await session.execute( + select(Workspace).where(Workspace.slug == slug) + ) + if exists.scalar_one_or_none(): + continue + + workspace = Workspace(name=name, slug=slug, created_by=uid) + session.add(workspace) + await session.flush() + + member = WorkspaceMember( + workspace_id=workspace.id, user_id=uid, role="owner" + ) + session.add(member) + await session.commit() + await log_activity( + str(workspace.id), + user_id, + "workspace.created", + "workspace", + str(workspace.id), + {"name": name}, + ) + return { + "id": str(workspace.id), + "name": workspace.name, + "slug": workspace.slug, + "role": "owner", + } + except IntegrityError: + await session.rollback() + continue + + raise HTTPException(500, "Could not generate a unique workspace slug") + + +async def get_workspace(workspace_id: str, user_id: str): + wid = _to_uuid(workspace_id) + uid = _to_uuid(user_id) + async with async_session() as session: + result = await session.execute( + select(Workspace, WorkspaceMember.role) + .join(WorkspaceMember, Workspace.id == WorkspaceMember.workspace_id) + .where(Workspace.id == wid, WorkspaceMember.user_id == uid) + ) + row = result.first() + if not row: + raise HTTPException(404, "Workspace not found") + return { + "id": str(row[0].id), + "name": row[0].name, + "slug": row[0].slug, + "role": row[1], + "created_at": row[0].created_at, + } + + +async def update_workspace( + workspace_id: str, name: str | None, actor_id: str | None = None +): + wid = _to_uuid(workspace_id) + async with async_session() as session: + try: + from sqlalchemy import update + + stmt = update(Workspace).where(Workspace.id == wid) + values = {} + if name is not None: + values["name"] = name.strip() + if values: + stmt = stmt.values(**values) + result = await session.execute(stmt) + await session.commit() + + await log_activity( + workspace_id, + actor_id, + "workspace.updated", + "workspace", + workspace_id, + values, + ) + return {"ok": result.rowcount > 0, "name": values.get("name")} + return {"ok": True} + except Exception: + await session.rollback() + raise + + +async def delete_workspace(workspace_id: str, actor_id: str | None = None): + """Delete a workspace and everything scoped to it. + + Every workspace-scoped table declares ``ondelete="CASCADE"`` against + ``workspaces.id``, so removing this one row takes members, invites, activity, + catalog, conversations, dashboards and saved queries with it. That includes + the workspace's own activity log, so the deletion is recorded to the + application log rather than to an audit row that is about to vanish. + """ + wid = _to_uuid(workspace_id) + async with async_session() as session: + workspace = await session.get(Workspace, wid) + if not workspace: + raise HTTPException(404, "Workspace not found") + name = workspace.name + await session.delete(workspace) + await session.commit() + + log.info("workspace.deleted id=%s name=%r actor=%s", workspace_id, name, actor_id) + return {"ok": True} + + +async def leave_workspace(workspace_id: str, user_id: str): + """Remove the calling user from a workspace. + + A sole owner cannot leave — they must transfer ownership or delete the + workspace, otherwise it would be left with nobody able to administer it. + """ + wid = _to_uuid(workspace_id) + uid = _to_uuid(user_id) + async with async_session() as session: + result = await session.execute( + select(WorkspaceMember).where( + WorkspaceMember.workspace_id == wid, WorkspaceMember.user_id == uid + ) + ) + member = result.scalar_one_or_none() + if not member: + raise HTTPException(404, "Member not found") + + if member.role == "owner": + owners_count = await session.scalar( + select(func.count()) + .select_from(WorkspaceMember) + .where( + WorkspaceMember.workspace_id == wid, + WorkspaceMember.role == "owner", + ) + ) + if owners_count <= 1: + raise HTTPException( + 400, + "You are the sole owner — transfer ownership or delete the workspace first", + ) + + await session.delete(member) + await session.commit() + + await log_activity(workspace_id, user_id, "member.left", "member", user_id, {}) + return {"ok": True} + + +async def list_members(workspace_id: str): + wid = _to_uuid(workspace_id) + async with async_session() as session: + result = await session.execute( + select(WorkspaceMember, User.email) + .join(User, WorkspaceMember.user_id == User.id) + .where(WorkspaceMember.workspace_id == wid) + ) + rows = result.all() + return [ + { + "user_id": str(r[0].user_id), + "email": r[1], + "role": r[0].role, + "joined_at": r[0].joined_at, + "created_at": r[0].joined_at, + } + for r in rows + ] + + +async def invite_user(workspace_id: str, email: str, role: str | None, inviter_id: str): + wid = _to_uuid(workspace_id) + uid = _to_uuid(inviter_id) + + async with async_session() as session: + settings_res = await session.execute( + select(WorkspaceSettings).where(WorkspaceSettings.workspace_id == wid) + ) + settings = settings_res.scalar_one_or_none() + default_role = getattr(settings, "default_invite_role", "member") + expiry_days = getattr(settings, "invite_expiry_days", INVITE_EXPIRY_DAYS) + + target_role = role if role else default_role + if target_role not in ["admin", "member"]: + raise HTTPException(400, "Invalid role for invite") + + # Check if user is already a member + user_exists = await session.execute(select(User).where(User.email == email)) + user = user_exists.scalar_one_or_none() + if user: + member = await session.execute( + select(WorkspaceMember).where( + WorkspaceMember.workspace_id == wid, + WorkspaceMember.user_id == user.id, + ) + ) + if member.scalar_one_or_none(): + raise HTTPException(400, "User is already a member") + + token = generate_invite_code() + expires = datetime.now(timezone.utc) + timedelta(days=expiry_days) + + # Query existing invite for this workspace + email + existing = await session.execute( + select(WorkspaceInvite).where( + WorkspaceInvite.workspace_id == wid, + WorkspaceInvite.email == email, + ) + ) + invite = existing.scalar_one_or_none() + + if invite: + invite.role = target_role + invite.token = token + invite.invited_by = uid + invite.expires_at = expires + invite.accepted_at = None + else: + invite = WorkspaceInvite( + workspace_id=wid, + email=email, + role=target_role, + token=token, + invited_by=uid, + expires_at=expires, + ) + session.add(invite) + + try: + await session.commit() + + # Fetch workspace name for email + ws = await session.get(Workspace, wid) + if ws: + await send_workspace_invite_email(email, ws.name, token) + + await log_activity( + workspace_id, + inviter_id, + "member.invited", + "member", + str(invite.id), + {"email": email, "role": target_role}, + ) + return { + "id": str(invite.id), + "email": invite.email, + "role": invite.role, + "token": invite.token, + "expires_at": invite.expires_at, + } + except IntegrityError: + await session.rollback() + raise HTTPException(400, "Could not process invite") + + +async def update_member_role( + workspace_id: str, + target_user_id: str, + role: str, + actor_id: str | None = None, + actor_role: str = "admin", +): + wid = _to_uuid(workspace_id) + tuid = _to_uuid(target_user_id) + if role not in ["admin", "member"]: + raise HTTPException(400, "Invalid role") + + role_rank = {"member": 1, "admin": 2, "owner": 3} + actor_rank = role_rank.get(actor_role, 0) + if actor_rank < role_rank["admin"]: + raise HTTPException(403, "Insufficient permissions") + + async with async_session() as session: + result = await session.execute( + select(WorkspaceMember).where( + WorkspaceMember.workspace_id == wid, WorkspaceMember.user_id == tuid + ) + ) + member = result.scalar_one_or_none() + if not member: + raise HTTPException(404, "Member not found") + + target_rank = role_rank.get(member.role, 0) + # Callers may only modify members strictly below their own rank. + if target_rank >= actor_rank: + raise HTTPException( + 403, "Cannot change the role of a member at or above your own level" + ) + if role_rank.get(role, 0) >= actor_rank: + raise HTTPException(403, "Cannot assign a role at or above your own level") + + if member.role == "owner" and role != "owner": + raise HTTPException(400, "Cannot change role of the sole workspace owner") + + member.role = role + await session.commit() + await log_activity( + workspace_id, + actor_id, + "member.role_updated", + "member", + target_user_id, + {"new_role": role}, + ) + return {"ok": True} + + +async def transfer_ownership( + workspace_id: str, current_owner_id: str, target_user_id: str +): + """Hand the owner role to another member, demoting the caller to admin. + + Both role changes happen in one transaction so the workspace is never left + with two owners or none. + """ + wid = _to_uuid(workspace_id) + if current_owner_id == target_user_id: + raise HTTPException(400, "You already own this workspace") + try: + tuid = _to_uuid(target_user_id) + except (ValueError, AttributeError, TypeError): + raise HTTPException(404, "Member not found") + cuid = _to_uuid(current_owner_id) + + async with async_session() as session: + result = await session.execute( + select(WorkspaceMember).where( + WorkspaceMember.workspace_id == wid, + WorkspaceMember.user_id.in_([tuid, cuid]), + ) + ) + by_user = {m.user_id: m for m in result.scalars().all()} + target = by_user.get(tuid) + actor = by_user.get(cuid) + if not target: + raise HTTPException(404, "Member not found") + if not actor or actor.role != "owner": + raise HTTPException(403, "Only the owner can transfer ownership") + + target.role = "owner" + actor.role = "admin" + await session.commit() + + await log_activity( + workspace_id, + current_owner_id, + "member.ownership_transferred", + "member", + target_user_id, + {"previous_owner": current_owner_id}, + ) + return {"ok": True, "new_owner_id": target_user_id} + + +def _normalize_email(raw: str) -> str | None: + """Return a validated, lower-cased address, or None if it isn't one. + + Bulk invites report bad addresses per row instead of rejecting the whole + batch, so validation happens here rather than in the request model. + """ + if not isinstance(raw, str): + return None + candidate = raw.strip().lower() + if not candidate: + return None + try: + return str(_EMAIL_ADAPTER.validate_python(candidate)).lower() + except ValidationError: + return None + + +async def bulk_invite( + workspace_id: str, emails: list[str], role: str, inviter_id: str +) -> dict: + """Invite many people at once, reporting the outcome for each address. + + One bad or duplicate address must not sink the rest of the batch, so each + invite is attempted independently and its status recorded. + """ + if role not in ["admin", "member"]: + raise HTTPException(400, "Invalid role for invite") + + results: list[dict] = [{}] * len(emails) + valid_tasks = [] + valid_indices = [] + seen: set[str] = set() + + sem = asyncio.Semaphore(10) + + async def _invite_worker(email: str): + async with sem: + try: + await invite_user(workspace_id, email, role, inviter_id) + return {"email": email, "status": "invited"} + except HTTPException as e: + detail = str(e.detail or "") + status = "already_member" if "already a member" in detail else "failed" + return {"email": email, "status": status, "detail": detail} + + for i, raw in enumerate(emails): + email = _normalize_email(raw) + if not email: + results[i] = {"email": (raw or "").strip(), "status": "invalid"} + continue + if email in seen: + results[i] = {"email": email, "status": "duplicate"} + continue + seen.add(email) + valid_indices.append(i) + valid_tasks.append(_invite_worker(email)) + + if valid_tasks: + worker_results = await asyncio.gather(*valid_tasks) + for idx, res in zip(valid_indices, worker_results): + results[idx] = res + + invited = sum(1 for r in results if r["status"] == "invited") + return {"invited": invited, "total": len(results), "results": results} + + +async def remove_member( + workspace_id: str, + target_user_id: str, + actor_id: str | None = None, + actor_role: str = "admin", +): + wid = _to_uuid(workspace_id) + tuid = _to_uuid(target_user_id) + actor_uuid = _to_uuid(actor_id) if actor_id else None + + role_rank = {"member": 1, "admin": 2, "owner": 3} + actor_rank = role_rank.get(actor_role, 0) + + async with async_session() as session: + result = await session.execute( + select(WorkspaceMember).where( + WorkspaceMember.workspace_id == wid, WorkspaceMember.user_id == tuid + ) + ) + member = result.scalar_one_or_none() + if not member: + raise HTTPException(404, "Member not found") + + target_rank = role_rank.get(member.role, 0) + + if actor_uuid != tuid: + if target_rank >= actor_rank: + raise HTTPException( + 403, "Cannot remove a member at or above your own level" + ) + if actor_rank < role_rank["admin"]: + raise HTTPException(403, "Insufficient permissions") + + if member.role == "owner": + owners_count = await session.scalar( + select(func.count()) + .select_from(WorkspaceMember) + .where( + WorkspaceMember.workspace_id == wid, + WorkspaceMember.role == "owner", + ) + ) + if owners_count <= 1: + raise HTTPException(400, "Cannot remove the sole owner of a workspace") + + await session.delete(member) + await session.commit() + await log_activity( + workspace_id, actor_id, "member.removed", "member", target_user_id, {} + ) + return {"ok": True} + + +async def get_invites(user_email: str): + async with async_session() as session: + now = datetime.now(timezone.utc) + result = await session.execute( + select(WorkspaceInvite, Workspace.name) + .join(Workspace, WorkspaceInvite.workspace_id == Workspace.id) + .where( + WorkspaceInvite.email == user_email, + WorkspaceInvite.accepted_at.is_(None), + WorkspaceInvite.expires_at > now, + ) + ) + rows = result.all() + return [ + { + "id": str(r[0].id), + "workspace_id": str(r[0].workspace_id), + "workspace_name": r[1], + "role": r[0].role, + "token": r[0].token, + "expires_at": r[0].expires_at, + } + for r in rows + ] + + +async def list_pending_invites(workspace_id: str): + """Invites for this workspace that are neither accepted nor expired.""" + wid = _to_uuid(workspace_id) + inviter = aliased(User) + async with async_session() as session: + now = datetime.now(timezone.utc) + result = await session.execute( + select(WorkspaceInvite, inviter.email) + .outerjoin(inviter, WorkspaceInvite.invited_by == inviter.id) + .where( + WorkspaceInvite.workspace_id == wid, + WorkspaceInvite.accepted_at.is_(None), + WorkspaceInvite.expires_at > now, + ) + .order_by(WorkspaceInvite.expires_at.desc()) + ) + return [ + { + "id": str(r[0].id), + "email": r[0].email, + "role": r[0].role, + "invited_by": r[1], + "expires_at": r[0].expires_at, + } + for r in result.all() + ] + + +async def _get_invite(session, wid, invite_id: str) -> WorkspaceInvite: + try: + iid = _to_uuid(invite_id) + except (ValueError, AttributeError, TypeError): + raise HTTPException(404, "Invite not found") + result = await session.execute( + select(WorkspaceInvite).where( + WorkspaceInvite.id == iid, + WorkspaceInvite.workspace_id == wid, + ) + ) + invite = result.scalar_one_or_none() + if not invite: + raise HTTPException(404, "Invite not found") + return invite + + +async def rescind_invite( + workspace_id: str, invite_id: str, actor_id: str | None = None +): + wid = _to_uuid(workspace_id) + async with async_session() as session: + invite = await _get_invite(session, wid, invite_id) + if invite.accepted_at: + raise HTTPException(400, "Invite has already been accepted") + email = invite.email + await session.delete(invite) + await session.commit() + + await log_activity( + workspace_id, + actor_id, + "member.invite_revoked", + "member", + invite_id, + {"email": email}, + ) + return {"ok": True} + + +async def resend_invite(workspace_id: str, invite_id: str, actor_id: str | None = None): + """Re-send an invite email and push its expiry out. + + The existing code is kept so any already-delivered email still works; only an + expired invite gets a fresh token. + """ + wid = _to_uuid(workspace_id) + async with async_session() as session: + invite = await _get_invite(session, wid, invite_id) + if invite.accepted_at: + raise HTTPException(400, "Invite has already been accepted") + + settings_res = await session.execute( + select(WorkspaceSettings).where(WorkspaceSettings.workspace_id == wid) + ) + settings = settings_res.scalar_one_or_none() + expiry_days = getattr(settings, "invite_expiry_days", INVITE_EXPIRY_DAYS) + + now = datetime.now(timezone.utc) + if invite.expires_at < now: + invite.token = generate_invite_code() + invite.expires_at = now + timedelta(days=expiry_days) + + workspace = await session.get(Workspace, wid) + workspace_name = workspace.name if workspace else None + await session.commit() + + email, token, expires_at = invite.email, invite.token, invite.expires_at + + sent = False + if workspace_name: + sent = await send_workspace_invite_email(email, workspace_name, token) + + await log_activity( + workspace_id, + actor_id, + "member.invite_resent", + "member", + invite_id, + {"email": email}, + ) + return {"ok": True, "email_sent": sent, "expires_at": expires_at} + + +async def accept_invite(token: str, user_id: str, user_email: str): + uid = _to_uuid(user_id) + token = normalize_invite_code(token) + async with async_session() as session: + result = await session.execute( + select(WorkspaceInvite).where( + WorkspaceInvite.token == token, WorkspaceInvite.email == user_email + ) + ) + invite = result.scalar_one_or_none() + if not invite: + raise HTTPException(404, "Invite not found or invalid email") + if invite.accepted_at: + raise HTTPException(400, "Invite already accepted") + if invite.expires_at < datetime.now(invite.expires_at.tzinfo): + raise HTTPException(400, "Invite expired") + + try: + member = WorkspaceMember( + workspace_id=invite.workspace_id, user_id=uid, role=invite.role + ) + invite.accepted_at = datetime.now(invite.expires_at.tzinfo) + session.add(member) + await session.commit() + await log_activity( + str(invite.workspace_id), + user_id, + "member.joined", + "member", + user_id, + {"role": invite.role}, + ) + return {"ok": True, "workspace_id": str(invite.workspace_id)} + except IntegrityError: + await session.rollback() + raise HTTPException(400, "User is already a member of this workspace") + + +def _permission_matrix(overrides): + """The full role → permission → allowed map. `None` gives the defaults.""" + return { + role: resolve_role_permissions(role, overrides) + for role in ("owner", "admin", "member") + } + + +async def get_workspace_settings(workspace_id: str): + wid = _to_uuid(workspace_id) + async with async_session() as session: + result = await session.execute( + select(WorkspaceSettings).where(WorkspaceSettings.workspace_id == wid) + ) + settings = result.scalar_one_or_none() + if not settings: + settings = WorkspaceSettings(workspace_id=wid) + session.add(settings) + await session.commit() + await session.refresh(settings) + + role_perms = settings.role_permissions or {} + return { + "workspace_id": str(settings.workspace_id), + "default_invite_role": settings.default_invite_role, + "invite_expiry_days": settings.invite_expiry_days, + "activity_retention_days": settings.activity_retention_days, + "role_permissions": role_perms, + "resolved_matrix": _permission_matrix(role_perms), + # What the matrix would be with nothing customised. The settings UI + # needs this to tell "this is the default" apart from "someone chose + # this" — without it the UI has to hardcode a second copy of the + # defaults, which silently goes stale when the registry changes. + "default_matrix": _permission_matrix(None), + } + + +async def update_workspace_settings( + workspace_id: str, + update_data: dict, + actor_id: str | None = None, +): + wid = _to_uuid(workspace_id) + + # Exclude any owner overrides + role_perms = update_data.get("role_permissions") + if isinstance(role_perms, dict): + role_perms = dict(role_perms) + role_perms.pop("owner", None) + + async with async_session() as session: + result = await session.execute( + select(WorkspaceSettings).where(WorkspaceSettings.workspace_id == wid) + ) + settings = result.scalar_one_or_none() + if not settings: + settings = WorkspaceSettings(workspace_id=wid) + session.add(settings) + + if ( + "default_invite_role" in update_data + and update_data["default_invite_role"] is not None + ): + settings.default_invite_role = update_data["default_invite_role"] + if ( + "invite_expiry_days" in update_data + and update_data["invite_expiry_days"] is not None + ): + settings.invite_expiry_days = update_data["invite_expiry_days"] + if ( + "activity_retention_days" in update_data + and update_data["activity_retention_days"] is not None + ): + settings.activity_retention_days = update_data["activity_retention_days"] + if role_perms is not None: + if isinstance(role_perms, dict): + if update_data.get("role_permissions") == {}: + settings.role_permissions = {} + else: + current_perms = dict(settings.role_permissions or {}) + current_perms.pop("owner", None) + merged = {} + for role_name, perms in current_perms.items(): + if role_name != "owner": + merged[role_name] = ( + dict(perms) if isinstance(perms, dict) else perms + ) + for role_name, perms in role_perms.items(): + if role_name == "owner": + continue + if isinstance(perms, dict): + if role_name in merged and isinstance( + merged[role_name], dict + ): + merged[role_name] = {**merged[role_name], **perms} + else: + merged[role_name] = dict(perms) + else: + merged[role_name] = perms + merged.pop("owner", None) + settings.role_permissions = merged + else: + settings.role_permissions = role_perms + + await session.commit() + await session.refresh(settings) + + await log_activity( + workspace_id, + actor_id, + "workspace.settings_updated", + "workspace", + workspace_id, + update_data, + ) + + curr_perms = settings.role_permissions or {} + return { + "workspace_id": str(settings.workspace_id), + "default_invite_role": settings.default_invite_role, + "invite_expiry_days": settings.invite_expiry_days, + "activity_retention_days": settings.activity_retention_days, + "role_permissions": curr_perms, + "resolved_matrix": _permission_matrix(curr_perms), + "default_matrix": _permission_matrix(None), + } diff --git a/backend/app/database.py b/backend/app/database.py index 12bc31f7..c2c0f217 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -120,11 +120,22 @@ def db_id_for(url): return hashlib.sha256(sanitize_url(url).encode()).hexdigest()[:16] +class _ConnectionsDict(dict): + def __getitem__(self, key): + if key in self: + return super().__getitem__(key) + if isinstance(key, str): + for k, v in self.items(): + if k == key or (isinstance(k, tuple) and k[0] == key): + return v + raise KeyError(key) + + class DatabaseManager: def __init__( self, readonly=True, sample_rows=3, max_rows=500, statement_timeout=30 ): - self._connections = {} + self._connections = _ConnectionsDict() self.readonly = readonly self.sample_rows = sample_rows self.max_rows = max_rows @@ -132,26 +143,39 @@ def __init__( # it. Enforced server-side per connection (see _apply_statement_timeout). self.statement_timeout = statement_timeout - def connected(self, user_id): - return user_id in self._connections + def connected(self, workspace_id, db_id=None): + if db_id: + return (workspace_id, db_id) in self._connections + return any(k[0] == workspace_id for k in self._connections.keys()) - def _get(self, user_id): - try: - return self._connections[user_id] - except KeyError: - raise HTTPException(409, "No Database Connected") + def _get(self, workspace_id, db_id=None): + if db_id: + try: + return self._connections[(workspace_id, db_id)] + except KeyError: + raise HTTPException(409, "No Database Connected") + + # Fallback to the first connected database for this workspace + for k, v in self._connections.items(): + if k[0] == workspace_id: + return v + raise HTTPException(409, "No Database Connected") - def disconnect(self, user_id): + def disconnect(self, workspace_id, db_id=None): """Reset all connection state so the server accepts a new connect call.""" - if user_id not in self._connections: - return - try: - self._connections[user_id]["engine"].dispose() # release pooled connections - except Exception as e: - logger.warning("Error disposing engine on disconnect: %s", e) - del self._connections[user_id] + keys_to_delete = [] + for k in self._connections.keys(): + if k[0] == workspace_id and (db_id is None or k[1] == db_id): + keys_to_delete.append(k) + + for k in keys_to_delete: + try: + self._connections[k]["engine"].dispose() + except Exception as e: + logger.warning("Error disposing engine on disconnect: %s", e) + del self._connections[k] - def connect(self, user_id, url): + def connect(self, workspace_id, url): import os if os.environ.get("RUNNING_IN_DOCKER") == "true": @@ -170,11 +194,12 @@ def connect(self, user_id, url): with engine.connect() as c: c.execute(text("SELECT 1")) tables = len(inspect(engine).get_table_names()) - old_connection = self._connections.get(user_id) - self._connections[user_id] = { + db_id = db_id_for(url) + old_connection = self._connections.get((workspace_id, db_id)) + self._connections[(workspace_id, db_id)] = { "engine": engine, "url": url, - "db_id": db_id_for(url), + "db_id": db_id, "dialect": url.split(":")[0].split("+")[0], "_schema_cache": None, "_table_count": tables, @@ -183,22 +208,22 @@ def connect(self, user_id, url): old_connection["engine"].dispose() return { "ok": True, - "dialect": self._connections[user_id]["dialect"], + "dialect": self._connections[(workspace_id, db_id)]["dialect"], "tables": tables, - "db_id": self._connections[user_id]["db_id"], + "db_id": self._connections[(workspace_id, db_id)]["db_id"], "url": sanitize_url(url), } except Exception as e: return {"ok": False, "error": _sanitize_db_error(e)} - def _q(self, user_id, n): - c = self._get(user_id) + def _q(self, workspace_id, n): + c = self._get(workspace_id) n = str(n) if c["dialect"] == "mysql": return f"`{n.replace('`', '``')}`" return f'"{n.replace(chr(34), chr(34) * 2)}"' - def get_schema(self, user_id, refresh=False): + def get_schema(self, workspace_id, db_id=None, refresh=False): """Introspect the connected database into the schema dict. STRUCTURE (columns, primary/foreign keys) is collected for EVERY table @@ -207,7 +232,7 @@ def get_schema(self, user_id, refresh=False): distinct values) is capped, at ENRICH_MAX tables, preferring the largest tables when row counts are known. """ - c = self._get(user_id) + c = self._get(workspace_id, db_id=db_id) if c["_schema_cache"] and not refresh: return c["_schema_cache"] inspector = inspect(c["engine"]) @@ -355,7 +380,7 @@ def _sort_key(t): try: res = conn.execute( text( - f"SELECT * FROM {self._q(user_id, tbl)} LIMIT {self.sample_rows}" + f"SELECT * FROM {self._q(workspace_id, tbl)} LIMIT {self.sample_rows}" ) ) names = list(res.keys()) @@ -372,7 +397,7 @@ def _sort_key(t): if rc is None: try: rc = conn.execute( - text(f"SELECT COUNT(*) FROM {self._q(user_id, tbl)}") + text(f"SELECT COUNT(*) FROM {self._q(workspace_id, tbl)}") ).scalar() except Exception as e: logger.warning("Error fetching row count for %s: %s", tbl, e) @@ -390,7 +415,7 @@ def _sort_key(t): try: dv = conn.execute( text( - f"SELECT DISTINCT {self._q(user_id, col['name'])} FROM {self._q(user_id, tbl)} LIMIT 12" + f"SELECT DISTINCT {self._q(workspace_id, col['name'])} FROM {self._q(workspace_id, tbl)} LIMIT 12" ) ).fetchall() vals = [row[0] for row in dv if row[0] is not None] @@ -414,9 +439,9 @@ def _sort_key(t): c["_schema_cache"] = schema return schema - def schema_as_text(self, user_id): - c = self._get(user_id) - schema = self.get_schema(user_id) + def schema_as_text(self, workspace_id, db_id=None): + c = self._get(workspace_id, db_id) + schema = self.get_schema(workspace_id, db_id=db_id) lines = [f"Database dialect: {c['dialect']}"] for tbl, info in schema.items(): rc = ( @@ -441,7 +466,7 @@ def schema_as_text(self, user_id): lines.append(f" sample: {info['sample_rows'][:1]}") return "\n".join(lines) - def _readonly_violation(self, user_id, cleaned): + def _readonly_violation(self, workspace_id, cleaned, db_id=None): """Return an error string if `cleaned` is not a safe read-only query, else None. Primary check parses the SQL into an AST (sqlglot) and rejects anything @@ -453,7 +478,7 @@ def _readonly_violation(self, user_id, cleaned): """ if not cleaned: return "Empty statement." - c = self._get(user_id) + c = self._get(workspace_id, db_id) glot_dialect = _GLOT_DIALECT.get(c["dialect"], c["dialect"]) try: stmts = sqlglot.parse(cleaned, dialect=glot_dialect) @@ -512,11 +537,11 @@ def _apply_statement_timeout(self, conn, dialect): except SQLAlchemyError as e: logger.warning("Could not set statement timeout for %s: %s", dialect, e) - def execute(self, user_id, sql): - c = self._get(user_id) + def execute(self, workspace_id, sql, db_id=None): + c = self._get(workspace_id, db_id) cleaned = sql.strip().rstrip(";").strip() if self.readonly: - violation = self._readonly_violation(user_id, cleaned) + violation = self._readonly_violation(workspace_id, cleaned, db_id) if violation: return {"error": violation, "sql": cleaned} try: @@ -551,14 +576,14 @@ def execute(self, user_id, sql): } return {"error": _sanitize_db_error(e), "sql": cleaned} - def get_db_id(self, user_id): - return self._get(user_id)["db_id"] + def get_db_id(self, workspace_id, db_id=None): + return self._get(workspace_id, db_id)["db_id"] - def get_dialect(self, user_id): - return self._get(user_id)["dialect"] + def get_dialect(self, workspace_id, db_id=None): + return self._get(workspace_id, db_id)["dialect"] - def get_info(self, user_id): - c = self._get(user_id) + def get_info(self, workspace_id, db_id=None): + c = self._get(workspace_id, db_id) return { "url": sanitize_url(c["url"]), "dialect": c["dialect"], diff --git a/backend/app/dependencies.py b/backend/app/dependencies.py index 8cb07818..8100db28 100644 --- a/backend/app/dependencies.py +++ b/backend/app/dependencies.py @@ -1,7 +1,13 @@ -from fastapi import HTTPException, Cookie, Request +from fastapi import HTTPException, Cookie, Request, Depends, Header import jwt +from sqlalchemy import select from backend.app.secrets import get_jwt_secret +from backend.app.pgdatabase.engine import async_session +from backend.app.models.workspace import WorkspaceMember +from backend.app.models.workspace_settings import WorkspaceSettings +from backend.app.permissions import has_permission +from backend.app.pgdatabase.serialization import _to_uuid async def get_current_user(access_token: str = Cookie(None)): @@ -18,6 +24,79 @@ async def get_current_user(access_token: str = Cookie(None)): raise HTTPException(status_code=401, detail="Invalid Token") +async def get_current_workspace( + x_workspace_id: str = Header(None), user=Depends(get_current_user) +): + if not x_workspace_id: + raise HTTPException(status_code=400, detail="X-Workspace-Id header missing") + try: + wid = _to_uuid(x_workspace_id) + uid = _to_uuid(user.get("user_id", user.get("sub"))) + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="Invalid workspace_id or user_id") + + async with async_session() as session: + result = await session.execute( + select(WorkspaceMember).where( + WorkspaceMember.workspace_id == wid, WorkspaceMember.user_id == uid + ) + ) + member = result.scalar_one_or_none() + if not member: + raise HTTPException( + status_code=403, detail="Not a member of this workspace" + ) + + return { + "workspace_id": str(member.workspace_id), + "role": member.role, + "user_id": str(member.user_id), + } + + +def require_role(minimum_role: str): + def role_dependency(workspace=Depends(get_current_workspace)): + roles = {"member": 1, "admin": 2, "owner": 3} + user_role = workspace["role"] + if roles.get(user_role, 0) < roles.get(minimum_role, 1): + raise HTTPException(status_code=403, detail="Insufficient permissions") + return workspace + + return role_dependency + + +async def workspace_has_permission(workspace, permission_key: str) -> bool: + """Resolve whether the current workspace member holds a fine-grained permission.""" + if workspace["role"] == "owner": + return True + + wid = _to_uuid(workspace["workspace_id"]) + async with async_session() as session: + result = await session.execute( + select(WorkspaceSettings).where(WorkspaceSettings.workspace_id == wid) + ) + settings = result.scalar_one_or_none() + + role_permissions = getattr(settings, "role_permissions", None) + return has_permission(workspace["role"], role_permissions, permission_key) + + +def require_permission(permission_key: str): + async def permission_dependency(workspace=Depends(get_current_workspace)): + if not await workspace_has_permission(workspace, permission_key): + raise HTTPException( + status_code=403, detail=f"Permission '{permission_key}' required" + ) + + return workspace + + return permission_dependency + + +def get_current_db_id(x_db_id: str = Header(None)): + return x_db_id + + def get_db(request: Request): return request.app.state.db diff --git a/backend/app/llm.py b/backend/app/llm.py index 4e29ab8d..0513a59e 100644 --- a/backend/app/llm.py +++ b/backend/app/llm.py @@ -47,6 +47,15 @@ def parse_json(text): return json.loads(s) +CHART_TYPES = ["table", "bar", "line", "area", "pie", "number"] +DEFAULT_CHART = { + "type": "table", + "x_axis": "", + "y_axis": "", + "title": "", + "reason": "", +} + SQL_SCHEMA = { "name": "sql_result", "schema": { @@ -65,8 +74,37 @@ def parse_json(text): "items": {"type": "string"}, "description": "Any assumptions made while interpreting the question.", }, + "chart": { + "type": "object", + "description": "How this result is best visualised.", + "properties": { + "type": { + "type": "string", + "enum": CHART_TYPES, + "description": "Chart best suited to the shape of the result.", + }, + "x_axis": { + "type": "string", + "description": "Column alias for the category/time axis. Empty for table/number.", + }, + "y_axis": { + "type": "string", + "description": "Column alias for the numeric value. Empty for table.", + }, + "title": { + "type": "string", + "description": "Short chart title, e.g. 'Revenue by month'.", + }, + "reason": { + "type": "string", + "description": "One short clause on why this chart fits.", + }, + }, + "required": ["type", "x_axis", "y_axis", "title", "reason"], + "additionalProperties": False, + }, }, - "required": ["sql", "restatement", "assumptions"], + "required": ["sql", "restatement", "assumptions", "chart"], "additionalProperties": False, }, } @@ -191,6 +229,31 @@ def parse_json(text): } +def parse_chart_spec(raw): + """Normalise the model's chart choice, falling back to a plain table. + + A bad or missing chart must never fail the query — the answer is still + useful rendered as a table. + """ + if not isinstance(raw, dict): + return dict(DEFAULT_CHART) + chart_type = raw.get("type") + if not isinstance(chart_type, str) or chart_type.lower() not in CHART_TYPES: + return dict(DEFAULT_CHART) + + def _text(key): + v = raw.get(key) + return v.strip() if isinstance(v, str) else "" + + return { + "type": chart_type.lower(), + "x_axis": _text("x_axis"), + "y_axis": _text("y_axis"), + "title": _text("title"), + "reason": _text("reason"), + } + + def parse_sql_output(raw): if isinstance(raw, dict): obj = raw @@ -216,7 +279,12 @@ def parse_sql_output(raw): if isinstance(restatement, str) else ("" if restatement is None else str(restatement)) ).strip() - return {"sql": sql, "restatement": restatement, "assumptions": assumptions} + return { + "sql": sql, + "restatement": restatement, + "assumptions": assumptions, + "chart": parse_chart_spec(obj.get("chart")), + } class LLMProvider(ABC): @@ -501,7 +569,7 @@ def build_sql_system_prompt( ): hint = _DIALECT_HINTS.get(dialect, "") return ( - "Answer in English\n" + "CRITICAL: You MUST answer entirely in English. Do not use Chinese or Japanese.\n" f"You are an expert {dialect} analyst. Convert the user's question into " "exactly one read-only SELECT query.\n\n" "Rules:\n" @@ -515,7 +583,19 @@ def build_sql_system_prompt( "[brackets], match those values exactly.\n" "6. Qualify column names with table aliases whenever more than one " "table is involved.\n" - f"7. {hint}\n\n" + f"7. {hint}\n" + "8. Also choose how the result should be visualised, based on the shape " + "of the rows your query returns:\n" + " - line: a measure over time (dates, months, quarters).\n" + " - area: a cumulative or stacked measure over time.\n" + " - bar: a measure compared across categories.\n" + " - pie: parts of one whole, only when there are 2-6 categories and " + "the values are all positive.\n" + " - number: a single row with a single aggregate value.\n" + " - table: anything else — many columns, raw records, or no clear " + "numeric measure. Prefer table when no chart genuinely helps.\n" + " x_axis and y_axis must be column aliases that appear in your SELECT " + "list. Leave them empty for table, and set only y_axis for number.\n\n" f"Schema (col PK = primary key, col->t.c = foreign key, [v1,v2] = " f"example values):\n{schema_text}\n\n" f"{_glossary_block(glossary)}" @@ -526,7 +606,10 @@ def build_sql_system_prompt( '{"sql":"",' '"restatement":"",' '"assumptions":[""]}' + 'or date range; empty list if none>"],' + '"chart":{"type":"","x_axis":"",' + '"y_axis":"","title":"",' + '"reason":""}}' ) @@ -573,6 +656,7 @@ async def shortlist_tables(provider, question, schema, max_columns=4): lines.append(f"{t}({head}{more})") catalog = "\n".join(lines) system = ( + "CRITICAL: You MUST answer entirely in English. Do not use Chinese or Japanese.\n" "You map questions to database tables. Below is the complete catalog " "of tables (name and columns only).\n" "Return every table that might be needed to answer the user's " @@ -607,7 +691,7 @@ async def shortlist_tables(provider, question, schema, max_columns=4): async def explain_sql(provider, sql, dialect): system = ( - "Answer in English\n" + "CRITICAL: You MUST answer entirely in English. Do not use Chinese or Japanese.\n" f"You explain {dialect} SQL to non-technical business users.\n" "Describe what the query returns in 2-4 short plain sentences: which " "data it looks at, how it filters/groups, and how results are ordered " @@ -621,7 +705,7 @@ async def explain_sql(provider, sql, dialect): async def suggest_catalog(provider, schema_text): system = ( - "Answer in English\n" + "CRITICAL: You MUST answer entirely in English. Do not use Chinese or Japanese.\n" "You are a data analyst documenting a database for non-technical users.\n" f"{schema_text}\n\n" "Produce a concise semantic catalog:\n" @@ -660,7 +744,7 @@ async def generate_starters(provider, schema_text, dialect): schema_text[:300], ) system = ( - "Answer in English\n" + "CRITICAL: You MUST answer entirely in English. Do not use Chinese or Japanese.\n" f"You are a database analyst. {dialect} database.\n{schema_text}\n\n" "Generate 3 common useful questions a non-technical user would ask, each with the SQL and " "a one-sentence plain-English description.\n" diff --git a/backend/app/logbook.py b/backend/app/logbook.py index 7192e575..9123b8aa 100644 --- a/backend/app/logbook.py +++ b/backend/app/logbook.py @@ -1,88 +1,26 @@ -"""Local session logging for validation.""" +"""Per-query id minting. + +This used to write a JSONL audit log under the config directory. The durable +record of every query and its feedback now lives in Postgres (`query_history`), +so nothing is written to disk here — the class survives only to hand each query +a short id that feedback is recorded against. +""" -import json -import time import logging -from pathlib import Path -from datetime import datetime -from logging.handlers import RotatingFileHandler +import uuid logger = logging.getLogger(__name__) -MAX_BYTES = 10 * 1024 * 1024 # 10MB -BACKUP_COUNT = 5 - class SessionLog: - def __init__(self, base_dir): - self.dir = Path(base_dir) / "sessions" - self.dir.mkdir(parents=True, exist_ok=True) - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - self.file = self.dir / f"session-{stamp}.jsonl" - self._setup_rotating_handler() - - def _setup_rotating_handler(self): - self._handler = RotatingFileHandler( - str(self.file), - maxBytes=MAX_BYTES, - backupCount=BACKUP_COUNT, - encoding="utf-8", - ) - - def _append(self, record): - record.update( - { - "ts": time.time(), - "ts_human": datetime.now().isoformat(timespec="seconds"), - } - ) - try: - line = json.dumps(record, default=str) - self._handler.emit( - logging.LogRecord( - name=__name__, - level=logging.INFO, - pathname="", - lineno=0, - msg=line, - args=(), - exc_info=None, - ) - ) - except Exception as e: - logger.error("Failed to append to session log: %s", e) + def __init__(self, base_dir=None): + # base_dir is accepted for backward compatibility and ignored — no files + # are written. + pass def log_query(self, db_id, question, result): - import uuid - - qid = uuid.uuid4().hex[:12] - self._append( - { - "event": "query", - "query_id": qid, - "db_id": db_id, - "question": question, - "sql": result.get("sql"), - "confidence": result.get("confidence"), - "confidence_reason": result.get("confidence_reason"), - "based_on_verified": result.get("based_on_verified"), - "row_count": len(result.get("rows", []) or []), - "execution_error": result.get("execution_error"), - # Audit trail for linking quality: exactly which tables the AI - # was shown (including schema-retry additions) and how many - # generation attempts the repair loop needed. - "tables_used": result.get("tables_used"), - "attempts": result.get("attempts"), - } - ) - return qid + """Return a short id for this query. Persistence is Postgres's job.""" + return uuid.uuid4().hex[:12] def log_feedback(self, query_id, verdict, reason=""): - self._append( - { - "event": "feedback", - "query_id": query_id, - "verdict": verdict, - "reason": reason, - } - ) + return None diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 00000000..424ca446 --- /dev/null +++ b/backend/app/models/__init__.py @@ -0,0 +1,46 @@ +from backend.app.models.base import Base, _utcnow, _uuid7 +from backend.app.models.orm_user import User +from backend.app.models.workspace import Workspace, WorkspaceMember, WorkspaceInvite +from backend.app.models.workspace_settings import WorkspaceSettings +from backend.app.models.activity import WorkspaceActivityLog +from backend.app.models.conversation import Conversation, QueryHistory +from backend.app.models.recent_connection import RecentConnection +from backend.app.models.saved_query import SavedQuery +from backend.app.models.dashboard import Dashboard, DashboardPanel +from backend.app.models.catalog import ( + VerifiedQA, + Glossary, + CatalogColumn, + CatalogMetric, + CatalogJoin, + CatalogSynonym, + CatalogValueMapping, +) +from backend.app.models.auth_token import PasswordResetToken, OtpCode + +__all__ = [ + "Base", + "_utcnow", + "_uuid7", + "User", + "Workspace", + "WorkspaceMember", + "WorkspaceInvite", + "WorkspaceSettings", + "WorkspaceActivityLog", + "Conversation", + "QueryHistory", + "RecentConnection", + "VerifiedQA", + "Glossary", + "CatalogColumn", + "CatalogMetric", + "CatalogJoin", + "CatalogSynonym", + "CatalogValueMapping", + "PasswordResetToken", + "OtpCode", + "SavedQuery", + "Dashboard", + "DashboardPanel", +] diff --git a/backend/app/models/activity.py b/backend/app/models/activity.py new file mode 100644 index 00000000..bd8353aa --- /dev/null +++ b/backend/app/models/activity.py @@ -0,0 +1,34 @@ +import uuid +from datetime import datetime +from sqlalchemy import String, ForeignKey, DateTime, Index +from sqlalchemy.dialects.postgresql import UUID as PgUUID, JSONB +from sqlalchemy.orm import Mapped, mapped_column +from typing import Optional, Any +from backend.app.models.base import Base, _utcnow, _uuid7 + + +class WorkspaceActivityLog(Base): + __tablename__ = "workspace_activity_log" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + actor_id: Mapped[Optional[uuid.UUID]] = mapped_column( + PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + event_type: Mapped[str] = mapped_column(String, nullable=False) + resource_type: Mapped[str] = mapped_column(String, nullable=False) + resource_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) + metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) + + __table_args__ = ( + Index("ix_activity_workspace_date", "workspace_id", "created_at"), + Index("ix_activity_workspace_event", "workspace_id", "event_type"), + ) diff --git a/backend/app/models/api.py b/backend/app/models/api.py index 760a02a9..9503aea0 100644 --- a/backend/app/models/api.py +++ b/backend/app/models/api.py @@ -7,6 +7,7 @@ class ConfigUpdate(BaseModel): class ConnectReq(BaseModel): db_url: str + alias_name: str | None = None class ContextTurn(BaseModel): @@ -39,6 +40,8 @@ class FeedbackReq(BaseModel): class RawSQLReq(BaseModel): sql: str conversation_id: str | None = None + # Re-running SQL that is already in history shouldn't create a duplicate entry. + save_history: bool = True class GlossaryItem(BaseModel): diff --git a/backend/app/models/auth_token.py b/backend/app/models/auth_token.py new file mode 100644 index 00000000..58a4b146 --- /dev/null +++ b/backend/app/models/auth_token.py @@ -0,0 +1,48 @@ +import uuid +from datetime import datetime + +from sqlalchemy import String, Boolean, UniqueConstraint, DateTime, ForeignKey +from sqlalchemy.dialects.postgresql import UUID as PgUUID +from sqlalchemy.orm import Mapped, mapped_column + +from backend.app.models.base import Base, _utcnow, _uuid7 + + +class PasswordResetToken(Base): + __tablename__ = "password_reset_tokens" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + user_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + jti_hash: Mapped[str] = mapped_column(String, nullable=False, unique=True) + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + consumed: Mapped[bool] = mapped_column(nullable=False, default=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) + + +class OtpCode(Base): + __tablename__ = "otp_codes" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + user_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + code_hash: Mapped[str] = mapped_column(String, nullable=False) + purpose: Mapped[str] = mapped_column(String, nullable=False) + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + used: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) + __table_args__ = ( + UniqueConstraint("user_id", "purpose", name="uq_otp_user_purpose"), + ) diff --git a/backend/app/models/base.py b/backend/app/models/base.py new file mode 100644 index 00000000..6f71c668 --- /dev/null +++ b/backend/app/models/base.py @@ -0,0 +1,34 @@ +from sqlalchemy.ext.compiler import compiles +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import DeclarativeBase +import os +import time +import uuid +from datetime import datetime, timezone + + +def _utcnow(): + return datetime.now(timezone.utc) + + +def _uuid7(): + try: + return uuid.uuid7() + except AttributeError: + pass + ts = int(time.time() * 1000) + rand = int.from_bytes(os.urandom(10), "big") + rand_a = rand >> 68 + rand_b = rand & ((1 << 62) - 1) + return uuid.UUID( + int=(ts << 80) | (0x7 << 76) | (rand_a << 64) | (0x2 << 62) | rand_b + ) + + +class Base(DeclarativeBase): + pass + + +@compiles(JSONB, "sqlite") +def _compile_jsonb_sqlite(type_, compiler, **kw): + return "JSON" diff --git a/backend/app/models/catalog.py b/backend/app/models/catalog.py new file mode 100644 index 00000000..d2ce1201 --- /dev/null +++ b/backend/app/models/catalog.py @@ -0,0 +1,136 @@ +import uuid +from datetime import datetime + +from sqlalchemy import String, Text, DateTime, UniqueConstraint, Index, ForeignKey +from sqlalchemy.dialects.postgresql import UUID as PgUUID +from sqlalchemy.orm import Mapped, mapped_column + +from backend.app.models.base import Base, _utcnow, _uuid7 + + +class VerifiedQA(Base): + __tablename__ = "verified_qas" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + db_id: Mapped[str] = mapped_column(String, nullable=False) + question: Mapped[str] = mapped_column(Text, nullable=False) + sql: Mapped[str] = mapped_column(Text, nullable=False, default="") + restatement: Mapped[str] = mapped_column(Text, nullable=False, default="") + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) + __table_args__ = ( + UniqueConstraint( + "workspace_id", "db_id", "question", name="uq_verified_qa_user_db_question" + ), + Index("ix_verified_qa_user_db", "workspace_id", "db_id"), + ) + + +class Glossary(Base): + __tablename__ = "glossary_terms" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + db_id: Mapped[str] = mapped_column(String, nullable=False) + term: Mapped[str] = mapped_column(String, nullable=False) + maps_to: Mapped[str] = mapped_column(String, nullable=False) + sql_hint: Mapped[str] = mapped_column(String, nullable=False, default="") + __table_args__ = (Index("ix_glossary_user_db", "workspace_id", "db_id"),) + + +class CatalogColumn(Base): + __tablename__ = "catalog_columns" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + db_id: Mapped[str] = mapped_column(String, nullable=False) + table_name: Mapped[str] = mapped_column(String, nullable=False) + column_name: Mapped[str] = mapped_column(String, nullable=False) + description: Mapped[str] = mapped_column(Text, nullable=False) + __table_args__ = (Index("ix_cat_col_user_db", "workspace_id", "db_id"),) + + +class CatalogMetric(Base): + __tablename__ = "catalog_metrics" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + db_id: Mapped[str] = mapped_column(String, nullable=False) + name: Mapped[str] = mapped_column(String, nullable=False) + description: Mapped[str] = mapped_column(Text, nullable=False, default="") + sql_expression: Mapped[str] = mapped_column(Text, nullable=False) + __table_args__ = (Index("ix_cat_met_user_db", "workspace_id", "db_id"),) + + +class CatalogJoin(Base): + __tablename__ = "catalog_joins" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + db_id: Mapped[str] = mapped_column(String, nullable=False) + tables: Mapped[str] = mapped_column(String, nullable=False) + join_condition: Mapped[str] = mapped_column(Text, nullable=False) + description: Mapped[str] = mapped_column(Text, nullable=False, default="") + __table_args__ = (Index("ix_cat_join_user_db", "workspace_id", "db_id"),) + + +class CatalogSynonym(Base): + __tablename__ = "catalog_synonyms" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + db_id: Mapped[str] = mapped_column(String, nullable=False) + term: Mapped[str] = mapped_column(String, nullable=False) + entity_type: Mapped[str] = mapped_column(String, nullable=False) + entity_name: Mapped[str] = mapped_column(String, nullable=False) + __table_args__ = (Index("ix_cat_syn_user_db", "workspace_id", "db_id"),) + + +class CatalogValueMapping(Base): + __tablename__ = "catalog_value_mappings" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + db_id: Mapped[str] = mapped_column(String, nullable=False) + table_name: Mapped[str] = mapped_column(String, nullable=False) + column_name: Mapped[str] = mapped_column(String, nullable=False) + db_value: Mapped[str] = mapped_column(String, nullable=False) + business_label: Mapped[str] = mapped_column(String, nullable=False) + __table_args__ = (Index("ix_cat_val_user_db", "workspace_id", "db_id"),) diff --git a/backend/app/models/conversation.py b/backend/app/models/conversation.py new file mode 100644 index 00000000..d8de0002 --- /dev/null +++ b/backend/app/models/conversation.py @@ -0,0 +1,67 @@ +import uuid +from datetime import datetime +from typing import Optional + +from sqlalchemy import String, Text, DateTime, ForeignKey +from sqlalchemy.dialects.postgresql import JSONB, UUID as PgUUID +from sqlalchemy.orm import Mapped, mapped_column + +from backend.app.models.base import Base, _utcnow, _uuid7 + + +class Conversation(Base): + __tablename__ = "conversations" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + # Owning user. Conversations are private to the user that created them, + # scoped within their workspace — other members cannot see them. + user_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ) + title: Mapped[str] = mapped_column(String, nullable=False, default="") + database_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) + + +class QueryHistory(Base): + __tablename__ = "query_history" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + user_id: Mapped[Optional[uuid.UUID]] = mapped_column( + PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=True + ) + question: Mapped[str] = mapped_column(Text, nullable=False) + sql: Mapped[str] = mapped_column(Text, nullable=False, default="") + result: Mapped[Optional[dict]] = mapped_column(JSONB, nullable=True) + confidence: Mapped[str] = mapped_column(String, nullable=False, default="low") + restatement: Mapped[str] = mapped_column(Text, nullable=False, default="") + # The model's chart choice for this result. Null for turns recorded before + # charts were part of the SQL generation output. + chart: Mapped[Optional[dict]] = mapped_column(JSONB, nullable=True) + conversation_id: Mapped[Optional[uuid.UUID]] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("conversations.id", ondelete="SET NULL"), + nullable=True, + ) + timestamp: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) diff --git a/backend/app/models/dashboard.py b/backend/app/models/dashboard.py new file mode 100644 index 00000000..f2f17a14 --- /dev/null +++ b/backend/app/models/dashboard.py @@ -0,0 +1,45 @@ +from sqlalchemy import Column, String, DateTime, ForeignKey +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import relationship + +from backend.app.models.base import Base, _utcnow, _uuid7 + + +class Dashboard(Base): + __tablename__ = "dashboards" + id = Column(UUID(as_uuid=True), primary_key=True, default=_uuid7) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + name = Column(String(255), nullable=False) + description = Column(String) + created_by = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL")) + created_at = Column(DateTime(timezone=True), default=_utcnow) + updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow) + + panels = relationship( + "DashboardPanel", back_populates="dashboard", cascade="all, delete" + ) + + +class DashboardPanel(Base): + __tablename__ = "dashboard_panels" + id = Column(UUID(as_uuid=True), primary_key=True, default=_uuid7) + dashboard_id = Column( + UUID(as_uuid=True), + ForeignKey("dashboards.id", ondelete="CASCADE"), + nullable=False, + ) + saved_query_id = Column( + UUID(as_uuid=True), ForeignKey("saved_queries.id", ondelete="SET NULL") + ) + title = Column(String(255)) + visualization_type = Column(String) + viz_config = Column(JSONB, default=lambda: {}) + position = Column(JSONB, default=lambda: {"x": 0, "y": 0, "w": 4, "h": 4}) + created_at = Column(DateTime(timezone=True), default=_utcnow) + updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow) + + dashboard = relationship("Dashboard", back_populates="panels") diff --git a/backend/app/models/orm_user.py b/backend/app/models/orm_user.py new file mode 100644 index 00000000..d39eeae0 --- /dev/null +++ b/backend/app/models/orm_user.py @@ -0,0 +1,34 @@ +import uuid +from datetime import datetime +from typing import Optional + +from sqlalchemy import Boolean, String, DateTime +from sqlalchemy.dialects.postgresql import UUID as PgUUID, JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from backend.app.models.base import Base, _utcnow, _uuid7 + + +class User(Base): + __tablename__ = "users" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + email: Mapped[str] = mapped_column(String, unique=True, nullable=False) + hashed_pass: Mapped[str] = mapped_column(String, nullable=False, default="") + role: Mapped[str] = mapped_column(String, nullable=False, default="user") + google_id: Mapped[Optional[str]] = mapped_column(String, unique=True, nullable=True) + supabase_id: Mapped[Optional[str]] = mapped_column( + String, unique=True, nullable=True + ) + email_verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + first_name: Mapped[Optional[str]] = mapped_column(String, nullable=True) + last_name: Mapped[Optional[str]] = mapped_column(String, nullable=True) + avatar_url: Mapped[Optional[str]] = mapped_column(String, nullable=True) + metadata_: Mapped[Optional[dict]] = mapped_column( + "metadata", JSONB, nullable=True, default=dict + ) + tour_completed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) diff --git a/backend/app/models/recent_connection.py b/backend/app/models/recent_connection.py new file mode 100644 index 00000000..cd515b4d --- /dev/null +++ b/backend/app/models/recent_connection.py @@ -0,0 +1,31 @@ +import uuid +from datetime import datetime +from typing import Optional + +from sqlalchemy import String, Integer, Text, UniqueConstraint, DateTime, ForeignKey +from sqlalchemy.dialects.postgresql import UUID as PgUUID +from sqlalchemy.orm import Mapped, mapped_column + +from backend.app.models.base import Base, _utcnow, _uuid7 + + +class RecentConnection(Base): + __tablename__ = "recent_connections" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + db_url: Mapped[str] = mapped_column(Text, nullable=False) + display_url: Mapped[str] = mapped_column(String, nullable=False) + alias_name: Mapped[Optional[str]] = mapped_column(String, nullable=True) + dialect: Mapped[str] = mapped_column(String, nullable=False) + db_id: Mapped[str] = mapped_column(String, nullable=False) + table_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + connected_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) + __table_args__ = (UniqueConstraint("workspace_id", "db_id"),) diff --git a/backend/app/models/saved_query.py b/backend/app/models/saved_query.py new file mode 100644 index 00000000..80409d55 --- /dev/null +++ b/backend/app/models/saved_query.py @@ -0,0 +1,26 @@ +from sqlalchemy import Column, String, DateTime, ForeignKey +from sqlalchemy.dialects.postgresql import JSONB, UUID + +from backend.app.models.base import Base, _utcnow, _uuid7 + + +class SavedQuery(Base): + __tablename__ = "saved_queries" + id = Column(UUID(as_uuid=True), primary_key=True, default=_uuid7) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + created_by = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL")) + name = Column(String(255), nullable=False) + description = Column(String) + question = Column(String) + sql = Column(String) + columns = Column(JSONB) + database_id = Column(String) + visualization_type = Column(String, default="table") + viz_config = Column(JSONB, default=lambda: {}) + last_result = Column(JSONB) + created_at = Column(DateTime(timezone=True), default=_utcnow) + updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow) diff --git a/backend/app/models/user.py b/backend/app/models/user.py index 94a67c61..4c6651f8 100644 --- a/backend/app/models/user.py +++ b/backend/app/models/user.py @@ -1,6 +1,6 @@ from enum import Enum -from typing import Optional -from pydantic import BaseModel, EmailStr, field_validator +from typing import Any, Optional +from pydantic import BaseModel, EmailStr, field_validator, AnyHttpUrl, Field class Role(Enum): @@ -48,12 +48,20 @@ class UserInDB(BaseModel): google_id: Optional[str] = None supabase_id: Optional[str] = None email_verified: bool = False + first_name: Optional[str] = None + last_name: Optional[str] = None + avatar_url: Optional[AnyHttpUrl] = None + metadata: Optional[dict[str, Any]] = Field(None, max_length=50) class UserPublic(BaseModel): id: str = "" email: EmailStr role: Role + first_name: Optional[str] = None + last_name: Optional[str] = None + avatar_url: Optional[AnyHttpUrl] = None + metadata: Optional[dict[str, Any]] = Field(None, max_length=50) class SupabaseLogin(BaseModel): @@ -66,3 +74,7 @@ class VerifyEmailOTP(BaseModel): class UpdateProfile(BaseModel): email: Optional[EmailStr] = None + first_name: Optional[str] = None + last_name: Optional[str] = None + avatar_url: Optional[AnyHttpUrl] = None + metadata: Optional[dict[str, Any]] = Field(None, max_length=50) diff --git a/backend/app/models/workspace.py b/backend/app/models/workspace.py new file mode 100644 index 00000000..87f27516 --- /dev/null +++ b/backend/app/models/workspace.py @@ -0,0 +1,102 @@ +""" +New: workspaces +id, name, slug, created_by (FK users), created_at + +New: workspace_members — this is the crux of the whole system +id, workspace_id (FK), user_id (FK), role (enum: owner / admin / member), joined_at, invited_by +Unique constraint on (workspace_id, user_id). Index on user_id alone too — you'll query "what workspaces is this user in" constantly for the workspace switcher. + +New: workspace_invites (can trail slightly behind if you want to ship core scoping first) +id, workspace_id, email, role, token, invited_by, expires_at, accepted_at + +Modified (9 tables): swap user_id → workspace_id, rebuild the (user_id, db_id) indexes as (workspace_id, db_id). QueryHistory gets both workspace_id and keeps user_id. +""" + +import uuid +from datetime import datetime +from sqlalchemy import ( + String, + UniqueConstraint, + CheckConstraint, + ForeignKey, + DateTime, + Index, +) +from typing import Optional +from sqlalchemy.dialects.postgresql import UUID as PgUUID +from sqlalchemy.orm import Mapped, mapped_column +from backend.app.models.base import Base, _utcnow, _uuid7 + + +class WorkspaceMember(Base): + __tablename__ = "workspace_members" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + user_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + role: Mapped[str] = mapped_column(String, nullable=False, default="member") + invited_by: Mapped[Optional[uuid.UUID]] = mapped_column( + PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + joined_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) + __table_args__ = ( + UniqueConstraint("workspace_id", "user_id", name="uq_workspace_member"), + CheckConstraint( + "role IN ('owner', 'admin', 'member')", name="ck_workspace_member_role" + ), + Index("ix_workspace_member_user", "user_id"), + ) + + +class WorkspaceInvite(Base): + __tablename__ = "workspace_invites" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + ) + email: Mapped[str] = mapped_column(String, nullable=False) + role: Mapped[str] = mapped_column(String, nullable=False, default="member") + token: Mapped[str] = mapped_column(String, unique=True, nullable=False) + invited_by: Mapped[Optional[uuid.UUID]] = mapped_column( + PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + accepted_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + __table_args__ = ( + UniqueConstraint("workspace_id", "email", name="uq_workspace_invite_email"), + CheckConstraint( + "role IN ('owner', 'admin', 'member')", name="ck_workspace_invite_role" + ), + ) + + +class Workspace(Base): + __tablename__ = "workspaces" + id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), primary_key=True, default=_uuid7 + ) + name: Mapped[str] = mapped_column(String, nullable=False) + slug: Mapped[str] = mapped_column(String, unique=True, nullable=False) + created_by: Mapped[Optional[uuid.UUID]] = mapped_column( + PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow + ) diff --git a/backend/app/models/workspace_api.py b/backend/app/models/workspace_api.py new file mode 100644 index 00000000..64f21569 --- /dev/null +++ b/backend/app/models/workspace_api.py @@ -0,0 +1,93 @@ +from typing import Any, Literal, Optional +from datetime import datetime +from pydantic import BaseModel, EmailStr, Field, field_validator + +RoleLiteral = Literal["admin", "member"] + +# Kept in sync with the client-side validator in `frontend/src/lib/validation.ts` +# so the two agree on what counts as a usable workspace name. +WORKSPACE_NAME_MIN = 2 +WORKSPACE_NAME_MAX = 60 + +# Upper bound on a single bulk-invite request, so one paste of a huge CSV can't +# turn into an unbounded run of invite emails. +MAX_BULK_INVITES = 100 + + +def _validate_workspace_name(value: str) -> str: + trimmed = (value or "").strip() + if len(trimmed) < WORKSPACE_NAME_MIN: + raise ValueError( + f"Workspace name must be at least {WORKSPACE_NAME_MIN} characters" + ) + if len(trimmed) > WORKSPACE_NAME_MAX: + raise ValueError( + f"Workspace name must be at most {WORKSPACE_NAME_MAX} characters" + ) + return trimmed + + +class WorkspaceCreate(BaseModel): + name: str + + @field_validator("name") + @classmethod + def check_name(cls, v: str) -> str: + return _validate_workspace_name(v) + + +class WorkspaceMemberRoleUpdate(BaseModel): + role: RoleLiteral + + +class WorkspaceInviteCreate(BaseModel): + email: EmailStr + role: RoleLiteral = "member" + + +class WorkspaceBulkInviteCreate(BaseModel): + # Plain strings, not EmailStr: a single malformed address should come back + # as one `invalid` row rather than 422-ing the whole batch. + emails: list[str] = Field(min_length=1, max_length=MAX_BULK_INVITES) + role: RoleLiteral = "member" + + +class WorkspaceOwnershipTransfer(BaseModel): + user_id: str + + +class WorkspaceUpdate(BaseModel): + name: str | None = None + + @field_validator("name") + @classmethod + def check_name(cls, v: str | None) -> str | None: + return None if v is None else _validate_workspace_name(v) + + +class ActivityLogResponse(BaseModel): + id: str + workspace_id: str + actor_id: Optional[str] = None + actor_email: Optional[str] = None + event_type: str + resource_type: str + resource_id: Optional[str] = None + metadata_: dict[str, Any] + created_at: datetime + + +class WorkspaceSettingsUpdate(BaseModel): + default_invite_role: Optional[RoleLiteral] = None + invite_expiry_days: Optional[int] = Field(None, ge=1, le=365) + activity_retention_days: Optional[int] = Field(None, ge=1, le=365) + role_permissions: Optional[dict[str, Any]] = None + + +class WorkspaceSettingsResponse(BaseModel): + workspace_id: str + default_invite_role: str + invite_expiry_days: int + activity_retention_days: int + role_permissions: dict[str, Any] + resolved_matrix: dict[str, dict[str, bool]] diff --git a/backend/app/models/workspace_settings.py b/backend/app/models/workspace_settings.py new file mode 100644 index 00000000..f485bb2d --- /dev/null +++ b/backend/app/models/workspace_settings.py @@ -0,0 +1,39 @@ +import uuid +from datetime import datetime + +from sqlalchemy import String, Integer, ForeignKey, DateTime +from sqlalchemy.dialects.postgresql import UUID as PgUUID, JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from backend.app.models.base import Base, _utcnow + + +class WorkspaceSettings(Base): + __tablename__ = "workspace_settings" + + workspace_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + primary_key=True, + ) + default_invite_role: Mapped[str] = mapped_column( + String(20), nullable=False, default="member" + ) + invite_expiry_days: Mapped[int] = mapped_column(Integer, nullable=False, default=7) + activity_retention_days: Mapped[int] = mapped_column( + Integer, nullable=False, default=30 + ) + role_permissions: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow, nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=_utcnow, onupdate=_utcnow, nullable=False + ) + + def __init__(self, **kw): + kw.setdefault("default_invite_role", "member") + kw.setdefault("invite_expiry_days", 7) + kw.setdefault("activity_retention_days", 30) + kw.setdefault("role_permissions", {}) + super().__init__(**kw) diff --git a/backend/app/permissions.py b/backend/app/permissions.py new file mode 100644 index 00000000..9eb49ece --- /dev/null +++ b/backend/app/permissions.py @@ -0,0 +1,256 @@ +""" +RBAC Permissions Registry and Resolution Utilities for BoloDB. + +Defines 21 fine-grained capabilities categorized across 7 resources: +- members: members.view, members.invite, members.update_role, members.remove +- connections: connections.view, connections.manage, connections.view_schema +- catalog: catalog.view, catalog.manage +- dashboards: dashboards.view, dashboards.create, dashboards.manage +- queries: queries.execute, queries.explain, queries.save, queries.delete_saved +- activity: activity.view, activity.export +- workspace_management: workspace.view, workspace.update, workspace.settings +""" + +from typing import Any, Dict, Optional, Union + +PERMISSIONS: Dict[str, Dict[str, Any]] = { + # Resource: members + "members.view": { + "key": "members.view", + "name": "View Members", + "category": "members", + "description": "View workspace member list and member details", + "default_roles": ["owner", "admin", "member"], + }, + "members.invite": { + "key": "members.invite", + "name": "Invite Members", + "category": "members", + "description": "Invite new members to join the workspace", + "default_roles": ["owner", "admin"], + }, + "members.update_role": { + "key": "members.update_role", + "name": "Update Member Roles", + "category": "members", + "description": "Change roles of workspace members", + "default_roles": ["owner", "admin"], + }, + "members.remove": { + "key": "members.remove", + "name": "Remove Members", + "category": "members", + "description": "Remove members from the workspace", + "default_roles": ["owner", "admin"], + }, + # Resource: connections + "connections.view": { + "key": "connections.view", + "name": "View Connections", + "category": "connections", + "description": "View configured database connections", + "default_roles": ["owner", "admin", "member"], + }, + "connections.manage": { + "key": "connections.manage", + "name": "Manage Connections", + "category": "connections", + "description": "Create, edit, or delete database connections", + "default_roles": ["owner", "admin"], + }, + "connections.view_schema": { + "key": "connections.view_schema", + "name": "View Connection Schema", + "category": "connections", + "description": "View schema and metadata for database connections", + "default_roles": ["owner", "admin", "member"], + }, + # Resource: catalog + "catalog.view": { + "key": "catalog.view", + "name": "View Catalog", + "category": "catalog", + "description": "View data catalog, verified Q&A, and metrics", + "default_roles": ["owner", "admin", "member"], + }, + "catalog.manage": { + "key": "catalog.manage", + "name": "Manage Catalog", + "category": "catalog", + "description": "Create or update data catalog definitions", + "default_roles": ["owner", "admin"], + }, + # Resource: dashboards + "dashboards.view": { + "key": "dashboards.view", + "name": "View Dashboards", + "category": "dashboards", + "description": "View dashboards and visualization panels", + "default_roles": ["owner", "admin", "member"], + }, + "dashboards.create": { + "key": "dashboards.create", + "name": "Create Dashboards", + "category": "dashboards", + "description": "Create new dashboards and panels", + "default_roles": ["owner", "admin"], + }, + "dashboards.manage": { + "key": "dashboards.manage", + "name": "Manage Dashboards", + "category": "dashboards", + "description": "Edit or delete existing dashboards", + "default_roles": ["owner", "admin"], + }, + # Resource: queries + "queries.execute": { + "key": "queries.execute", + "name": "Execute Queries", + "category": "queries", + "description": "Execute natural language and SQL queries", + "default_roles": ["owner", "admin", "member"], + }, + "queries.explain": { + "key": "queries.explain", + "name": "Explain Queries", + "category": "queries", + "description": "Generate query explanations and execution plans", + "default_roles": ["owner", "admin", "member"], + }, + "queries.save": { + "key": "queries.save", + "name": "Save Queries", + "category": "queries", + "description": "Save queries for workspace access", + "default_roles": ["owner", "admin", "member"], + }, + "queries.delete_saved": { + "key": "queries.delete_saved", + "name": "Delete Saved Queries", + "category": "queries", + "description": "Delete saved queries", + "default_roles": ["owner", "admin"], + }, + # Resource: activity + "activity.view": { + "key": "activity.view", + "name": "View Activity", + "category": "activity", + "description": "View workspace activity logs", + "default_roles": ["owner", "admin"], + }, + "activity.export": { + "key": "activity.export", + "name": "Export Activity", + "category": "activity", + "description": "Export workspace activity logs", + "default_roles": ["owner", "admin"], + }, + # Resource: workspace_management + "workspace.view": { + "key": "workspace.view", + "name": "View Workspace", + "category": "workspace_management", + "description": "View workspace details and configuration", + "default_roles": ["owner", "admin", "member"], + }, + "workspace.update": { + "key": "workspace.update", + "name": "Update Workspace", + "category": "workspace_management", + "description": "Update workspace basic profile details", + "default_roles": ["owner", "admin"], + }, + "workspace.settings": { + "key": "workspace.settings", + "name": "Manage Workspace Settings", + "category": "workspace_management", + "description": "Manage workspace defaults and role permission matrix", + "default_roles": ["owner", "admin"], + }, +} + +MEMBER_DEFAULT_PERMISSIONS: set[str] = { + "members.view", + "connections.view", + "connections.view_schema", + "catalog.view", + "dashboards.view", + "queries.execute", + "queries.explain", + "queries.save", + "workspace.view", +} + +DEFAULT_ROLE_PERMISSIONS: Dict[str, Dict[str, bool]] = { + "owner": {perm_key: True for perm_key in PERMISSIONS}, + "admin": {perm_key: True for perm_key in PERMISSIONS}, + "member": { + perm_key: (perm_key in MEMBER_DEFAULT_PERMISSIONS) for perm_key in PERMISSIONS + }, +} + + +def resolve_role_permissions( + role: str, overrides: Optional[Dict[str, Any]] = None +) -> Dict[str, bool]: + """ + Resolve the complete map of permission capabilities for a given role, applying + sparse overrides if provided. + + The 'owner' role always has all permissions set to True (non-editable bypass). + """ + if role == "owner": + return {perm_key: True for perm_key in PERMISSIONS} + + base = dict( + DEFAULT_ROLE_PERMISSIONS.get( + role, {perm_key: False for perm_key in PERMISSIONS} + ) + ) + + if not overrides: + return base + + # Extract role-specific overrides dictionary + role_overrides = None + if role in overrides and isinstance(overrides[role], dict): + role_overrides = overrides[role] + elif any(k in PERMISSIONS for k in overrides.keys()): + role_overrides = overrides + + if isinstance(role_overrides, dict): + for perm_key, enabled in role_overrides.items(): + if perm_key in PERMISSIONS and isinstance(enabled, bool): + base[perm_key] = enabled + + return base + + +def has_permission( + role: str, + overrides: Optional[Union[Dict[str, Any], str]] = None, + permission_key: Optional[str] = None, +) -> bool: + """ + Check if a specific role possesses a fine-grained capability key. + + Supports dual invocation formats: + - has_permission(role, overrides_dict, "queries.execute") + - has_permission(role, "queries.execute") [when no overrides provided] + - has_permission(role, overrides=overrides_dict, permission_key="queries.execute") + """ + if isinstance(overrides, str) and permission_key is None: + permission_key = overrides + overrides = None + + if not permission_key or permission_key not in PERMISSIONS: + return False + + if role == "owner": + return True + + resolved = resolve_role_permissions( + role, overrides if isinstance(overrides, dict) else None + ) + return resolved.get(permission_key, False) diff --git a/backend/app/pgdatabase/__init__.py b/backend/app/pgdatabase/__init__.py index d2914d82..6754f41d 100644 --- a/backend/app/pgdatabase/__init__.py +++ b/backend/app/pgdatabase/__init__.py @@ -1,7 +1,7 @@ """PostgreSQL application state: users, conversations, history, connections.""" from backend.app.pgdatabase.engine import get_engine, dispose_db, async_session -from backend.app.pgdatabase.models import ( +from backend.app.models import ( Base, User, Conversation, @@ -16,6 +16,9 @@ CatalogJoin, CatalogSynonym, CatalogValueMapping, + Workspace, + WorkspaceMember, + WorkspaceInvite, ) from backend.app.pgdatabase.users import ( get_user_by_email, @@ -38,6 +41,8 @@ get_recent_connections, delete_recent_connection, get_recent_connection_by_db_id, + get_latest_recent_connection, + update_recent_connection_alias, ) from backend.app.pgdatabase.conversations import ( create_conversation, @@ -70,6 +75,9 @@ "CatalogJoin", "CatalogSynonym", "CatalogValueMapping", + "Workspace", + "WorkspaceMember", + "WorkspaceInvite", "get_user_by_email", "get_user_by_google_id", "get_user_by_supabase_id", @@ -86,6 +94,8 @@ "get_recent_connections", "delete_recent_connection", "get_recent_connection_by_db_id", + "get_latest_recent_connection", + "update_recent_connection_alias", "create_conversation", "conversation_owned_by", "get_conversations", diff --git a/backend/app/pgdatabase/connections.py b/backend/app/pgdatabase/connections.py index 60ec9bb6..378b4634 100644 --- a/backend/app/pgdatabase/connections.py +++ b/backend/app/pgdatabase/connections.py @@ -1,4 +1,12 @@ -"""Recent connections CRUD with encryption.""" +"""Recent connections CRUD with encryption. + +Stored database URLs contain credentials, so they are encrypted at rest with a +key derived from ``RECENT_CONNECTIONS_KEY`` in the environment. That is the only +source — nothing is written to disk, so pointing a deploy at a fresh volume or +rotating the app image can never orphan the key. The env var must stay stable: +change it and previously saved connections can no longer be decrypted and have +to be re-added. +""" import base64 import hashlib @@ -10,69 +18,28 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert from backend.app.pgdatabase.engine import async_session -from backend.app.pgdatabase.models import RecentConnection +from backend.app.models.recent_connection import RecentConnection +from backend.app.models.base import _uuid7 from backend.app.pgdatabase.serialization import _to_uuid, serialize_doc -from backend.app.pgdatabase.models import _uuid7 -from backend.app.config import CONFIG_DIR - -_CONNECTIONS_KEY_FILE = CONFIG_DIR / "connections.key" +class ConnectionKeyError(RuntimeError): + """Raised when RECENT_CONNECTIONS_KEY is missing or unusable. -def _recent_connections_master_cipher(): - master = os.getenv("RECENT_CONNECTIONS_MASTER_KEY") - if not master: - return None - master_key = base64.urlsafe_b64encode(hashlib.sha256(master.encode()).digest()) - return Fernet(master_key) + A dedicated type so callers can tell "the server is misconfigured" (every + save will fail until an operator fixes the environment) apart from a + transient database error (worth retrying). + """ def _build_recent_connection_cipher(): secret = os.getenv("RECENT_CONNECTIONS_KEY") - if secret: - key = base64.urlsafe_b64encode(hashlib.sha256(secret.encode()).digest()) - return Fernet(key) - CONFIG_DIR.mkdir(parents=True, exist_ok=True) - master_cipher = _recent_connections_master_cipher() - if _CONNECTIONS_KEY_FILE.exists(): - persisted = _CONNECTIONS_KEY_FILE.read_text().strip() - if master_cipher: - if persisted.startswith("v1:"): - try: - loaded_secret = master_cipher.decrypt( - persisted[3:].encode() - ).decode() - except (InvalidToken, ValueError, TypeError): - raise RuntimeError( - "Failed to decrypt connections key file with RECENT_CONNECTIONS_MASTER_KEY. " - "The master key may have changed or the key file is corrupted." - ) - else: - raise RuntimeError( - "RECENT_CONNECTIONS_MASTER_KEY is set but connections key file " - "is not encrypted. Re-run with RECENT_CONNECTIONS_MASTER_KEY unset " - "to regenerate, or set RECENT_CONNECTIONS_KEY directly." - ) - else: - loaded_secret = persisted - key = base64.urlsafe_b64encode(hashlib.sha256(loaded_secret.encode()).digest()) - return Fernet(key) - if master_cipher: - new_secret = base64.urlsafe_b64encode(os.urandom(32)).decode() - _CONNECTIONS_KEY_FILE.write_text( - "v1:" + master_cipher.encrypt(new_secret.encode()).decode() - ) - try: - os.chmod(_CONNECTIONS_KEY_FILE, 0o600) - except OSError: - pass - key = base64.urlsafe_b64encode(hashlib.sha256(new_secret.encode()).digest()) - else: - raise RuntimeError( - "No encryption key configured for recent connections. Set " - "RECENT_CONNECTIONS_KEY or RECENT_CONNECTIONS_MASTER_KEY (or " - "both) in the environment, or set JWT_SECRET for migration." + if not secret: + raise ConnectionKeyError( + "RECENT_CONNECTIONS_KEY is not set. It is required to encrypt saved " + "database connections — set a stable secret in the environment." ) + key = base64.urlsafe_b64encode(hashlib.sha256(secret.encode()).digest()) return Fernet(key) @@ -91,9 +58,13 @@ def _encrypt_connection_url(db_url): def _decrypt_connection_url(value): + # A missing or unusable key configuration must not short-circuit the legacy + # and plaintext paths below — rows written before the current key scheme are + # still readable through them, and that migration path is the whole point of + # the fallbacks. Hence RuntimeError is caught alongside the Fernet errors. try: return _recent_connection_cipher().decrypt(value.encode()).decode() - except (InvalidToken, ValueError, TypeError): + except (InvalidToken, ValueError, TypeError, RuntimeError): pass jwt_secret = os.getenv("JWT_SECRET") if jwt_secret: @@ -113,9 +84,9 @@ def _decrypt_connection_url(value): async def save_recent_connection( - user_id, db_url, display_url, dialect, db_id, table_count + workspace_id, db_url, display_url, dialect, db_id, table_count, alias_name=None ): - uid = _to_uuid(user_id) + wid = _to_uuid(workspace_id) encrypted_url = _encrypt_connection_url(db_url) async with async_session() as session: try: @@ -123,20 +94,26 @@ async def save_recent_connection( pg_insert(RecentConnection) .values( id=_uuid7(), - user_id=uid, + workspace_id=wid, db_url=encrypted_url, display_url=display_url, dialect=dialect, db_id=db_id, table_count=table_count, + alias_name=alias_name, ) .on_conflict_do_update( - index_elements=["user_id", "db_id"], + index_elements=["workspace_id", "db_id"], set_=dict( db_url=encrypted_url, display_url=display_url, dialect=dialect, table_count=table_count, + alias_name=( + alias_name + if alias_name is not None + else RecentConnection.alias_name + ), connected_at=func.now(), ), ) @@ -148,34 +125,58 @@ async def save_recent_connection( raise -async def get_recent_connections(user_id: str, limit: int = 5): - uid = _to_uuid(user_id) +def _serialize_connection(row, include_url: bool = False) -> dict: + d = { + "id": row.id, + "workspace_id": row.workspace_id, + "display_url": row.display_url, + "alias_name": row.alias_name, + "dialect": row.dialect, + "db_id": row.db_id, + "table_count": row.table_count, + "connected_at": row.connected_at, + } + if include_url: + d["db_url"] = _decrypt_connection_url(row.db_url) + return serialize_doc(d) + + +async def get_recent_connections(workspace_id: str, limit: int = 50): + wid = _to_uuid(workspace_id) async with async_session() as session: result = await session.execute( select(RecentConnection) - .where(RecentConnection.user_id == uid) + .where(RecentConnection.workspace_id == wid) .order_by(RecentConnection.connected_at.desc()) .limit(limit) ) - rows = result.scalars().all() - out = [] - for row in rows: - d = { - "id": row.id, - "user_id": row.user_id, - "display_url": row.display_url, - "dialect": row.dialect, - "db_id": row.db_id, - "table_count": row.table_count, - "connected_at": row.connected_at, - } - out.append(serialize_doc(d)) - return out - - -async def delete_recent_connection(user_id: str, connection_id: str) -> bool: + return [_serialize_connection(row) for row in result.scalars().all()] + + +async def get_latest_recent_connection(workspace_id: str) -> Optional[dict]: + """The workspace's most recently used connection, with its URL decrypted. + + Used to restore a workspace's database after a server restart, when nothing + tells us *which* database to reconnect to. + """ + try: + wid = _to_uuid(workspace_id) + except (ValueError, TypeError): + return None + async with async_session() as session: + result = await session.execute( + select(RecentConnection) + .where(RecentConnection.workspace_id == wid) + .order_by(RecentConnection.connected_at.desc()) + .limit(1) + ) + row = result.scalar_one_or_none() + return _serialize_connection(row, include_url=True) if row else None + + +async def delete_recent_connection(workspace_id: str, connection_id: str) -> bool: try: - uid = _to_uuid(user_id) + wid = _to_uuid(workspace_id) cid = _to_uuid(connection_id) except (ValueError, TypeError): return False @@ -183,7 +184,7 @@ async def delete_recent_connection(user_id: str, connection_id: str) -> bool: try: result = await session.execute( delete(RecentConnection).where( - RecentConnection.id == cid, RecentConnection.user_id == uid + RecentConnection.id == cid, RecentConnection.workspace_id == wid ) ) await session.commit() @@ -193,27 +194,42 @@ async def delete_recent_connection(user_id: str, connection_id: str) -> bool: raise -async def get_recent_connection_by_db_id(user_id: str, db_id: str) -> Optional[dict]: - uid = _to_uuid(user_id) +async def get_recent_connection_by_db_id( + workspace_id: str, db_id: str +) -> Optional[dict]: + try: + wid = _to_uuid(workspace_id) + except (ValueError, TypeError): + return None async with async_session() as session: result = await session.execute( select(RecentConnection).where( - RecentConnection.user_id == uid, RecentConnection.db_id == db_id + RecentConnection.workspace_id == wid, RecentConnection.db_id == db_id ) ) conn = result.scalar_one_or_none() - if conn is None: - return None - d = { - "id": conn.id, - "user_id": conn.user_id, - "db_url": conn.db_url, - "display_url": conn.display_url, - "dialect": conn.dialect, - "db_id": conn.db_id, - "table_count": conn.table_count, - "connected_at": conn.connected_at, - } - if "db_url" in d: - d["db_url"] = _decrypt_connection_url(d["db_url"]) - return serialize_doc(d) + return _serialize_connection(conn, include_url=True) if conn else None + + +async def update_recent_connection_alias( + workspace_id: str, connection_id: str, alias_name: str | None +) -> bool: + try: + wid = _to_uuid(workspace_id) + cid = _to_uuid(connection_id) + except (ValueError, TypeError): + return False + from sqlalchemy import update + + async with async_session() as session: + try: + result = await session.execute( + update(RecentConnection) + .where(RecentConnection.id == cid, RecentConnection.workspace_id == wid) + .values(alias_name=alias_name) + ) + await session.commit() + return result.rowcount > 0 + except Exception: + await session.rollback() + raise diff --git a/backend/app/pgdatabase/conversations.py b/backend/app/pgdatabase/conversations.py index 8e7eca33..98a180ac 100644 --- a/backend/app/pgdatabase/conversations.py +++ b/backend/app/pgdatabase/conversations.py @@ -6,7 +6,8 @@ from sqlalchemy import select, delete, update, text from backend.app.pgdatabase.engine import async_session -from backend.app.pgdatabase.models import Conversation, QueryHistory, _utcnow +from backend.app.models.conversation import Conversation, QueryHistory +from backend.app.models.base import _utcnow from backend.app.pgdatabase.serialization import _to_uuid, serialize_doc # Max result rows returned per turn when restoring a conversation. @@ -15,12 +16,14 @@ MAX_SAVED_ROWS = 500 -async def create_conversation(user_id, title="", database_id=None): - uid = _to_uuid(user_id) +async def create_conversation(workspace_id, user_id, title="", database_id=None): + uid = _to_uuid(workspace_id) + usr = _to_uuid(user_id) async with async_session() as session: try: conv = Conversation( - user_id=uid, + workspace_id=uid, + user_id=usr, title=title, database_id=database_id, ) @@ -32,7 +35,8 @@ async def create_conversation(user_id, title="", database_id=None): raise d = { "id": conv.id, - "user_id": conv.user_id, + "_id": conv.id, + "workspace_id": conv.workspace_id, "title": conv.title, "database_id": conv.database_id, "created_at": conv.created_at, @@ -41,27 +45,33 @@ async def create_conversation(user_id, title="", database_id=None): return serialize_doc(d) -async def conversation_owned_by(user_id: str, conversation_id: str) -> bool: +async def conversation_owned_by( + workspace_id: str, user_id: str, conversation_id: str +) -> bool: try: - uid = _to_uuid(user_id) + uid = _to_uuid(workspace_id) + usr = _to_uuid(user_id) cid = _to_uuid(conversation_id) except (ValueError, TypeError): return False async with async_session() as session: result = await session.execute( select(Conversation.id).where( - Conversation.id == cid, Conversation.user_id == uid + Conversation.id == cid, + Conversation.workspace_id == uid, + Conversation.user_id == usr, ) ) return result.scalar_one_or_none() is not None -async def get_conversations(user_id: str): - uid = _to_uuid(user_id) +async def get_conversations(workspace_id: str, user_id: str): + uid = _to_uuid(workspace_id) + usr = _to_uuid(user_id) async with async_session() as session: conv_result = await session.execute( select(Conversation) - .where(Conversation.user_id == uid) + .where(Conversation.workspace_id == uid, Conversation.user_id == usr) .order_by(Conversation.updated_at.desc()) ) convs = conv_result.scalars().all() @@ -76,7 +86,7 @@ async def get_conversations(user_id: str): "SELECT qh.conversation_id, COUNT(*) AS turn_count, " "MAX(qh.timestamp) AS last_ts " "FROM query_history qh " - "WHERE qh.user_id = :uid AND qh.conversation_id = ANY(CAST(:cids AS uuid[])) " + "WHERE qh.workspace_id = :uid AND qh.conversation_id = ANY(CAST(:cids AS uuid[])) " "GROUP BY qh.conversation_id" ), {"uid": uid, "cids": cid_params}, @@ -91,7 +101,7 @@ async def get_conversations(user_id: str): text( "SELECT DISTINCT ON (qh.conversation_id) qh.conversation_id, qh.question " "FROM query_history qh " - "WHERE qh.user_id = :uid AND qh.conversation_id = ANY(CAST(:cids AS uuid[])) " + "WHERE qh.workspace_id = :uid AND qh.conversation_id = ANY(CAST(:cids AS uuid[])) " "ORDER BY qh.conversation_id, qh.timestamp DESC" ), {"uid": uid, "cids": cid_params}, @@ -102,7 +112,8 @@ async def get_conversations(user_id: str): for conv in convs: d = { "id": conv.id, - "user_id": conv.user_id, + "_id": conv.id, + "workspace_id": conv.workspace_id, "title": conv.title, "database_id": conv.database_id, "created_at": conv.created_at, @@ -115,16 +126,19 @@ async def get_conversations(user_id: str): return out -async def get_conversation(user_id: str, conversation_id: str): +async def get_conversation(workspace_id: str, user_id: str, conversation_id: str): try: - uid = _to_uuid(user_id) + uid = _to_uuid(workspace_id) + usr = _to_uuid(user_id) cid = _to_uuid(conversation_id) except (ValueError, TypeError): return None async with async_session() as session: result = await session.execute( select(Conversation).where( - Conversation.id == cid, Conversation.user_id == uid + Conversation.id == cid, + Conversation.workspace_id == uid, + Conversation.user_id == usr, ) ) conv = result.scalar_one_or_none() @@ -132,7 +146,8 @@ async def get_conversation(user_id: str, conversation_id: str): return None d = { "id": conv.id, - "user_id": conv.user_id, + "_id": conv.id, + "workspace_id": conv.workspace_id, "title": conv.title, "database_id": conv.database_id, "created_at": conv.created_at, @@ -144,7 +159,7 @@ async def get_conversation(user_id: str, conversation_id: str): select(QueryHistory) .where( QueryHistory.conversation_id == cid, - QueryHistory.user_id == uid, + QueryHistory.workspace_id == uid, ) .order_by(QueryHistory.timestamp.asc()) ) @@ -157,13 +172,15 @@ async def get_conversation(user_id: str, conversation_id: str): truncated = len(result) > MAX_RESTORED_ROWS td = { "id": turn.id, - "user_id": turn.user_id, + "_id": turn.id, + "workspace_id": turn.workspace_id, "question": turn.question, "sql": turn.sql, "result": result[:MAX_RESTORED_ROWS], "result_truncated": truncated, "confidence": turn.confidence, "restatement": turn.restatement, + "chart": turn.chart, "conversation_id": turn.conversation_id, "timestamp": turn.timestamp, } @@ -172,9 +189,12 @@ async def get_conversation(user_id: str, conversation_id: str): return d -async def rename_conversation(user_id: str, conversation_id: str, title: str) -> bool: +async def rename_conversation( + workspace_id: str, user_id: str, conversation_id: str, title: str +) -> bool: try: - uid = _to_uuid(user_id) + uid = _to_uuid(workspace_id) + usr = _to_uuid(user_id) cid = _to_uuid(conversation_id) except (ValueError, TypeError): return False @@ -182,7 +202,11 @@ async def rename_conversation(user_id: str, conversation_id: str, title: str) -> try: result = await session.execute( update(Conversation) - .where(Conversation.id == cid, Conversation.user_id == uid) + .where( + Conversation.id == cid, + Conversation.workspace_id == uid, + Conversation.user_id == usr, + ) .values(title=title, updated_at=_utcnow()) ) await session.commit() @@ -192,27 +216,30 @@ async def rename_conversation(user_id: str, conversation_id: str, title: str) -> raise -async def touch_conversation(conversation_id: str): +async def touch_conversation(conversation_id: str, user_id: str = None): try: cid = _to_uuid(conversation_id) + usr = _to_uuid(user_id) if user_id else None except (ValueError, TypeError): return async with async_session() as session: try: - await session.execute( - update(Conversation) - .where(Conversation.id == cid) - .values(updated_at=_utcnow()) - ) + stmt = update(Conversation).where(Conversation.id == cid) + if usr is not None: + stmt = stmt.where(Conversation.user_id == usr) + await session.execute(stmt.values(updated_at=_utcnow())) await session.commit() except Exception: await session.rollback() raise -async def delete_conversation(user_id: str, conversation_id: str) -> bool: +async def delete_conversation( + workspace_id: str, user_id: str, conversation_id: str +) -> bool: try: - uid = _to_uuid(user_id) + uid = _to_uuid(workspace_id) + usr = _to_uuid(user_id) cid = _to_uuid(conversation_id) except (ValueError, TypeError): return False @@ -220,7 +247,9 @@ async def delete_conversation(user_id: str, conversation_id: str) -> bool: async with session.begin(): owner = await session.execute( select(Conversation.id).where( - Conversation.id == cid, Conversation.user_id == uid + Conversation.id == cid, + Conversation.workspace_id == uid, + Conversation.user_id == usr, ) ) if owner.scalar_one_or_none() is None: @@ -228,32 +257,39 @@ async def delete_conversation(user_id: str, conversation_id: str) -> bool: await session.execute( delete(QueryHistory).where( QueryHistory.conversation_id == cid, - QueryHistory.user_id == uid, + QueryHistory.workspace_id == uid, ) ) await session.execute( delete(Conversation).where( - Conversation.id == cid, Conversation.user_id == uid + Conversation.id == cid, + Conversation.workspace_id == uid, + Conversation.user_id == usr, ) ) return True -async def clear_conversations(user_id: str): - uid = _to_uuid(user_id) +async def clear_conversations(workspace_id: str, user_id: str): + uid = _to_uuid(workspace_id) + usr = _to_uuid(user_id) async with async_session() as session: async with session.begin(): conv_ids_result = await session.execute( - select(Conversation.id).where(Conversation.user_id == uid) + select(Conversation.id).where( + Conversation.workspace_id == uid, Conversation.user_id == usr + ) ) conv_ids = [row[0] for row in conv_ids_result] if conv_ids: await session.execute( delete(QueryHistory).where( QueryHistory.conversation_id.in_(conv_ids), - QueryHistory.user_id == uid, + QueryHistory.workspace_id == uid, ) ) await session.execute( - delete(Conversation).where(Conversation.user_id == uid) + delete(Conversation).where( + Conversation.workspace_id == uid, Conversation.user_id == usr + ) ) diff --git a/backend/app/pgdatabase/dashboards.py b/backend/app/pgdatabase/dashboards.py new file mode 100644 index 00000000..b0763f0a --- /dev/null +++ b/backend/app/pgdatabase/dashboards.py @@ -0,0 +1,221 @@ +from sqlalchemy import select, delete, update +from sqlalchemy.orm import selectinload +from backend.app.pgdatabase.engine import async_session +from backend.app.models.dashboard import Dashboard, DashboardPanel +from backend.app.models.saved_query import SavedQuery +from backend.app.models.base import _utcnow +from backend.app.pgdatabase.serialization import _to_uuid, serialize_doc + + +async def create_dashboard( + workspace_id: str, created_by: str, name: str, description: str = None +): + uid = _to_uuid(workspace_id) + c_uid = _to_uuid(created_by) if created_by else None + + async with async_session() as session: + try: + d = Dashboard( + workspace_id=uid, created_by=c_uid, name=name, description=description + ) + session.add(d) + await session.commit() + await session.refresh(d) + return serialize_doc(d.__dict__) + except Exception: + await session.rollback() + raise + + +async def list_dashboards(workspace_id: str): + uid = _to_uuid(workspace_id) + async with async_session() as session: + result = await session.execute( + select(Dashboard) + .where(Dashboard.workspace_id == uid) + .order_by(Dashboard.updated_at.desc()) + ) + return [serialize_doc(r.__dict__) for r in result.scalars().all()] + + +async def get_dashboard(workspace_id: str, dashboard_id: str): + uid = _to_uuid(workspace_id) + did = _to_uuid(dashboard_id) + async with async_session() as session: + result = await session.execute( + select(Dashboard) + .options(selectinload(Dashboard.panels)) + .where(Dashboard.id == did, Dashboard.workspace_id == uid) + ) + d = result.scalar_one_or_none() + if not d: + return None + + doc = serialize_doc(d.__dict__) + doc["panels"] = [serialize_doc(p.__dict__) for p in d.panels] + return doc + + +async def update_dashboard(workspace_id: str, dashboard_id: str, **kwargs): + uid = _to_uuid(workspace_id) + did = _to_uuid(dashboard_id) + async with async_session() as session: + try: + kwargs["updated_at"] = _utcnow() + result = await session.execute( + update(Dashboard) + .where(Dashboard.id == did, Dashboard.workspace_id == uid) + .values(**kwargs) + ) + await session.commit() + return result.rowcount > 0 + except Exception: + await session.rollback() + raise + + +async def delete_dashboard(workspace_id: str, dashboard_id: str): + uid = _to_uuid(workspace_id) + did = _to_uuid(dashboard_id) + async with async_session() as session: + try: + result = await session.execute( + delete(Dashboard).where( + Dashboard.id == did, Dashboard.workspace_id == uid + ) + ) + await session.commit() + return result.rowcount > 0 + except Exception: + await session.rollback() + raise + + +async def add_panel(workspace_id: str, dashboard_id: str, **kwargs): + wid = _to_uuid(workspace_id) + did = _to_uuid(dashboard_id) + if "saved_query_id" in kwargs and kwargs["saved_query_id"]: + kwargs["saved_query_id"] = _to_uuid(kwargs["saved_query_id"]) + + async with async_session() as session: + try: + d = await session.execute( + select(Dashboard).where( + Dashboard.id == did, Dashboard.workspace_id == wid + ) + ) + if not d.scalar_one_or_none(): + raise ValueError("Dashboard not found in workspace") + + if kwargs.get("saved_query_id"): + sq = await session.execute( + select(SavedQuery.id).where( + SavedQuery.id == kwargs["saved_query_id"], + SavedQuery.workspace_id == wid, + ) + ) + if not sq.scalar_one_or_none(): + raise ValueError("Saved query not found in workspace") + + panel = DashboardPanel(dashboard_id=did, **kwargs) + session.add(panel) + await session.commit() + await session.refresh(panel) + return serialize_doc(panel.__dict__) + except Exception: + await session.rollback() + raise + + +async def update_panel(workspace_id: str, dashboard_id: str, panel_id: str, **kwargs): + wid = _to_uuid(workspace_id) + did = _to_uuid(dashboard_id) + pid = _to_uuid(panel_id) + + if "saved_query_id" in kwargs and kwargs["saved_query_id"]: + kwargs["saved_query_id"] = _to_uuid(kwargs["saved_query_id"]) + + async with async_session() as session: + try: + d = await session.execute( + select(Dashboard).where( + Dashboard.id == did, Dashboard.workspace_id == wid + ) + ) + if not d.scalar_one_or_none(): + raise ValueError("Dashboard not found in workspace") + + if kwargs.get("saved_query_id"): + sq = await session.execute( + select(SavedQuery.id).where( + SavedQuery.id == kwargs["saved_query_id"], + SavedQuery.workspace_id == wid, + ) + ) + if not sq.scalar_one_or_none(): + raise ValueError("Saved query not found in workspace") + + kwargs["updated_at"] = _utcnow() + result = await session.execute( + update(DashboardPanel) + .where(DashboardPanel.id == pid, DashboardPanel.dashboard_id == did) + .values(**kwargs) + ) + await session.commit() + return result.rowcount > 0 + except Exception: + await session.rollback() + raise + + +async def delete_panel(workspace_id: str, dashboard_id: str, panel_id: str): + wid = _to_uuid(workspace_id) + did = _to_uuid(dashboard_id) + pid = _to_uuid(panel_id) + async with async_session() as session: + try: + d = await session.execute( + select(Dashboard).where( + Dashboard.id == did, Dashboard.workspace_id == wid + ) + ) + if not d.scalar_one_or_none(): + return False + + result = await session.execute( + delete(DashboardPanel).where( + DashboardPanel.id == pid, DashboardPanel.dashboard_id == did + ) + ) + await session.commit() + return result.rowcount > 0 + except Exception: + await session.rollback() + raise + + +async def update_panels_batch(workspace_id: str, dashboard_id: str, updates: list): + wid = _to_uuid(workspace_id) + did = _to_uuid(dashboard_id) + async with async_session() as session: + try: + d = await session.execute( + select(Dashboard).where( + Dashboard.id == did, Dashboard.workspace_id == wid + ) + ) + if not d.scalar_one_or_none(): + return False + + for u in updates: + pid = _to_uuid(u["id"]) + await session.execute( + update(DashboardPanel) + .where(DashboardPanel.id == pid, DashboardPanel.dashboard_id == did) + .values(position=u["position"], updated_at=_utcnow()) + ) + await session.commit() + return True + except Exception: + await session.rollback() + raise diff --git a/backend/app/pgdatabase/history.py b/backend/app/pgdatabase/history.py index a86727ff..94da6653 100644 --- a/backend/app/pgdatabase/history.py +++ b/backend/app/pgdatabase/history.py @@ -3,24 +3,35 @@ from sqlalchemy import select, delete, func, text from backend.app.pgdatabase.engine import async_session -from backend.app.pgdatabase.models import QueryHistory +from backend.app.models.conversation import QueryHistory from backend.app.pgdatabase.serialization import _to_uuid, serialize_doc async def save_query( - user_id, question, sql, result, confidence, conversation_id=None, restatement="" + workspace_id, + question, + sql, + result, + confidence, + conversation_id=None, + restatement="", + user_id=None, + chart=None, ): - uid = _to_uuid(user_id) + wid = _to_uuid(workspace_id) + uid = _to_uuid(user_id) if user_id else None conv_id = _to_uuid(conversation_id) if conversation_id else None async with async_session() as session: try: qh = QueryHistory( + workspace_id=wid, user_id=uid, question=question, sql=sql, result=result, confidence=confidence, restatement=restatement, + chart=chart, conversation_id=conv_id, ) session.add(qh) @@ -30,12 +41,12 @@ async def save_query( raise -async def get_query_history(user_id: str, limit: int = 100): - uid = _to_uuid(user_id) +async def get_query_history(workspace_id: str, limit: int = 100): + uid = _to_uuid(workspace_id) async with async_session() as session: result = await session.execute( select(QueryHistory) - .where(QueryHistory.user_id == uid) + .where(QueryHistory.workspace_id == uid) .order_by(QueryHistory.timestamp.desc()) .limit(limit) ) @@ -44,6 +55,7 @@ async def get_query_history(user_id: str, limit: int = 100): for row in rows: d = { "id": row.id, + "workspace_id": row.workspace_id, "user_id": row.user_id, "question": row.question, "sql": row.sql, @@ -57,18 +69,18 @@ async def get_query_history(user_id: str, limit: int = 100): return out -async def get_query_stats(user_id: str): - uid = _to_uuid(user_id) +async def get_query_stats(workspace_id: str): + uid = _to_uuid(workspace_id) async with async_session() as session: total_q = await session.scalar( - select(func.count()).where(QueryHistory.user_id == uid) + select(func.count()).where(QueryHistory.workspace_id == uid) ) total = total_q or 0 conf_rows = await session.execute( text( "SELECT LOWER(COALESCE(confidence, 'low')) AS level, COUNT(*) AS count " - "FROM query_history WHERE user_id = :uid GROUP BY LOWER(COALESCE(confidence, 'low'))" + "FROM query_history WHERE workspace_id = :uid GROUP BY LOWER(COALESCE(confidence, 'low'))" ).bindparams(uid=uid) ) confidence_counts = {"High": 0, "Medium": 0, "Low": 0} @@ -79,7 +91,7 @@ async def get_query_stats(user_id: str): day_rows = await session.execute( text( "SELECT DATE(timestamp) AS date, COUNT(*) AS count " - "FROM query_history WHERE user_id = :uid " + "FROM query_history WHERE workspace_id = :uid " "GROUP BY DATE(timestamp) ORDER BY date DESC LIMIT 90" ).bindparams(uid=uid) ) @@ -88,7 +100,7 @@ async def get_query_stats(user_id: str): sql_rows = await session.execute( text( - "SELECT sql FROM query_history WHERE user_id = :uid " + "SELECT sql FROM query_history WHERE workspace_id = :uid " "ORDER BY timestamp DESC LIMIT 200" ).bindparams(uid=uid) ) @@ -140,9 +152,9 @@ async def get_query_stats(user_id: str): } -async def delete_history_entry(user_id: str, entry_id: str) -> bool: +async def delete_history_entry(workspace_id: str, entry_id: str) -> bool: try: - uid = _to_uuid(user_id) + uid = _to_uuid(workspace_id) eid = _to_uuid(entry_id) except (ValueError, TypeError): return False @@ -150,7 +162,7 @@ async def delete_history_entry(user_id: str, entry_id: str) -> bool: try: result = await session.execute( delete(QueryHistory).where( - QueryHistory.id == eid, QueryHistory.user_id == uid + QueryHistory.id == eid, QueryHistory.workspace_id == uid ) ) await session.commit() @@ -160,12 +172,12 @@ async def delete_history_entry(user_id: str, entry_id: str) -> bool: raise -async def clear_history(user_id: str): - uid = _to_uuid(user_id) +async def clear_history(workspace_id: str): + uid = _to_uuid(workspace_id) async with async_session() as session: try: await session.execute( - delete(QueryHistory).where(QueryHistory.user_id == uid) + delete(QueryHistory).where(QueryHistory.workspace_id == uid) ) await session.commit() except Exception: diff --git a/backend/app/pgdatabase/knowledge.py b/backend/app/pgdatabase/knowledge.py index 874131ac..895c7b6b 100644 --- a/backend/app/pgdatabase/knowledge.py +++ b/backend/app/pgdatabase/knowledge.py @@ -8,7 +8,7 @@ from sqlalchemy import select, delete, func from sqlalchemy.exc import IntegrityError -from backend.app.pgdatabase.models import ( +from backend.app.models.catalog import ( VerifiedQA, Glossary, CatalogColumn, @@ -56,12 +56,12 @@ def __init__(self, session_factory): """ self._session_factory = session_factory - async def add_verified(self, user_id, db_id, question, sql, restatement=""): + async def add_verified(self, workspace_id, db_id, question, sql, restatement=""): """ Add a verified question-and-answer entry unless a sufficiently similar question already exists. Parameters: - user_id: Identifier of the user who owns the entry. + workspace_id: Identifier of the user who owns the entry. db_id: Identifier of the database associated with the entry. question: Natural-language question associated with the SQL query. sql: SQL query that answers the question. @@ -71,7 +71,7 @@ async def add_verified(self, user_id, db_id, question, sql, restatement=""): async with self._session_factory() as session: result = await session.execute( select(VerifiedQA).where( - VerifiedQA.user_id == user_id, VerifiedQA.db_id == db_id + VerifiedQA.workspace_id == workspace_id, VerifiedQA.db_id == db_id ) ) tb = _tokens(question) @@ -81,7 +81,7 @@ async def add_verified(self, user_id, db_id, question, sql, restatement=""): return session.add( VerifiedQA( - user_id=user_id, + workspace_id=workspace_id, db_id=db_id, question=question, sql=sql, @@ -93,18 +93,18 @@ async def add_verified(self, user_id, db_id, question, sql, restatement=""): except IntegrityError: await session.rollback() logger.warning( - "Duplicate verified QA skipped (user=%s db=%s question=%s)", - user_id, + "Duplicate verified QA skipped (workspace=%s db=%s question=%s)", + workspace_id, db_id, question, ) - async def get_verified(self, user_id, db_id): + async def get_verified(self, workspace_id, db_id): """ Retrieve verified question-and-answer entries for a user and database, with newest entries first. Parameters: - user_id: Identifier of the user. + workspace_id: Identifier of the user. db_id: Identifier of the database. Returns: @@ -113,7 +113,9 @@ async def get_verified(self, user_id, db_id): async with self._session_factory() as session: result = await session.execute( select(VerifiedQA) - .where(VerifiedQA.user_id == user_id, VerifiedQA.db_id == db_id) + .where( + VerifiedQA.workspace_id == workspace_id, VerifiedQA.db_id == db_id + ) .order_by(VerifiedQA.created_at.desc()) ) return [ @@ -121,12 +123,12 @@ async def get_verified(self, user_id, db_id): for r in result.scalars().all() ] - async def count_verified(self, user_id, db_id): + async def count_verified(self, workspace_id, db_id): """ Count verified Q&A entries for a user and database. Parameters: - user_id: Identifier of the user. + workspace_id: Identifier of the user. db_id: Identifier of the database. Returns: @@ -135,12 +137,14 @@ async def count_verified(self, user_id, db_id): async with self._session_factory() as session: result = await session.execute( select(func.count()).where( - VerifiedQA.user_id == user_id, VerifiedQA.db_id == db_id + VerifiedQA.workspace_id == workspace_id, VerifiedQA.db_id == db_id ) ) return result.scalar() or 0 - async def retrieve_similar(self, user_id, db_id, question, k=3, threshold=0.25): + async def retrieve_similar( + self, workspace_id, db_id, question, k=3, threshold=0.25 + ): """ Finds verified questions that are similar to the provided question. @@ -155,14 +159,14 @@ async def retrieve_similar(self, user_id, db_id, question, k=3, threshold=0.25): scored = [] tb = _tokens(question) b_lower = question.lower() - for c in await self.get_verified(user_id, db_id): + for c in await self.get_verified(workspace_id, db_id): s = _similarity(c["question"], question, tb, b_lower) if s >= threshold: scored.append({**c, "similarity": round(s, 3)}) scored.sort(key=lambda x: -x["similarity"]) return scored[:k] - async def set_glossary(self, user_id, db_id, terms): + async def set_glossary(self, workspace_id, db_id, terms): """ Replace the glossary entries for a user and database. @@ -172,13 +176,13 @@ async def set_glossary(self, user_id, db_id, terms): async with self._session_factory() as session: await session.execute( delete(Glossary).where( - Glossary.user_id == user_id, Glossary.db_id == db_id + Glossary.workspace_id == workspace_id, Glossary.db_id == db_id ) ) for t in terms: session.add( Glossary( - user_id=user_id, + workspace_id=workspace_id, db_id=db_id, term=t.get("term", ""), maps_to=t.get("maps_to", ""), @@ -187,11 +191,11 @@ async def set_glossary(self, user_id, db_id, terms): ) await session.commit() - async def get_glossary(self, user_id, db_id): + async def get_glossary(self, workspace_id, db_id): """Retrieve glossary entries for a user and database. Parameters: - user_id: The user identifier. + workspace_id: The user identifier. db_id: The database identifier. Returns: @@ -200,7 +204,7 @@ async def get_glossary(self, user_id, db_id): async with self._session_factory() as session: result = await session.execute( select(Glossary).where( - Glossary.user_id == user_id, Glossary.db_id == db_id + Glossary.workspace_id == workspace_id, Glossary.db_id == db_id ) ) return [ @@ -236,7 +240,7 @@ async def get_glossary(self, user_id, db_id): ), } - async def set_catalog(self, user_id, db_id, catalog): + async def set_catalog(self, workspace_id, db_id, catalog): """ Replace the specified catalog sections for a user and database. @@ -249,22 +253,23 @@ async def set_catalog(self, user_id, db_id, catalog): continue await session.execute( delete(model_class).where( - model_class.user_id == user_id, model_class.db_id == db_id + model_class.workspace_id == workspace_id, + model_class.db_id == db_id, ) ) for entry in catalog[key] or []: - kwargs = {"user_id": user_id, "db_id": db_id} + kwargs = {"workspace_id": workspace_id, "db_id": db_id} for c, ok in zip(cols, in_keys): kwargs[c] = str(entry.get(ok, "") or "") session.add(model_class(**kwargs)) await session.commit() - async def get_catalog(self, user_id, db_id): + async def get_catalog(self, workspace_id, db_id): """ Retrieve the semantic catalog entries for a user and database. Parameters: - user_id: Identifier of the user whose catalog entries are retrieved. + workspace_id: Identifier of the user whose catalog entries are retrieved. db_id: Identifier of the database whose catalog entries are retrieved. Returns: @@ -275,7 +280,8 @@ async def get_catalog(self, user_id, db_id): for key, (model_class, cols, out_keys) in self._CATALOG_CLASSES.items(): result = await session.execute( select(model_class).where( - model_class.user_id == user_id, model_class.db_id == db_id + model_class.workspace_id == workspace_id, + model_class.db_id == db_id, ) ) rows = [] @@ -287,20 +293,20 @@ async def get_catalog(self, user_id, db_id): out[key] = rows return out - async def catalog_is_empty(self, user_id, db_id): + async def catalog_is_empty(self, workspace_id, db_id): """Determine whether a user's catalog contains any entries. Parameters: - user_id: The user whose catalog is checked. + workspace_id: The user whose catalog is checked. db_id: The database whose catalog is checked. Returns: bool: `True` if all catalog sections are empty, `False` otherwise. """ - catalog = await self.get_catalog(user_id, db_id) + catalog = await self.get_catalog(workspace_id, db_id) return not any(catalog.values()) - async def seed_sample(self, user_id, db_id): + async def seed_sample(self, workspace_id, db_id): """Seed sample knowledge for a new sample-database connection. All operations run in one transaction so partial failures roll back @@ -309,70 +315,84 @@ async def seed_sample(self, user_id, db_id): async with self._session_factory() as session: count = await session.execute( select(func.count()).where( - VerifiedQA.user_id == user_id, VerifiedQA.db_id == db_id + VerifiedQA.workspace_id == workspace_id, VerifiedQA.db_id == db_id ) ) if count.scalar(): return await session.execute( delete(Glossary).where( - Glossary.user_id == user_id, Glossary.db_id == db_id + Glossary.workspace_id == workspace_id, Glossary.db_id == db_id ) ) for t in [ { "term": "Revenue", - "maps_to": "orders.total_amount", - "sql_hint": "Sum of total_amount on orders with status = completed", + "maps_to": "orders.total", + "sql_hint": "Sum of orders.total, which already includes every order position", }, { "term": "Active customer", - "maps_to": "orders.created_at", + "maps_to": "orders.ordertimestamp", "sql_hint": "A customer with at least one order in the last 90 days", }, + { + "term": "Article", + "maps_to": "articles.productid", + "sql_hint": "One product in a specific size and colour — products group articles", + }, { "term": "Top product", - "maps_to": "order_items.quantity", - "sql_hint": "Product ranked by units sold (sum of quantity)", + "maps_to": "order_positions.amount", + "sql_hint": "Product ranked by units sold: sum of order_positions.amount, joined through articles", }, ]: - session.add(Glossary(user_id=user_id, db_id=db_id, **t)) + session.add(Glossary(workspace_id=workspace_id, db_id=db_id, **t)) for q, s, r in [ ( - "How many orders were completed last month?", - "SELECT COUNT(*) AS completed_orders\nFROM orders\nWHERE status = 'completed'\n AND created_at >= date('now','start of month','-1 month')\n AND created_at < date('now','start of month');", - "Count of orders with status 'completed' created in the previous calendar month", + "How many orders were placed last month?", + "SELECT COUNT(*) AS orders_placed\nFROM orders\nWHERE ordertimestamp >= date('now','start of month','-1 month')\n AND ordertimestamp < date('now','start of month');", + "Count of orders placed in the previous calendar month", ), ( "Which product category brings in the most revenue?", - "SELECT p.category, ROUND(SUM(oi.quantity*oi.unit_price)) AS revenue\nFROM order_items oi\nJOIN products p ON p.id = oi.product_id\nJOIN orders o ON o.id = oi.order_id\nWHERE o.status = 'completed'\nGROUP BY p.category\nORDER BY revenue DESC;", + "SELECT p.category, ROUND(SUM(op.amount * op.price), 2) AS revenue\nFROM order_positions op\nJOIN articles a ON a.id = op.articleid\nJOIN products p ON p.id = a.productid\nGROUP BY p.category\nORDER BY revenue DESC;", "Total revenue per product category, highest first", ), ( - "How many customers do we have in each segment?", - "SELECT segment, COUNT(*) AS customers\nFROM customers\nGROUP BY segment\nORDER BY customers DESC;", - "Count of customers grouped by segment", + "Who are our top 10 customers by total spend?", + "SELECT c.firstname || ' ' || c.lastname AS customer, ROUND(SUM(o.total), 2) AS total_spend\nFROM orders o\nJOIN customers c ON c.id = o.customerid\nGROUP BY c.id\nORDER BY total_spend DESC\nLIMIT 10;", + "The ten customers with the highest sum of order totals", + ), + ( + "Which articles are out of stock?", + "SELECT a.id, a.description, s.count AS in_stock\nFROM articles a\nJOIN stock s ON s.articleid = a.id\nWHERE s.count = 0\n AND a.currentlyactive = 1;", + "Active articles whose stock count has reached zero", ), ]: session.add( VerifiedQA( - user_id=user_id, db_id=db_id, question=q, sql=s, restatement=r + workspace_id=workspace_id, + db_id=db_id, + question=q, + sql=s, + restatement=r, ) ) await session.commit() - async def trust_level(self, user_id, db_id): + async def trust_level(self, workspace_id, db_id): """ Summarize the trust level for a user's database from its verified answers. Parameters: - user_id: Identifier of the user. + workspace_id: Identifier of the user. db_id: Identifier of the database. Returns: A dictionary containing the trust `level`, verified-answer count, percentage, and explanatory note. """ - n = await self.count_verified(user_id, db_id) + n = await self.count_verified(workspace_id, db_id) if n >= 7: return { "level": "Trusted", diff --git a/backend/app/pgdatabase/models.py b/backend/app/pgdatabase/models.py deleted file mode 100644 index 49c3da6f..00000000 --- a/backend/app/pgdatabase/models.py +++ /dev/null @@ -1,275 +0,0 @@ -"""SQLAlchemy ORM models for application state.""" - -import os -import time -import uuid -from datetime import datetime, timezone -from typing import Optional - -from sqlalchemy import ( - Boolean, - String, - Integer, - UniqueConstraint, - ForeignKey, - Text, - DateTime, - Index, -) -from sqlalchemy.dialects.postgresql import JSONB, UUID as PgUUID -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column - - -def _utcnow(): - return datetime.now(timezone.utc) - - -def _uuid7(): - try: - return uuid.uuid7() - except AttributeError: - pass - ts = int(time.time() * 1000) - rand = int.from_bytes(os.urandom(10), "big") - rand_a = rand >> 68 - rand_b = rand & ((1 << 62) - 1) - return uuid.UUID( - int=(ts << 80) | (0x7 << 76) | (rand_a << 64) | (0x2 << 62) | rand_b - ) - - -class Base(DeclarativeBase): - pass - - -class User(Base): - __tablename__ = "users" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - email: Mapped[str] = mapped_column(String, unique=True, nullable=False) - hashed_pass: Mapped[str] = mapped_column(String, nullable=False, default="") - role: Mapped[str] = mapped_column(String, nullable=False, default="user") - google_id: Mapped[Optional[str]] = mapped_column(String, unique=True, nullable=True) - supabase_id: Mapped[Optional[str]] = mapped_column( - String, unique=True, nullable=True - ) - email_verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - tour_completed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=_utcnow - ) - - -class Conversation(Base): - __tablename__ = "conversations" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - title: Mapped[str] = mapped_column(String, nullable=False, default="") - database_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=_utcnow - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=_utcnow - ) - - -class QueryHistory(Base): - __tablename__ = "query_history" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - question: Mapped[str] = mapped_column(Text, nullable=False) - sql: Mapped[str] = mapped_column(Text, nullable=False, default="") - result: Mapped[Optional[dict]] = mapped_column(JSONB, nullable=True) - confidence: Mapped[str] = mapped_column(String, nullable=False, default="low") - restatement: Mapped[str] = mapped_column(Text, nullable=False, default="") - conversation_id: Mapped[Optional[uuid.UUID]] = mapped_column( - PgUUID(as_uuid=True), - ForeignKey("conversations.id", ondelete="SET NULL"), - nullable=True, - ) - timestamp: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=_utcnow - ) - - -class RecentConnection(Base): - __tablename__ = "recent_connections" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - db_url: Mapped[str] = mapped_column(Text, nullable=False) - display_url: Mapped[str] = mapped_column(String, nullable=False) - dialect: Mapped[str] = mapped_column(String, nullable=False) - db_id: Mapped[str] = mapped_column(String, nullable=False) - table_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - connected_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=_utcnow - ) - __table_args__ = (UniqueConstraint("user_id", "db_id"),) - - -class VerifiedQA(Base): - __tablename__ = "verified_qas" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - db_id: Mapped[str] = mapped_column(String, nullable=False) - question: Mapped[str] = mapped_column(Text, nullable=False) - sql: Mapped[str] = mapped_column(Text, nullable=False, default="") - restatement: Mapped[str] = mapped_column(Text, nullable=False, default="") - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=_utcnow - ) - __table_args__ = ( - UniqueConstraint( - "user_id", "db_id", "question", name="uq_verified_qa_user_db_question" - ), - Index("ix_verified_qa_user_db", "user_id", "db_id"), - ) - - -class Glossary(Base): - __tablename__ = "glossary_terms" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - db_id: Mapped[str] = mapped_column(String, nullable=False) - term: Mapped[str] = mapped_column(String, nullable=False) - maps_to: Mapped[str] = mapped_column(String, nullable=False) - sql_hint: Mapped[str] = mapped_column(String, nullable=False, default="") - __table_args__ = (Index("ix_glossary_user_db", "user_id", "db_id"),) - - -class CatalogColumn(Base): - __tablename__ = "catalog_columns" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - db_id: Mapped[str] = mapped_column(String, nullable=False) - table_name: Mapped[str] = mapped_column(String, nullable=False) - column_name: Mapped[str] = mapped_column(String, nullable=False) - description: Mapped[str] = mapped_column(Text, nullable=False) - __table_args__ = (Index("ix_cat_col_user_db", "user_id", "db_id"),) - - -class CatalogMetric(Base): - __tablename__ = "catalog_metrics" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - db_id: Mapped[str] = mapped_column(String, nullable=False) - name: Mapped[str] = mapped_column(String, nullable=False) - description: Mapped[str] = mapped_column(Text, nullable=False, default="") - sql_expression: Mapped[str] = mapped_column(Text, nullable=False) - __table_args__ = (Index("ix_cat_met_user_db", "user_id", "db_id"),) - - -class CatalogJoin(Base): - __tablename__ = "catalog_joins" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - db_id: Mapped[str] = mapped_column(String, nullable=False) - tables: Mapped[str] = mapped_column(String, nullable=False) - join_condition: Mapped[str] = mapped_column(Text, nullable=False) - description: Mapped[str] = mapped_column(Text, nullable=False, default="") - __table_args__ = (Index("ix_cat_join_user_db", "user_id", "db_id"),) - - -class CatalogSynonym(Base): - __tablename__ = "catalog_synonyms" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - db_id: Mapped[str] = mapped_column(String, nullable=False) - term: Mapped[str] = mapped_column(String, nullable=False) - entity_type: Mapped[str] = mapped_column(String, nullable=False) - entity_name: Mapped[str] = mapped_column(String, nullable=False) - __table_args__ = (Index("ix_cat_syn_user_db", "user_id", "db_id"),) - - -class CatalogValueMapping(Base): - __tablename__ = "catalog_value_mappings" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - db_id: Mapped[str] = mapped_column(String, nullable=False) - table_name: Mapped[str] = mapped_column(String, nullable=False) - column_name: Mapped[str] = mapped_column(String, nullable=False) - db_value: Mapped[str] = mapped_column(String, nullable=False) - business_label: Mapped[str] = mapped_column(String, nullable=False) - __table_args__ = (Index("ix_cat_val_user_db", "user_id", "db_id"),) - - -class PasswordResetToken(Base): - __tablename__ = "password_reset_tokens" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - jti_hash: Mapped[str] = mapped_column(String, nullable=False, unique=True) - expires_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - consumed: Mapped[bool] = mapped_column(nullable=False, default=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=_utcnow - ) - - -class OtpCode(Base): - __tablename__ = "otp_codes" - id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), primary_key=True, default=_uuid7 - ) - user_id: Mapped[uuid.UUID] = mapped_column( - PgUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - code_hash: Mapped[str] = mapped_column(String, nullable=False) - purpose: Mapped[str] = mapped_column(String, nullable=False) - expires_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - used: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=_utcnow - ) - __table_args__ = ( - UniqueConstraint("user_id", "purpose", name="uq_otp_user_purpose"), - ) diff --git a/backend/app/pgdatabase/otp.py b/backend/app/pgdatabase/otp.py index 314e8c57..5aac0016 100644 --- a/backend/app/pgdatabase/otp.py +++ b/backend/app/pgdatabase/otp.py @@ -8,7 +8,7 @@ from sqlalchemy import delete as sqlalchemy_delete, select, update as sqlalchemy_update from backend.app.pgdatabase.engine import get_engine -from backend.app.pgdatabase.models import OtpCode +from backend.app.models.auth_token import OtpCode logger = logging.getLogger(__name__) diff --git a/backend/app/pgdatabase/saved_queries.py b/backend/app/pgdatabase/saved_queries.py new file mode 100644 index 00000000..90404c1d --- /dev/null +++ b/backend/app/pgdatabase/saved_queries.py @@ -0,0 +1,84 @@ +from sqlalchemy import select, delete, update +from backend.app.pgdatabase.engine import async_session +from backend.app.models.saved_query import SavedQuery +from backend.app.models.base import _utcnow +from backend.app.pgdatabase.serialization import _to_uuid, serialize_doc + + +async def create_saved_query(workspace_id: str, created_by: str, **kwargs): + uid = _to_uuid(workspace_id) + c_uid = _to_uuid(created_by) if created_by else None + + async with async_session() as session: + try: + sq = SavedQuery(workspace_id=uid, created_by=c_uid, **kwargs) + session.add(sq) + await session.commit() + await session.refresh(sq) + return serialize_doc(sq.__dict__) + except Exception: + await session.rollback() + raise + + +async def list_saved_queries(workspace_id: str, limit: int = 50, offset: int = 0): + uid = _to_uuid(workspace_id) + async with async_session() as session: + result = await session.execute( + select(SavedQuery) + .where(SavedQuery.workspace_id == uid) + .order_by(SavedQuery.updated_at.desc()) + .limit(limit) + .offset(offset) + ) + return [serialize_doc(r.__dict__) for r in result.scalars().all()] + + +async def get_saved_query(workspace_id: str, query_id: str): + uid = _to_uuid(workspace_id) + qid = _to_uuid(query_id) + async with async_session() as session: + result = await session.execute( + select(SavedQuery).where( + SavedQuery.id == qid, SavedQuery.workspace_id == uid + ) + ) + sq = result.scalar_one_or_none() + if sq: + return serialize_doc(sq.__dict__) + return None + + +async def update_saved_query(workspace_id: str, query_id: str, **kwargs): + uid = _to_uuid(workspace_id) + qid = _to_uuid(query_id) + async with async_session() as session: + try: + kwargs["updated_at"] = _utcnow() + result = await session.execute( + update(SavedQuery) + .where(SavedQuery.id == qid, SavedQuery.workspace_id == uid) + .values(**kwargs) + ) + await session.commit() + return result.rowcount > 0 + except Exception: + await session.rollback() + raise + + +async def delete_saved_query(workspace_id: str, query_id: str): + uid = _to_uuid(workspace_id) + qid = _to_uuid(query_id) + async with async_session() as session: + try: + result = await session.execute( + delete(SavedQuery).where( + SavedQuery.id == qid, SavedQuery.workspace_id == uid + ) + ) + await session.commit() + return result.rowcount > 0 + except Exception: + await session.rollback() + raise diff --git a/backend/app/pgdatabase/serialization.py b/backend/app/pgdatabase/serialization.py index 2bb5e399..39c3e593 100644 --- a/backend/app/pgdatabase/serialization.py +++ b/backend/app/pgdatabase/serialization.py @@ -3,6 +3,19 @@ import uuid from datetime import datetime +_UUID_KEYS = ( + "id", + "user_id", + "workspace_id", + "conversation_id", + "dashboard_id", + "saved_query_id", + "database_id", + "created_by", + "invited_by", + "actor_id", +) + def _to_uuid(val: str | uuid.UUID) -> uuid.UUID: if isinstance(val, uuid.UUID): @@ -13,13 +26,16 @@ def _to_uuid(val: str | uuid.UUID) -> uuid.UUID: def serialize_doc(doc): if doc is None: return None - out = doc.copy() - out["_id"] = str(out.pop("id", "")) - if "user_id" in out and out["user_id"] is not None: - out["user_id"] = str(out["user_id"]) - if "conversation_id" in out and out["conversation_id"] is not None: - out["conversation_id"] = str(out["conversation_id"]) - for key in ("created_at", "updated_at", "timestamp", "connected_at"): + out = {k: v for k, v in doc.items() if not k.startswith("_")} + raw_id = out.pop("id", None) + if raw_id is not None: + sid = str(raw_id) + out["_id"] = sid + out["id"] = sid # dual keys for API/frontend compatibility + for key in _UUID_KEYS: + if key in out and out[key] is not None and key != "id": + out[key] = str(out[key]) + for key in ("created_at", "updated_at", "timestamp", "connected_at", "joined_at"): if key in out and isinstance(out[key], datetime): out[key] = out[key].isoformat() return out diff --git a/backend/app/pgdatabase/users.py b/backend/app/pgdatabase/users.py index 85c09b2a..fa879a3d 100644 --- a/backend/app/pgdatabase/users.py +++ b/backend/app/pgdatabase/users.py @@ -8,7 +8,7 @@ from backend.app.models.user import UserInDB from backend.app.pgdatabase.engine import async_session -from backend.app.pgdatabase.models import User +from backend.app.models.orm_user import User from backend.app.pgdatabase.serialization import _to_uuid, serialize_doc logger = logging.getLogger(__name__) @@ -40,6 +40,10 @@ def _user_to_dict(user) -> dict: "email_verified": user.email_verified, "tour_completed": user.tour_completed, "created_at": user.created_at, + "first_name": user.first_name, + "last_name": user.last_name, + "avatar_url": user.avatar_url, + "metadata": user.metadata_, } ) @@ -116,6 +120,10 @@ async def get_user_by_id(user_id: str) -> Optional[dict]: "email_verified", "email", "tour_completed", + "first_name", + "last_name", + "avatar_url", + "metadata", } ) @@ -125,6 +133,10 @@ async def update_user(user_id: str, **fields): if unexpected: logger.warning("Blocked update of disallowed user fields: %s", unexpected) return False + + if "metadata" in fields: + fields["metadata_"] = fields.pop("metadata") + try: uid = _to_uuid(user_id) except (ValueError, TypeError): diff --git a/backend/app/repair.py b/backend/app/repair.py index 3808d51e..0d62bb74 100644 --- a/backend/app/repair.py +++ b/backend/app/repair.py @@ -85,6 +85,7 @@ async def run_repair_loop( "ok": bool, # True if a candidate passed all checks "sql": str, # the accepted SQL, or the last attempt on failure "restatement": str, + "chart": dict | None, # the generator's chart choice, if it made one "result": dict | None, # execute() output when ok and an executor was given "attempts": [ {"sql", "stage", "errors"?}, ... ], } @@ -130,6 +131,7 @@ async def run_repair_loop( "sql": sql, "restatement": last.get("restatement", ""), "assumptions": last.get("assumptions", []), + "chart": last.get("chart"), "result": result, "attempts": attempts, } @@ -140,6 +142,7 @@ async def run_repair_loop( "sql": sql, "restatement": last.get("restatement", ""), "assumptions": last.get("assumptions", []), + "chart": last.get("chart"), "result": None, "attempts": attempts, } @@ -149,6 +152,7 @@ async def run_repair_loop( "sql": (last.get("sql") or "").strip(), "restatement": last.get("restatement", ""), "assumptions": last.get("assumptions", []), + "chart": last.get("chart"), "result": None, "attempts": attempts, } diff --git a/backend/app/routes/auth.py b/backend/app/routes/auth.py index a46e707b..9622e373 100644 --- a/backend/app/routes/auth.py +++ b/backend/app/routes/auth.py @@ -81,6 +81,16 @@ async def me(user_token=Depends(get_current_user)): return JSONResponse({"message": "My details", "content": user}) +@router.patch("/me") +async def update_me( + req: backend.app.models.user.UpdateProfile, user_token=Depends(get_current_user) +): + user = await backend.app.controllers.auth.update_profile(user_token["user_id"], req) + if not user: + raise HTTPException(status_code=404, detail="User not found") + return JSONResponse({"message": "Profile updated", "content": user}) + + @router.post("/login") @limiter.limit("10/minute") async def login(request: Request, login_data: UserLogin): diff --git a/backend/app/routes/catalog.py b/backend/app/routes/catalog.py index c15ed6d3..0c55d73e 100644 --- a/backend/app/routes/catalog.py +++ b/backend/app/routes/catalog.py @@ -2,7 +2,13 @@ from fastapi import APIRouter, Depends, Request, HTTPException -from backend.app.dependencies import get_current_user, get_db, get_kb, get_providers +from backend.app.dependencies import ( + require_permission, + get_current_db_id, + get_db, + get_kb, + get_providers, +) from backend.app.models.api import CatalogPayload from backend.app.ratelimit import limiter import backend.app.controllers.catalog as ctrl @@ -13,13 +19,14 @@ @router.get("/api/catalog") async def get_catalog( - user_token=Depends(get_current_user), + workspace=Depends(require_permission("catalog.view")), db=Depends(get_db), kb=Depends(get_kb), + db_id=Depends(get_current_db_id), ): """Return the saved semantic catalog for the connected database.""" try: - return await ctrl.get_catalog(user_token["user_id"], db, kb) + return await ctrl.get_catalog(workspace["workspace_id"], db, kb, db_id) except HTTPException: raise except Exception: @@ -30,13 +37,16 @@ async def get_catalog( @router.post("/api/catalog") async def save_catalog( payload: CatalogPayload, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("catalog.manage")), db=Depends(get_db), kb=Depends(get_kb), + db_id=Depends(get_current_db_id), ): """Replace the connected database's semantic catalog with ``payload``.""" try: - return await ctrl.save_catalog(user_token["user_id"], db, kb, payload) + return await ctrl.save_catalog( + workspace["workspace_id"], db, kb, payload, db_id + ) except HTTPException: raise except Exception: @@ -48,13 +58,14 @@ async def save_catalog( @limiter.limit("10/minute") async def suggest_catalog( request: Request, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("catalog.manage")), db=Depends(get_db), providers=Depends(get_providers), + db_id=Depends(get_current_db_id), ): """Return AI + schema-derived catalog suggestions (not yet saved).""" try: - return await ctrl.suggest(user_token["user_id"], db, providers) + return await ctrl.suggest(workspace["workspace_id"], db, providers, db_id) except HTTPException: raise except Exception: diff --git a/backend/app/routes/connections.py b/backend/app/routes/connections.py index dfe6f22b..d2d48512 100644 --- a/backend/app/routes/connections.py +++ b/backend/app/routes/connections.py @@ -2,17 +2,23 @@ from fastapi import APIRouter, Depends, HTTPException -from backend.app.dependencies import get_current_user +from backend.app.dependencies import require_permission +from pydantic import BaseModel import backend.app.pgdatabase as mdb + +class ConnectionUpdate(BaseModel): + alias_name: str | None + + log = logging.getLogger(__name__) router = APIRouter() @router.get("/api/connections") -async def get_connections(user_token=Depends(get_current_user)): +async def get_connections(workspace=Depends(require_permission("connections.view"))): try: - connections = await mdb.get_recent_connections(user_token["user_id"]) + connections = await mdb.get_recent_connections(workspace["workspace_id"]) for c in connections: c.pop("db_url", None) return {"connections": connections} @@ -22,12 +28,34 @@ async def get_connections(user_token=Depends(get_current_user)): @router.delete("/api/connections/{connection_id}") -async def delete_connection(connection_id: str, user_token=Depends(get_current_user)): +async def delete_connection( + connection_id: str, workspace=Depends(require_permission("connections.manage")) +): try: deleted = await mdb.delete_recent_connection( - user_token["user_id"], connection_id + workspace["workspace_id"], connection_id ) return {"ok": deleted} except Exception: log.exception("Failed to delete connection") raise HTTPException(500, "Failed to delete connection") + + +@router.patch("/api/connections/{connection_id}") +async def update_connection( + connection_id: str, + req: ConnectionUpdate, + workspace=Depends(require_permission("connections.manage")), +): + try: + updated = await mdb.update_recent_connection_alias( + workspace["workspace_id"], connection_id, req.alias_name + ) + if not updated: + raise HTTPException(404, "Connection not found") + return {"ok": True} + except HTTPException: + raise + except Exception: + log.exception("Failed to update connection alias") + raise HTTPException(500, "Failed to update connection") diff --git a/backend/app/routes/conversations.py b/backend/app/routes/conversations.py index b41ff5d3..a4134a4f 100644 --- a/backend/app/routes/conversations.py +++ b/backend/app/routes/conversations.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import JSONResponse from pydantic import BaseModel -from backend.app.dependencies import get_current_user +from backend.app.dependencies import require_permission import backend.app.controllers.conversations as ctrl log = logging.getLogger(__name__) @@ -20,10 +20,11 @@ class RenameConversationReq(BaseModel): @router.get("") -async def list_conversations(user_token=Depends(get_current_user)): +async def list_conversations(workspace=Depends(require_permission("queries.execute"))): try: - user_id = user_token["user_id"] - convs = await ctrl.list_conversations(user_id) + workspace_id = workspace["workspace_id"] + user_id = workspace["user_id"] + convs = await ctrl.list_conversations(workspace_id, user_id) return JSONResponse({"conversations": convs}) except Exception: log.exception("Failed to list conversations") @@ -33,11 +34,13 @@ async def list_conversations(user_token=Depends(get_current_user)): @router.post("") async def create_conversation( req: CreateConversationReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.execute")), ): try: - user_id = user_token["user_id"] + workspace_id = workspace["workspace_id"] + user_id = workspace["user_id"] conv = await ctrl.create_conversation( + workspace_id, user_id, title=req.title, database_id=req.database_id, @@ -51,11 +54,12 @@ async def create_conversation( @router.get("/{conversation_id}") async def get_conversation( conversation_id: str, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.execute")), ): try: - user_id = user_token["user_id"] - conv = await ctrl.get_conversation(user_id, conversation_id) + workspace_id = workspace["workspace_id"] + user_id = workspace["user_id"] + conv = await ctrl.get_conversation(workspace_id, user_id, conversation_id) if not conv: raise HTTPException(status_code=404, detail="Conversation not found") return JSONResponse(conv) @@ -70,11 +74,14 @@ async def get_conversation( async def rename_conversation( conversation_id: str, req: RenameConversationReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.execute")), ): try: - user_id = user_token["user_id"] - success = await ctrl.rename_conversation(user_id, conversation_id, req.title) + workspace_id = workspace["workspace_id"] + user_id = workspace["user_id"] + success = await ctrl.rename_conversation( + workspace_id, user_id, conversation_id, req.title + ) if not success: raise HTTPException( status_code=404, detail="Conversation not found or unauthorized" @@ -90,11 +97,12 @@ async def rename_conversation( @router.delete("/{conversation_id}") async def delete_conversation( conversation_id: str, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.execute")), ): try: - user_id = user_token["user_id"] - success = await ctrl.delete_conversation(user_id, conversation_id) + workspace_id = workspace["workspace_id"] + user_id = workspace["user_id"] + success = await ctrl.delete_conversation(workspace_id, user_id, conversation_id) if not success: raise HTTPException( status_code=404, detail="Conversation not found or unauthorized" @@ -108,10 +116,11 @@ async def delete_conversation( @router.delete("") -async def clear_conversations(user_token=Depends(get_current_user)): +async def clear_conversations(workspace=Depends(require_permission("queries.execute"))): try: - user_id = user_token["user_id"] - await ctrl.clear_conversations(user_id) + workspace_id = workspace["workspace_id"] + user_id = workspace["user_id"] + await ctrl.clear_conversations(workspace_id, user_id) return JSONResponse({"message": "Cleared successfully"}) except Exception: log.exception("Failed to clear conversations") diff --git a/backend/app/routes/dashboards.py b/backend/app/routes/dashboards.py new file mode 100644 index 00000000..19580df6 --- /dev/null +++ b/backend/app/routes/dashboards.py @@ -0,0 +1,232 @@ +import logging +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import JSONResponse +from pydantic import BaseModel +from typing import Optional, List, Dict, Any +from backend.app.dependencies import require_permission, get_current_user +import backend.app.controllers.dashboards as ctrl + +log = logging.getLogger(__name__) +router = APIRouter(prefix="/api/dashboards", tags=["dashboards"]) + + +class CreateDashboardReq(BaseModel): + name: str + description: Optional[str] = None + + +class UpdateDashboardReq(BaseModel): + name: Optional[str] = None + description: Optional[str] = None + + +class AddPanelReq(BaseModel): + saved_query_id: Optional[str] = None + title: Optional[str] = None + visualization_type: Optional[str] = None + viz_config: Optional[dict] = None + position: Optional[dict] = None + + +class UpdatePanelReq(BaseModel): + title: Optional[str] = None + visualization_type: Optional[str] = None + viz_config: Optional[dict] = None + position: Optional[dict] = None + + +class BatchUpdatePanelPositionReq(BaseModel): + updates: List[Dict[str, Any]] # {"id": panel_id, "position": {...}} + + +@router.get("") +async def list_dashboards(workspace=Depends(require_permission("dashboards.view"))): + try: + dashboards = await ctrl.list_dashboards(workspace["workspace_id"]) + return JSONResponse({"dashboards": dashboards}) + except Exception: + log.exception("Failed to list dashboards") + raise HTTPException(500, "Failed to load dashboards") + + +@router.post("") +async def create_dashboard( + req: CreateDashboardReq, + workspace=Depends(require_permission("dashboards.create")), + user=Depends(get_current_user), +): + try: + dash = await ctrl.create_dashboard( + workspace["workspace_id"], user["user_id"], req.name, req.description + ) + return JSONResponse(dash) + except HTTPException: + raise + except Exception: + log.exception("Failed to create dashboard") + raise HTTPException(500, "Failed to create dashboard") + + +@router.get("/{dashboard_id}") +async def get_dashboard( + dashboard_id: str, workspace=Depends(require_permission("dashboards.view")) +): + try: + dash = await ctrl.get_dashboard(workspace["workspace_id"], dashboard_id) + if not dash: + raise HTTPException(404, "Dashboard not found") + return JSONResponse(dash) + except HTTPException: + raise + except Exception: + log.exception("Failed to get dashboard") + raise HTTPException(500, "Failed to load dashboard") + + +@router.patch("/{dashboard_id}") +async def update_dashboard( + dashboard_id: str, + req: UpdateDashboardReq, + workspace=Depends(require_permission("dashboards.manage")), +): + try: + success = await ctrl.update_dashboard( + workspace["workspace_id"], dashboard_id, req.model_dump(exclude_unset=True) + ) + if not success: + raise HTTPException(404, "Dashboard not found") + return JSONResponse({"message": "Updated successfully"}) + except HTTPException: + raise + except Exception: + log.exception("Failed to update dashboard") + raise HTTPException(500, "Failed to update dashboard") + + +@router.delete("/{dashboard_id}") +async def delete_dashboard( + dashboard_id: str, workspace=Depends(require_permission("dashboards.manage")) +): + try: + success = await ctrl.delete_dashboard(workspace["workspace_id"], dashboard_id) + if not success: + raise HTTPException(404, "Dashboard not found") + return JSONResponse({"message": "Deleted successfully"}) + except HTTPException: + raise + except Exception: + log.exception("Failed to delete dashboard") + raise HTTPException(500, "Failed to delete dashboard") + + +@router.get("/{dashboard_id}/data") +async def get_dashboard_data( + dashboard_id: str, + request: Request, + workspace=Depends(require_permission("dashboards.view")), + _execute_permission=Depends(require_permission("queries.execute")), +): + try: + db_manager = request.app.state.db + results = await ctrl.execute_dashboard_queries( + workspace["workspace_id"], dashboard_id, db_manager + ) + if results is None: + raise HTTPException(404, "Dashboard not found") + return JSONResponse(results) + except HTTPException: + raise + except Exception: + log.exception("Failed to execute dashboard queries") + raise HTTPException(500, "Failed to execute dashboard queries") + + +# --- Panels --- + + +@router.post("/{dashboard_id}/panels") +async def add_panel( + dashboard_id: str, + req: AddPanelReq, + workspace=Depends(require_permission("dashboards.manage")), +): + try: + panel = await ctrl.add_panel( + workspace["workspace_id"], dashboard_id, req.model_dump(exclude_unset=True) + ) + return JSONResponse(panel) + except HTTPException: + raise + except ValueError as e: + raise HTTPException(404, str(e)) + except Exception: + log.exception("Failed to add panel") + raise HTTPException(500, "Failed to add panel") + + +@router.patch("/{dashboard_id}/panels/batch") +async def batch_update_panels( + dashboard_id: str, + req: BatchUpdatePanelPositionReq, + workspace=Depends(require_permission("dashboards.manage")), +): + try: + success = await ctrl.update_panels_batch( + workspace["workspace_id"], + dashboard_id, + req.model_dump(exclude_unset=True)["updates"], + ) + if not success: + raise HTTPException(404, "Dashboard not found") + return JSONResponse({"message": "Updated successfully"}) + except HTTPException: + raise + except Exception: + log.exception("Failed to batch update panels") + raise HTTPException(500, "Failed to update panels") + + +@router.patch("/{dashboard_id}/panels/{panel_id}") +async def update_panel( + dashboard_id: str, + panel_id: str, + req: UpdatePanelReq, + workspace=Depends(require_permission("dashboards.manage")), +): + try: + success = await ctrl.update_panel( + workspace["workspace_id"], + dashboard_id, + panel_id, + req.model_dump(exclude_unset=True), + ) + if not success: + raise HTTPException(404, "Panel not found") + return JSONResponse({"message": "Updated successfully"}) + except HTTPException: + raise + except ValueError as e: + raise HTTPException(404, str(e)) + except Exception: + log.exception("Failed to update panel") + raise HTTPException(500, "Failed to update panel") + + +@router.delete("/{dashboard_id}/panels/{panel_id}") +async def delete_panel( + dashboard_id: str, + panel_id: str, + workspace=Depends(require_permission("dashboards.manage")), +): + try: + success = await ctrl.delete_panel( + workspace["workspace_id"], dashboard_id, panel_id + ) + if not success: + raise HTTPException(404, "Panel not found") + return JSONResponse({"message": "Deleted successfully"}) + except HTTPException: + raise + except Exception: + log.exception("Failed to delete panel") + raise HTTPException(500, "Failed to delete panel") diff --git a/backend/app/routes/database.py b/backend/app/routes/database.py index 7554c623..ab0466a8 100644 --- a/backend/app/routes/database.py +++ b/backend/app/routes/database.py @@ -1,52 +1,100 @@ +import logging + from fastapi import APIRouter, Depends, HTTPException -from backend.app.dependencies import get_current_user, get_db, get_kb, get_cfg +from backend.app.dependencies import ( + require_permission, + get_db, + get_kb, + get_cfg, + get_current_db_id, +) from backend.app.models.api import ConnectReq import backend.app.controllers.database as ctrl import backend.app.pgdatabase as mdb +log = logging.getLogger(__name__) router = APIRouter() +@router.get("/api/databases") +async def list_databases(workspace=Depends(require_permission("connections.view"))): + return await mdb.get_recent_connections(workspace["workspace_id"], limit=50) + + @router.post("/api/connect") async def connect( req: ConnectReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("connections.manage")), + _catalog_permission=Depends(require_permission("catalog.view")), db=Depends(get_db), kb=Depends(get_kb), cfg=Depends(get_cfg), ): - return await ctrl.connect(db, kb, cfg, req, user_id=user_token["user_id"]) + return await ctrl.connect( + db, + kb, + cfg, + req, + workspace_id=workspace["workspace_id"], + user_id=workspace.get("user_id"), + ) @router.post("/api/connect-sample") async def connect_sample( - user_token=Depends(get_current_user), + workspace=Depends(require_permission("connections.manage")), + _catalog_permission=Depends(require_permission("catalog.view")), db=Depends(get_db), kb=Depends(get_kb), cfg=Depends(get_cfg), ): - return await ctrl.connect_sample(db, kb, cfg, user_id=user_token["user_id"]) + return await ctrl.connect_sample( + db, + kb, + cfg, + workspace_id=workspace["workspace_id"], + user_id=workspace.get("user_id"), + ) @router.post("/api/disconnect") async def disconnect( - user_token=Depends(get_current_user), db=Depends(get_db), cfg=Depends(get_cfg) + workspace=Depends(require_permission("connections.manage")), + db_id: str = Depends(get_current_db_id), + db=Depends(get_db), + cfg=Depends(get_cfg), ): - user_id = user_token["user_id"] - return await ctrl.disconnect(user_id, db, cfg) + return await ctrl.disconnect( + workspace["workspace_id"], + db, + cfg, + db_id=db_id, + user_id=workspace.get("user_id"), + ) @router.get("/api/schema") async def schema( - refresh: bool = False, user_token=Depends(get_current_user), db=Depends(get_db) + refresh: bool = False, + workspace=Depends(require_permission("connections.view_schema")), + db_id: str = Depends(get_current_db_id), + db=Depends(get_db), ): - return await ctrl.get_schema(user_token["user_id"], db, refresh) + return await ctrl.get_schema(workspace["workspace_id"], db, refresh, db_id=db_id) @router.post("/api/reconnect") async def reconnect( request_body: dict, - user_token=Depends(get_current_user), + # Switching to a database the workspace already has is a *use* action, not a + # *manage* one: reconnect only ever targets a stored connection (looked up by + # db_id below and 404'd otherwise, using the saved URL — never a caller + # supplied one), so it can't add new databases. Gating it on connections.view + # lets members pick between the workspace's databases from the switcher; + # adding a brand-new connection still requires connections.manage via + # /api/connect. + workspace=Depends(require_permission("connections.view")), + _catalog_permission=Depends(require_permission("catalog.view")), db=Depends(get_db), kb=Depends(get_kb), cfg=Depends(get_cfg), @@ -54,8 +102,26 @@ async def reconnect( db_id = request_body.get("db_id", "") if not db_id: raise HTTPException(400, "db_id is required") - conn = await mdb.get_recent_connection_by_db_id(user_token["user_id"], db_id) + try: + conn = await mdb.get_recent_connection_by_db_id( + workspace["workspace_id"], db_id + ) + except RuntimeError: + log.exception("Could not decrypt stored connection for db_id=%s", db_id) + raise HTTPException( + 409, + "Saved credentials could not be decrypted — please re-add this connection.", + ) if not conn or not conn.get("db_url"): raise HTTPException(404, "Saved connection not found") - req = ConnectReq(db_url=conn["db_url"]) - return await ctrl.connect(db, kb, cfg, req, user_id=user_token["user_id"]) + # Carry the stored alias through, so reconnecting doesn't leave the header + # and the switcher labelling a named database by its host again. + req = ConnectReq(db_url=conn["db_url"], alias_name=conn.get("alias_name")) + return await ctrl.connect( + db, + kb, + cfg, + req, + workspace_id=workspace["workspace_id"], + user_id=workspace.get("user_id"), + ) diff --git a/backend/app/routes/history.py b/backend/app/routes/history.py index 4e05ced1..ba216446 100644 --- a/backend/app/routes/history.py +++ b/backend/app/routes/history.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import JSONResponse -from backend.app.dependencies import get_current_user +from backend.app.dependencies import require_permission import backend.app.pgdatabase as db log = logging.getLogger(__name__) @@ -12,11 +12,11 @@ @router.get("") async def get_history( limit: int = Query(default=100, ge=1, le=500), - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.execute")), ): try: - user_id = user_token["user_id"] - history = await db.get_query_history(user_id, limit) + workspace_id = workspace["workspace_id"] + history = await db.get_query_history(workspace_id, limit) return JSONResponse({"history": history}) except Exception: log.exception("Failed to get query history") @@ -24,10 +24,10 @@ async def get_history( @router.get("/stats") -async def get_stats(user_token=Depends(get_current_user)): +async def get_stats(workspace=Depends(require_permission("queries.execute"))): try: - user_id = user_token["user_id"] - stats = await db.get_query_stats(user_id) + workspace_id = workspace["workspace_id"] + stats = await db.get_query_stats(workspace_id) return JSONResponse(stats) except Exception: log.exception("Failed to get query stats") @@ -35,10 +35,13 @@ async def get_stats(user_token=Depends(get_current_user)): @router.delete("/{entry_id}") -async def delete_entry(entry_id: str, user_token=Depends(get_current_user)): +async def delete_entry( + entry_id: str, + workspace=Depends(require_permission("queries.execute")), +): try: - user_id = user_token["user_id"] - success = await db.delete_history_entry(user_id, entry_id) + workspace_id = workspace["workspace_id"] + success = await db.delete_history_entry(workspace_id, entry_id) if not success: raise HTTPException( status_code=404, detail="Entry not found or unauthorized" @@ -52,10 +55,10 @@ async def delete_entry(entry_id: str, user_token=Depends(get_current_user)): @router.delete("") -async def clear_history(user_token=Depends(get_current_user)): +async def clear_history(workspace=Depends(require_permission("queries.execute"))): try: - user_id = user_token["user_id"] - await db.clear_history(user_id) + workspace_id = workspace["workspace_id"] + await db.clear_history(workspace_id) return JSONResponse({"message": "Cleared successfully"}) except Exception: log.exception("Failed to clear history") diff --git a/backend/app/routes/onboard.py b/backend/app/routes/onboard.py index a345c281..4f812b53 100644 --- a/backend/app/routes/onboard.py +++ b/backend/app/routes/onboard.py @@ -1,7 +1,12 @@ import logging from fastapi import APIRouter, Depends, Request, HTTPException -from backend.app.dependencies import get_current_user, get_db, get_kb, get_providers +from backend.app.dependencies import ( + require_permission, + get_db, + get_kb, + get_providers, +) from backend.app.models.api import SaveOnboardReq from backend.app.ratelimit import limiter import backend.app.controllers.onboard as ctrl @@ -15,13 +20,13 @@ async def onboard_save( request: Request, req: SaveOnboardReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("catalog.manage")), db=Depends(get_db), kb=Depends(get_kb), ): try: - user_id = user_token["user_id"] - return await ctrl.save(user_id, db, kb, req) + workspace_id = workspace["workspace_id"] + return await ctrl.save(workspace_id, db, kb, req) except HTTPException: raise except Exception: @@ -33,14 +38,15 @@ async def onboard_save( @limiter.limit("5/minute") async def api_generate_starters( request: Request, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("catalog.manage")), + _execute_permission=Depends(require_permission("queries.execute")), db=Depends(get_db), kb=Depends(get_kb), providers=Depends(get_providers), ): try: - user_id = user_token["user_id"] - return await ctrl.generate_starters_async(user_id, db, kb, providers) + workspace_id = workspace["workspace_id"] + return await ctrl.generate_starters_async(workspace_id, db, kb, providers) except HTTPException: raise except Exception: diff --git a/backend/app/routes/query.py b/backend/app/routes/query.py index f163503d..2f657bcb 100644 --- a/backend/app/routes/query.py +++ b/backend/app/routes/query.py @@ -3,18 +3,21 @@ from fastapi import APIRouter, Depends, Request, HTTPException from fastapi.responses import StreamingResponse from backend.app.dependencies import ( - get_current_user, + require_permission, + workspace_has_permission, get_db, get_kb, get_cfg, get_providers, get_session_log, + get_current_db_id, ) from backend.app.models.api import QueryReq, FeedbackReq, VerifyReq, RawSQLReq from backend.app.ratelimit import limiter import backend.app.controllers.query as ctrl import backend.app.pgdatabase as mdb from backend.app.pgdatabase.conversations import MAX_SAVED_ROWS +from backend.app.controllers.activity import log_activity log = logging.getLogger(__name__) router = APIRouter() @@ -34,16 +37,28 @@ async def _format_sse(stream): async def query( request: Request, req: QueryReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.execute")), db=Depends(get_db), kb=Depends(get_kb), cfg=Depends(get_cfg), providers=Depends(get_providers), session_log=Depends(get_session_log), + db_id=Depends(get_current_db_id), ) -> dict: try: - user_id = user_token["user_id"] - out = await ctrl.run_query(user_id, db, kb, cfg, providers, session_log, req) + workspace_id = workspace["workspace_id"] + user_id = workspace.get("user_id") + out = await ctrl.run_query( + workspace_id, + db, + kb, + cfg, + providers, + session_log, + req, + db_id=db_id, + user_id=user_id, + ) if out.get("answered") and out.get("sql"): conf = out.get("confidence", "low") @@ -52,21 +67,31 @@ async def query( ) conversation_id = req.conversation_id if conversation_id and not await mdb.conversation_owned_by( - user_id, conversation_id + workspace_id, user_id, conversation_id ): conversation_id = None try: await mdb.save_query( - user_id=user_token["user_id"], + workspace_id=workspace_id, + user_id=user_id, question=req.question, sql=out["sql"], result=out.get("rows", []), confidence=conf_str, conversation_id=conversation_id, restatement=out.get("restatement", ""), + chart=out.get("chart"), ) if conversation_id: - await mdb.touch_conversation(conversation_id) + await mdb.touch_conversation(conversation_id, user_id) + await log_activity( + workspace_id, + user_id, + "query.executed", + "query", + None, + {"question": req.question, "db_id": db_id, "confidence": conf_str}, + ) except Exception: log.warning("Failed to persist query history", exc_info=True) return out @@ -82,17 +107,27 @@ async def query( async def query_stream( request: Request, req: QueryReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.execute")), db=Depends(get_db), kb=Depends(get_kb), cfg=Depends(get_cfg), providers=Depends(get_providers), session_log=Depends(get_session_log), + db_id=Depends(get_current_db_id), ) -> StreamingResponse: try: - user_id = user_token["user_id"] + workspace_id = workspace["workspace_id"] + user_id = workspace.get("user_id") stream = ctrl.run_query_stream( - user_id, db, kb, cfg, providers, session_log, req + workspace_id, + db, + kb, + cfg, + providers, + session_log, + req, + db_id=db_id, + user_id=user_id, ) return StreamingResponse( @@ -113,14 +148,23 @@ async def query_stream( @router.post("/api/feedback") async def feedback( req: FeedbackReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.execute")), db=Depends(get_db), kb=Depends(get_kb), session_log=Depends(get_session_log), + db_id=Depends(get_current_db_id), ) -> dict: try: - user_id = user_token["user_id"] - return await ctrl.feedback(user_id, db, kb, session_log, req) + workspace_id = workspace["workspace_id"] + # Positive feedback promotes a Q&A/SQL pair into shared verified knowledge, + # so it must satisfy the same catalog-manage gate as /api/verify. + if req.verdict == "correct" and not await workspace_has_permission( + workspace, "catalog.manage" + ): + raise HTTPException( + status_code=403, detail="Permission 'catalog.manage' required" + ) + return await ctrl.feedback(workspace_id, db, kb, session_log, req, db_id=db_id) except HTTPException: raise except Exception: @@ -131,13 +175,23 @@ async def feedback( @router.post("/api/verify") async def verify( req: VerifyReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("catalog.manage")), db=Depends(get_db), kb=Depends(get_kb), + db_id=Depends(get_current_db_id), ) -> dict: try: - user_id = user_token["user_id"] - return await ctrl.verify(user_id, db, kb, req) + workspace_id = workspace["workspace_id"] + result = await ctrl.verify(workspace_id, db, kb, req, db_id=db_id) + await log_activity( + workspace_id, + workspace.get("user_id"), + "knowledge.verified", + "knowledge", + None, + {"question": req.question, "db_id": db_id}, + ) + return result except HTTPException: raise except Exception: @@ -150,30 +204,47 @@ async def verify( async def execute( request: Request, req: RawSQLReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.execute")), db=Depends(get_db), + db_id=Depends(get_current_db_id), ) -> dict: try: - user_id = user_token["user_id"] - out = await ctrl.execute(user_id, db, req) + workspace_id = workspace["workspace_id"] + user_id = workspace.get("user_id") + out = await ctrl.execute(workspace_id, db, req, db_id=db_id) conversation_id = req.conversation_id if conversation_id and not await mdb.conversation_owned_by( - user_id, conversation_id + workspace_id, user_id, conversation_id ): conversation_id = None try: - await mdb.save_query( - user_id=user_id, - question=req.sql, - sql=req.sql, - result=(out.get("rows") or [])[:MAX_SAVED_ROWS], - confidence="High", - conversation_id=conversation_id, - restatement="Direct SQL execution", + if req.save_history: + await mdb.save_query( + workspace_id=workspace_id, + user_id=user_id, + question=req.sql, + sql=req.sql, + result=(out.get("rows") or [])[:MAX_SAVED_ROWS], + confidence="High", + conversation_id=conversation_id, + restatement="Direct SQL execution", + ) + if conversation_id: + await mdb.touch_conversation(conversation_id, user_id) + await log_activity( + workspace_id, + user_id, + "query.executed", + "query", + None, + { + "is_raw_sql": True, + "db_id": db_id, + "confidence": "High", + "is_rerun": not req.save_history, + }, ) - if conversation_id: - await mdb.touch_conversation(conversation_id) except Exception: log.warning("Failed to persist direct SQL history", exc_info=True) return out @@ -189,13 +260,14 @@ async def execute( async def explain( request: Request, req: RawSQLReq, - user_token=Depends(get_current_user), + workspace=Depends(require_permission("queries.explain")), db=Depends(get_db), providers=Depends(get_providers), + db_id=Depends(get_current_db_id), ) -> dict: try: - user_id = user_token["user_id"] - return await ctrl.explain(user_id, db, providers, req) + workspace_id = workspace["workspace_id"] + return await ctrl.explain(workspace_id, db, providers, req, db_id=db_id) except HTTPException: raise except Exception: diff --git a/backend/app/routes/saved_queries.py b/backend/app/routes/saved_queries.py new file mode 100644 index 00000000..9514aa35 --- /dev/null +++ b/backend/app/routes/saved_queries.py @@ -0,0 +1,111 @@ +import logging +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import JSONResponse +from pydantic import BaseModel +from typing import Optional +from backend.app.dependencies import require_permission, get_current_user +import backend.app.controllers.dashboards as ctrl + +log = logging.getLogger(__name__) +router = APIRouter(prefix="/api/saved-queries", tags=["saved_queries"]) + + +class SaveQueryReq(BaseModel): + name: str + description: Optional[str] = None + question: Optional[str] = None + sql: Optional[str] = None + columns: Optional[list] = None + database_id: Optional[str] = None + visualization_type: str = "table" + viz_config: Optional[dict] = None + last_result: Optional[list] = None + + +class UpdateSavedQueryReq(BaseModel): + name: Optional[str] = None + description: Optional[str] = None + visualization_type: Optional[str] = None + viz_config: Optional[dict] = None + + +@router.get("") +async def list_saved_queries( + workspace=Depends(require_permission("queries.execute")), +): + try: + sqs = await ctrl.list_saved_queries(workspace["workspace_id"]) + return JSONResponse({"saved_queries": sqs}) + except Exception: + log.exception("Failed to list saved queries") + raise HTTPException(500, "Failed to load saved queries") + + +@router.post("") +async def create_saved_query( + req: SaveQueryReq, + workspace=Depends(require_permission("queries.save")), + user=Depends(get_current_user), +): + try: + sq = await ctrl.create_saved_query( + workspace["workspace_id"], + user["user_id"], + req.model_dump(exclude_unset=True), + ) + return JSONResponse(sq) + except Exception: + log.exception("Failed to create saved query") + raise HTTPException(500, "Failed to save query") + + +@router.get("/{query_id}") +async def get_saved_query( + query_id: str, workspace=Depends(require_permission("queries.execute")) +): + try: + sq = await ctrl.get_saved_query(workspace["workspace_id"], query_id) + if not sq: + raise HTTPException(404, "Saved query not found") + return JSONResponse(sq) + except HTTPException: + raise + except Exception: + log.exception("Failed to get saved query") + raise HTTPException(500, "Failed to load saved query") + + +@router.patch("/{query_id}") +async def update_saved_query( + query_id: str, + req: UpdateSavedQueryReq, + workspace=Depends(require_permission("queries.save")), +): + try: + success = await ctrl.update_saved_query( + workspace["workspace_id"], query_id, req.model_dump(exclude_unset=True) + ) + if not success: + raise HTTPException(404, "Saved query not found") + return JSONResponse({"message": "Updated successfully"}) + except HTTPException: + raise + except Exception: + log.exception("Failed to update saved query") + raise HTTPException(500, "Failed to update saved query") + + +@router.delete("/{query_id}") +async def delete_saved_query( + query_id: str, workspace=Depends(require_permission("queries.delete_saved")) +): + try: + success = await ctrl.delete_saved_query(workspace["workspace_id"], query_id) + if not success: + raise HTTPException(404, "Saved query not found") + return JSONResponse({"message": "Deleted successfully"}) + except HTTPException: + raise + except Exception: + log.exception("Failed to delete saved query") + raise HTTPException(500, "Failed to delete saved query") diff --git a/backend/app/routes/system.py b/backend/app/routes/system.py index 9c06a520..9d9db12f 100644 --- a/backend/app/routes/system.py +++ b/backend/app/routes/system.py @@ -1,12 +1,14 @@ -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Header from fastapi.responses import JSONResponse from sqlalchemy import text from backend.app.dependencies import ( get_current_user, + get_current_workspace, get_db, get_cfg, get_kb, get_providers, + get_current_db_id, ) from backend.app.models.api import ConfigUpdate from backend.app.secrets import get_supabase_url, get_supabase_anon_key @@ -18,6 +20,8 @@ @router.get("/api/state") async def state( user_token=Depends(get_current_user), + x_workspace_id: str = Header(None), + x_db_id: str = Depends(get_current_db_id), db=Depends(get_db), cfg=Depends(get_cfg), kb=Depends(get_kb), @@ -29,7 +33,11 @@ async def state( The current application state. """ user_id = user_token["user_id"] - return await ctrl.get_state(user_id, db, cfg, kb) + verified_workspace_id = None + if x_workspace_id: + workspace = await get_current_workspace(x_workspace_id, user_token) + verified_workspace_id = workspace["workspace_id"] + return await ctrl.get_state(user_id, verified_workspace_id, x_db_id, db, cfg, kb) @router.post("/api/tour-complete") diff --git a/backend/app/routes/workspaces.py b/backend/app/routes/workspaces.py new file mode 100644 index 00000000..fd1a0019 --- /dev/null +++ b/backend/app/routes/workspaces.py @@ -0,0 +1,284 @@ +import csv +import io +import json + +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import StreamingResponse +from backend.app.dependencies import ( + get_current_user, + get_current_workspace, + require_permission, +) +from backend.app.pgdatabase import get_user_by_id +from backend.app.models.workspace_api import ( + WorkspaceBulkInviteCreate, + WorkspaceCreate, + WorkspaceMemberRoleUpdate, + WorkspaceInviteCreate, + WorkspaceOwnershipTransfer, + WorkspaceUpdate, + WorkspaceSettingsUpdate, +) +import backend.app.controllers.workspaces as ctrl +import backend.app.controllers.activity as activity_ctrl + +router = APIRouter() + + +@router.get("/api/workspaces") +async def list_workspaces(user=Depends(get_current_user)): + user_id = user.get("user_id", user.get("sub")) + return await ctrl.list_workspaces(user_id) + + +@router.post("/api/workspaces") +async def create_workspace(req: WorkspaceCreate, user=Depends(get_current_user)): + user_id = user.get("user_id", user.get("sub")) + return await ctrl.create_workspace(user_id, req.name) + + +@router.get("/api/workspaces/{workspace_id}") +async def get_workspace(workspace_id: str, user=Depends(get_current_user)): + user_id = user.get("user_id", user.get("sub")) + return await ctrl.get_workspace(workspace_id, user_id) + + +@router.get("/api/workspaces/{workspace_id}/settings") +async def get_workspace_settings( + workspace_id: str, + workspace=Depends(require_permission("workspace.settings")), +): + return await ctrl.get_workspace_settings(workspace["workspace_id"]) + + +@router.patch("/api/workspaces/{workspace_id}/settings") +async def update_workspace_settings( + workspace_id: str, + req: WorkspaceSettingsUpdate, + workspace=Depends(get_current_workspace), +): + if workspace["role"] != "owner": + raise HTTPException(403, "Owner role required") + return await ctrl.update_workspace_settings( + workspace["workspace_id"], + req.model_dump(exclude_unset=True), + actor_id=workspace["user_id"], + ) + + +@router.patch("/api/workspaces/{workspace_id}") +async def update_workspace( + workspace_id: str, + req: WorkspaceUpdate, + workspace=Depends(require_permission("workspace.update")), +): + return await ctrl.update_workspace( + workspace["workspace_id"], req.name, actor_id=workspace["user_id"] + ) + + +@router.delete("/api/workspaces/{workspace_id}") +async def delete_workspace( + workspace_id: str, + workspace=Depends(get_current_workspace), +): + if workspace["role"] != "owner": + raise HTTPException(403, "Owner role required") + return await ctrl.delete_workspace( + workspace["workspace_id"], actor_id=workspace["user_id"] + ) + + +@router.post("/api/workspaces/{workspace_id}/leave") +async def leave_workspace( + workspace_id: str, + workspace=Depends(get_current_workspace), +): + return await ctrl.leave_workspace(workspace["workspace_id"], workspace["user_id"]) + + +@router.get("/api/workspaces/{workspace_id}/members") +async def list_members( + workspace_id: str, workspace=Depends(require_permission("members.view")) +): + return await ctrl.list_members(workspace["workspace_id"]) + + +@router.post("/api/workspaces/{workspace_id}/members") +async def invite_user( + workspace_id: str, + req: WorkspaceInviteCreate, + workspace=Depends(require_permission("members.invite")), +): + return await ctrl.invite_user( + workspace["workspace_id"], req.email, req.role, workspace["user_id"] + ) + + +@router.post("/api/workspaces/{workspace_id}/members/bulk") +async def bulk_invite_users( + workspace_id: str, + req: WorkspaceBulkInviteCreate, + workspace=Depends(require_permission("members.invite")), +): + return await ctrl.bulk_invite( + workspace["workspace_id"], req.emails, req.role, workspace["user_id"] + ) + + +@router.post("/api/workspaces/{workspace_id}/transfer-ownership") +async def transfer_ownership( + workspace_id: str, + req: WorkspaceOwnershipTransfer, + workspace=Depends(get_current_workspace), +): + if workspace["role"] != "owner": + raise HTTPException(403, "Owner role required") + return await ctrl.transfer_ownership( + workspace["workspace_id"], workspace["user_id"], req.user_id + ) + + +@router.put("/api/workspaces/{workspace_id}/members/{user_id}") +async def update_member_role( + workspace_id: str, + user_id: str, + req: WorkspaceMemberRoleUpdate, + workspace=Depends(require_permission("members.update_role")), +): + return await ctrl.update_member_role( + workspace["workspace_id"], + user_id, + req.role, + actor_id=workspace["user_id"], + actor_role=workspace["role"], + ) + + +@router.delete("/api/workspaces/{workspace_id}/members/{user_id}") +async def remove_member( + workspace_id: str, + user_id: str, + workspace=Depends(require_permission("members.remove")), +): + return await ctrl.remove_member( + workspace["workspace_id"], + user_id, + actor_id=workspace["user_id"], + actor_role=workspace["role"], + ) + + +@router.get("/api/workspaces/{workspace_id}/invites") +async def list_pending_invites( + workspace_id: str, workspace=Depends(require_permission("members.invite")) +): + return await ctrl.list_pending_invites(workspace["workspace_id"]) + + +@router.delete("/api/workspaces/{workspace_id}/invites/{invite_id}") +async def rescind_invite( + workspace_id: str, + invite_id: str, + workspace=Depends(require_permission("members.invite")), +): + return await ctrl.rescind_invite( + workspace["workspace_id"], invite_id, actor_id=workspace["user_id"] + ) + + +@router.post("/api/workspaces/{workspace_id}/invites/{invite_id}/resend") +async def resend_invite( + workspace_id: str, + invite_id: str, + workspace=Depends(require_permission("members.invite")), +): + return await ctrl.resend_invite( + workspace["workspace_id"], invite_id, actor_id=workspace["user_id"] + ) + + +@router.get("/api/workspaces/invites/me") +async def get_invites(user=Depends(get_current_user)): + user_id = user.get("user_id", user.get("sub")) + user_details = await get_user_by_id(user_id) + email = user_details.get("email") if user_details else None + if not email: + return [] + return await ctrl.get_invites(email) + + +@router.post("/api/workspaces/invites/{token}/accept") +async def accept_invite(token: str, user=Depends(get_current_user)): + user_id = user.get("user_id", user.get("sub")) + user_details = await get_user_by_id(user_id) + email = user_details.get("email") if user_details else None + if not email: + raise HTTPException(400, "User email not found") + return await ctrl.accept_invite(token, user_id, email) + + +@router.get("/api/workspaces/{workspace_id}/activity") +async def get_activity_log( + workspace_id: str, + limit: int = Query(50, ge=1, le=100), + offset: int = Query(0, ge=0), + event_type: str = Query(None), + workspace=Depends(require_permission("activity.view")), +): + return await activity_ctrl.get_workspace_activity( + workspace["workspace_id"], limit, offset, event_type + ) + + +@router.get("/api/workspaces/{workspace_id}/activity/export") +async def export_activity_log( + workspace_id: str, + event_type: str = Query(None), + workspace=Depends(require_permission("activity.export")), +): + async def rows(): + buf = io.StringIO() + writer = csv.writer(buf) + + def flush(): + buf.seek(0) + chunk = buf.read() + buf.seek(0) + buf.truncate(0) + return chunk + + writer.writerow( + [ + "timestamp", + "actor_email", + "event_type", + "resource_type", + "resource_id", + "details", + ] + ) + yield flush() + + async for a in activity_ctrl.iter_workspace_activity( + workspace["workspace_id"], event_type + ): + created = a.get("created_at") + writer.writerow( + [ + created.isoformat() if created else "", + a.get("actor_email") or "", + a.get("event_type") or "", + a.get("resource_type") or "", + a.get("resource_id") or "", + json.dumps(a.get("metadata_") or {}, default=str), + ] + ) + yield flush() + + filename = f"activity-{workspace_id}.csv" + return StreamingResponse( + rows(), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) diff --git a/backend/app/server.py b/backend/app/server.py index 984872bd..a9070cf1 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -1,10 +1,11 @@ """FastAPI application.""" +import asyncio import logging import os import sys from pathlib import Path -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.concurrency import run_in_threadpool @@ -21,15 +22,20 @@ from backend.app.pgdatabase import KnowledgeService from backend.app.pgdatabase.engine import async_session -from backend.app.routes.auth import router as auth_router -from backend.app.routes.system import router as system_router -from backend.app.routes.database import router as database_router -from backend.app.routes.onboard import router as onboard_router -from backend.app.routes.query import router as query_router -from backend.app.routes.history import router as history_router -from backend.app.routes.connections import router as connections_router -from backend.app.routes.catalog import router as catalog_router -from backend.app.routes.conversations import router as conversations_router +from backend.app.routes import ( + auth, + catalog, + connections, + conversations, + database, + history, + onboard, + query, + system, + workspaces, + dashboards, + saved_queries, +) logger = logging.getLogger(__name__) @@ -80,9 +86,23 @@ async def lifespan(app): else: from backend.app.pgdatabase import dispose_db - yield + # In-process periodic pruning of the activity log. Safe here because the + # deployment is single-process; a second worker would double up the work + # (harmless, but wasteful), so the flag exists to turn it off. + cleanup_task = None + if cfgmod.ACTIVITY_CLEANUP_ENABLED: + from backend.app.controllers.activity import activity_cleanup_loop - await dispose_db() + cleanup_task = asyncio.create_task(activity_cleanup_loop()) + + try: + yield + finally: + if cleanup_task: + cleanup_task.cancel() + with suppress(asyncio.CancelledError): + await cleanup_task + await dispose_db() def create_app(initial_db_url="", readonly=True): @@ -114,7 +134,7 @@ def create_app(initial_db_url="", readonly=True): providers = ProviderManager(cfg) db = DatabaseManager(readonly=readonly) kbs = KnowledgeService(async_session) - session_log = SessionLog(cfgmod.CONFIG_DIR) + session_log = SessionLog() app = FastAPI(title="BoloDB", version="2.0.0", lifespan=lifespan) app.state.limiter = limiter @@ -148,14 +168,19 @@ def create_app(initial_db_url="", readonly=True): "in the server. Connect from the UI after signing in." ) - app.include_router(auth_router) - app.include_router(system_router) - app.include_router(database_router) - app.include_router(onboard_router) - app.include_router(query_router) - app.include_router(history_router) - app.include_router(connections_router) - app.include_router(catalog_router) - app.include_router(conversations_router) + app.include_router(auth.router) + app.include_router(system.router) + app.include_router(database.router) + app.include_router(onboard.router) + app.include_router(query.router) + app.include_router(history.router) + app.include_router(connections.router) + app.include_router(catalog.router) + app.include_router(conversations.router) + app.include_router(workspaces.router) + app.include_router(dashboards.router) + app.include_router(saved_queries.router) + + # Catch-all for API 404app return app diff --git a/backend/app/services/email.py b/backend/app/services/email.py index 93ac344b..f06a3018 100644 --- a/backend/app/services/email.py +++ b/backend/app/services/email.py @@ -96,3 +96,34 @@ async def send_password_reset_email(to: str, reset_link: str) -> bool: """ return await send_email(to, "Reset your BoloDB password", html) + + +async def send_workspace_invite_email( + to: str, workspace_name: str, invite_code: str +) -> bool: + """Send a workspace invite email.""" + import html + + safe_name = html.escape(workspace_name) + safe_code = html.escape(invite_code) + html_content = f""" + + + + +

You've been invited!

+

+ You have been invited to join the {safe_name} workspace on BoloDB. +

+
+ {safe_code} +
+

+ Copy this code and paste it into the "Join Workspace" screen to accept the invitation. +

+ + + """ + return await send_email( + to, f"Invitation to join {workspace_name} on BoloDB", html_content + ) diff --git a/backend/docker-entrypoint.sh b/backend/docker-entrypoint.sh new file mode 100644 index 00000000..b13aac17 --- /dev/null +++ b/backend/docker-entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# Bring the database schema up to date, then start the app. +# +# Point DATABASE_URL at a different database and the next deploy migrates it +# from wherever it is to the current head — a fresh database gets the whole +# history, an existing one gets only what it is missing. Without this the app +# starts happily against a stale schema and fails later, one endpoint at a +# time, wherever a missing table or column happens to be touched. +# +# Set RUN_MIGRATIONS=false to skip (e.g. a read-only replica, or when +# migrations are applied by a separate release step). +set -e + +if [ "${RUN_MIGRATIONS:-true}" = "true" ]; then + if [ -z "${DATABASE_URL}" ]; then + echo "entrypoint: DATABASE_URL is not set — cannot migrate" >&2 + exit 1 + fi + + # A database that is still accepting connections gets a few chances; compose + # brings services up together and the app usually wins the race. + attempt=1 + max_attempts="${MIGRATION_MAX_ATTEMPTS:-10}" + until (cd /app/backend && PYTHONPATH=/app alembic upgrade head); do + if [ "$attempt" -ge "$max_attempts" ]; then + echo "entrypoint: migrations failed after ${attempt} attempts — refusing to start" >&2 + exit 1 + fi + echo "entrypoint: migration attempt ${attempt}/${max_attempts} failed, retrying in 3s" >&2 + attempt=$((attempt + 1)) + sleep 3 + done + echo "entrypoint: database schema is up to date" +fi + +exec "$@" diff --git a/backend/sample_data.py b/backend/sample_data.py index 71e566c3..f12b4096 100644 --- a/backend/sample_data.py +++ b/backend/sample_data.py @@ -1,75 +1,306 @@ -"""Creates TechStore sample database on first run.""" +"""Builds the sample webshop database on first run. -import sqlite3 +The data is a trimmed copy of the PostgreSQL sample database at +https://github.com/JannikArndt/PostgreSQLSampleDatabase — a webshop with 1,000 +customers, 1,000 products across ~17,700 articles, and 2,000 orders with their +positions. Upstream ships `pg_dump` output; `backend/sample_db/webshop.dump.gz` +holds just the `COPY … FROM stdin` blocks from it (see +`backend/sample_db/build_sample_dump.py`), which this module replays into a +local SQLite file so trying the sample needs no database of your own. +""" + +import gzip +import logging import os +import sqlite3 +import tempfile from pathlib import Path +logger = logging.getLogger(__name__) + +DUMP_PATH = Path(__file__).parent / "sample_db" / "webshop.dump.gz" + +# Bumped whenever the schema or data changes, so an existing sample file from an +# older release is rebuilt instead of silently serving the previous dataset. +SAMPLE_VERSION = 2 + +SCHEMA = """ +CREATE TABLE colors ( + id INTEGER PRIMARY KEY, + name TEXT, + rgb TEXT +); + +CREATE TABLE sizes ( + id INTEGER PRIMARY KEY, + gender TEXT, + category TEXT, + size TEXT, + size_us TEXT, + size_uk TEXT, + size_eu TEXT +); + +CREATE TABLE labels ( + id INTEGER PRIMARY KEY, + name TEXT, + slugname TEXT +); + +CREATE TABLE products ( + id INTEGER PRIMARY KEY, + name TEXT, + labelid INTEGER REFERENCES labels (id), + category TEXT, + gender TEXT, + currentlyactive INTEGER, + created TEXT, + updated TEXT +); + +CREATE TABLE articles ( + id INTEGER PRIMARY KEY, + productid INTEGER REFERENCES products (id), + ean TEXT, + colorid INTEGER REFERENCES colors (id), + sizeid INTEGER REFERENCES sizes (id), + description TEXT, + originalprice REAL, + reducedprice REAL, + taxrate REAL, + discountinpercent INTEGER, + currentlyactive INTEGER, + created TEXT, + updated TEXT +); + +CREATE TABLE stock ( + id INTEGER PRIMARY KEY, + articleid INTEGER REFERENCES articles (id), + count INTEGER, + created TEXT, + updated TEXT +); + +CREATE TABLE customers ( + id INTEGER PRIMARY KEY, + firstname TEXT, + lastname TEXT, + gender TEXT, + email TEXT, + dateofbirth TEXT, + currentaddressid INTEGER, + created TEXT, + updated TEXT +); + +CREATE TABLE addresses ( + id INTEGER PRIMARY KEY, + customerid INTEGER REFERENCES customers (id), + firstname TEXT, + lastname TEXT, + address1 TEXT, + address2 TEXT, + city TEXT, + zip TEXT, + created TEXT, + updated TEXT +); + +CREATE TABLE orders ( + id INTEGER PRIMARY KEY, + customerid INTEGER REFERENCES customers (id), + ordertimestamp TEXT, + shippingaddressid INTEGER REFERENCES addresses (id), + total REAL, + shippingcost REAL, + created TEXT, + updated TEXT +); + +CREATE TABLE order_positions ( + id INTEGER PRIMARY KEY, + orderid INTEGER REFERENCES orders (id), + articleid INTEGER REFERENCES articles (id), + amount INTEGER, + price REAL, + created TEXT, + updated TEXT +); + +CREATE INDEX idx_articles_product ON articles (productid); +CREATE INDEX idx_stock_article ON stock (articleid); +CREATE INDEX idx_orders_customer ON orders (customerid); +CREATE INDEX idx_order_positions_order ON order_positions (orderid); +CREATE INDEX idx_order_positions_article ON order_positions (articleid); +""" + +# `order` and `customer` are awkward names to ask questions against — one is a +# SQL reserved word, the other reads as a single row — so both are pluralised +# on the way in. `size` and `customer` become `sizeid`/`customerid` to match the +# foreign keys they actually are (upstream's own CREATE scripts name them so). +TABLE_NAMES = {"order": "orders", "customer": "customers", "address": "addresses"} +COLUMN_NAMES = { + "articles": {"size": "sizeid"}, + "order": {"customer": "customerid"}, +} + +# Everything not listed loads as text. +COLUMN_TYPES = { + "id": "int", + "labelid": "int", + "productid": "int", + "colorid": "int", + "sizeid": "int", + "articleid": "int", + "orderid": "int", + "customerid": "int", + "currentaddressid": "int", + "shippingaddressid": "int", + "count": "int", + "amount": "int", + "discountinpercent": "int", + "taxrate": "real", + "originalprice": "money", + "reducedprice": "money", + "price": "money", + "total": "money", + "shippingcost": "money", + "currentlyactive": "bool", +} + +# pg_dump escapes these inside COPY text fields. +_UNESCAPE = {"\\t": "\t", "\\n": "\n", "\\r": "\r", "\\\\": "\\"} -def ensure_sample_db(): - path = Path(os.path.expanduser("~")) / ".bolodb" / "sample_techstore.db" + +# The generated SQLite file lives in the repo's data/ directory, not a user +# home or a persistent volume. It is a cache rebuilt from the vendored dump, so +# it can be thrown away and regenerated at any time. Overridable for tests. +_DATA_DIR = Path( + os.environ.get("BOLODB_DATA_DIR", Path(__file__).resolve().parent.parent / "data") +) + + +def sample_db_path() -> Path: + return _DATA_DIR / f"sample_webshop_v{SAMPLE_VERSION}.db" + + +def ensure_sample_db() -> str: + """Return a connection URL for the sample database, building it if needed.""" + path = sample_db_path() path.parent.mkdir(parents=True, exist_ok=True) - if path.exists(): - return f"sqlite:///{path.as_posix()}" - c = sqlite3.connect(str(path)) - c.executescript( - """ - CREATE TABLE customers(id INTEGER PRIMARY KEY,name TEXT,email TEXT,segment TEXT,country TEXT,created_at TEXT); - CREATE TABLE products(id INTEGER PRIMARY KEY,name TEXT,category TEXT,price REAL,stock INTEGER); - CREATE TABLE orders(id INTEGER PRIMARY KEY,customer_id INTEGER,status TEXT,total_amount REAL,created_at TEXT, - FOREIGN KEY(customer_id) REFERENCES customers(id)); - CREATE TABLE order_items(id INTEGER PRIMARY KEY,order_id INTEGER,product_id INTEGER,quantity INTEGER,unit_price REAL, - FOREIGN KEY(order_id) REFERENCES orders(id),FOREIGN KEY(product_id) REFERENCES products(id)); - CREATE TABLE payments(id INTEGER PRIMARY KEY,order_id INTEGER,method TEXT,amount REAL,paid_at TEXT, - FOREIGN KEY(order_id) REFERENCES orders(id)); - - INSERT INTO customers VALUES - (1,'Ananya Rao','ananya@mail.com','vip','India','2025-11-02'), - (2,'Marcus Hale','marcus@mail.com','business','USA','2026-01-15'), - (3,'Priya Menon','priya@mail.com','vip','India','2025-08-20'), - (4,'Devin Cole','devin@mail.com','consumer','UK','2026-03-10'), - (5,'Sofia Lund','sofia@mail.com','business','Sweden','2025-12-01'), - (6,'Tom Becker','tom@mail.com','consumer','Germany','2026-02-22'); - - INSERT INTO products VALUES - (1,'Aurora 14" Laptop','laptops',1299.00,3), - (2,'Zephyr Phone X','phones',699.00,14), - (3,'NovaBuds Pro','audio',149.00,6), - (4,'PixelView 27"','monitors',449.00,8), - (5,'Glide Wireless Mouse','accessories',49.00,11), - (6,'TypeMaster Keyboard','peripherals',129.00,22), - (7,'PowerHub USB-C','accessories',39.00,85), - (8,'SoundBar S1','audio',199.00,31); - - INSERT INTO orders VALUES - (101,1,'completed',7498.00,'2026-06-02'), - (102,2,'completed',6210.00,'2026-06-01'), - (103,3,'completed',5880.00,'2026-05-28'), - (104,4,'pending',4999.00,'2026-06-03'), - (105,5,'completed',5120.00,'2026-06-04'), - (106,6,'completed',798.00,'2026-05-20'), - (107,1,'completed',3499.00,'2026-04-12'), - (108,2,'cancelled',1299.00,'2026-03-01'), - (109,3,'completed',449.00,'2026-06-05'), - (110,4,'completed',149.00,'2026-05-30'); - - INSERT INTO order_items VALUES - (1,101,1,1,1299),(2,101,2,1,699),(3,102,6,1,129),(4,103,3,2,149), - (5,104,1,1,1299),(6,105,4,1,449),(7,106,5,2,49),(8,107,8,1,199), - (9,108,6,1,129),(10,109,4,1,449),(11,110,3,1,149); - - INSERT INTO payments VALUES - (1,101,'card',7498.00,'2026-06-02'), - (2,102,'paypal',6210.00,'2026-06-01'), - (3,103,'card',5880.00,'2026-05-28'), - (5,105,'bank',5120.00,'2026-06-04'), - (6,106,'card',798.00,'2026-05-20'); - """ - ) - c.commit() - c.close() - print(f"Sample database created: {path}") + if not path.exists(): + _build(path) + _remove_stale_samples(path) return f"sqlite:///{path.as_posix()}" +def _build(path: Path) -> None: + """Load the vendored dump into a fresh SQLite file at `path`. + + Built into a temporary file in the same directory and moved into place, so a + build that dies halfway (or two workers racing on first connect) can never + leave a half-loaded database behind for the next request to read. + """ + fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), suffix=".db") + os.close(fd) + tmp = Path(tmp_name) + try: + conn = sqlite3.connect(str(tmp)) + try: + conn.executescript(SCHEMA) + for table, columns, rows in _read_dump(): + placeholders = ", ".join("?" * len(columns)) + quoted = ", ".join(f'"{c}"' for c in columns) + conn.executemany( + f'INSERT INTO "{table}" ({quoted}) VALUES ({placeholders})', rows + ) + conn.commit() + finally: + conn.close() + os.replace(tmp, path) + logger.info("Sample database created: %s", path) + except Exception: + tmp.unlink(missing_ok=True) + raise + + +def _remove_stale_samples(current: Path) -> None: + """Delete sample files left by earlier versions of the sample dataset.""" + for old in current.parent.glob("sample_*.db"): + if old != current: + try: + old.unlink() + except OSError as e: + logger.warning("Could not remove old sample database %s: %s", old, e) + + +def _read_dump(): + """Yield (table, columns, rows) for each COPY block in the vendored dump.""" + with gzip.open(DUMP_PATH, "rt", encoding="utf-8") as fh: + table = None + columns: list[str] = [] + converters: list[str] = [] + rows: list[tuple] = [] + for line in fh: + line = line.rstrip("\n") + if line.startswith("COPY "): + header = line[len("COPY ") :] + source_table = header[: header.index(" (")].strip() + source_columns = [ + c.strip() + for c in header[header.index("(") + 1 : header.rindex(")")].split( + "," + ) + ] + renames = COLUMN_NAMES.get(source_table, {}) + table = TABLE_NAMES.get(source_table, source_table) + columns = [renames.get(c, c) for c in source_columns] + converters = [COLUMN_TYPES.get(c, "text") for c in columns] + rows = [] + elif line == "\\.": + yield table, columns, rows + table = None + elif table is not None: + values = line.split("\t") + rows.append( + tuple( + _convert(v, t) for v, t in zip(values, converters, strict=True) + ) + ) + + +def _convert(value: str, kind: str): + if value == "\\N": + return None + if kind == "text": + return _unescape(value) + if kind == "bool": + return 1 if value == "t" else 0 + if kind == "int": + return int(value) + if kind == "money": + # pg's `money` type dumps as `$1,234.56`, negatives as `-$1,234.56`. + cleaned = value.replace("$", "").replace(",", "") + return float(cleaned) if cleaned else None + return float(value) + + +def _unescape(value: str) -> str: + if "\\" not in value: + return value + out = [] + i = 0 + while i < len(value): + pair = value[i : i + 2] + if pair in _UNESCAPE: + out.append(_UNESCAPE[pair]) + i += 2 + else: + out.append(value[i]) + i += 1 + return "".join(out) + + if __name__ == "__main__": print(ensure_sample_db()) diff --git a/backend/sample_db/build_sample_dump.py b/backend/sample_db/build_sample_dump.py new file mode 100644 index 00000000..28cec46c --- /dev/null +++ b/backend/sample_db/build_sample_dump.py @@ -0,0 +1,135 @@ +"""Regenerate `webshop.dump.gz` from the upstream PostgreSQL sample database. + +The sample store BoloDB ships is a trimmed copy of +https://github.com/JannikArndt/PostgreSQLSampleDatabase — a webshop with 1,000 +customers, 1,000 products, ~17,700 articles and 2,000 orders. Upstream ships +`pg_dump` files that only Postgres can restore; BoloDB needs a zero-setup SQLite +file, so this script keeps the parts that carry data (the `COPY … FROM stdin` +blocks) and drops everything Postgres-specific around them. + +Run it only when refreshing the vendored data: + + python backend/sample_db/build_sample_dump.py /path/to/PostgreSQLSampleDatabase + +The output, `backend/sample_db/webshop.dump.gz`, is read at runtime by +`backend.sample_data.ensure_sample_db`. +""" + +import gzip +import sys +from datetime import datetime, timedelta +from pathlib import Path + +OUT = Path(__file__).parent / "webshop.dump.gz" + +# Load order respects the foreign keys between tables. +SOURCES = [ + ("colors", "create.sql"), + ("sizes", "create.sql"), + ("labels", "labels.sql"), + ("products", "products.sql"), + ("articles", "articles.sql"), + ("stock", "stock.sql"), + ("customer", "customer.sql"), + ("address", "address.sql"), + ("order", "order.sql"), + ("order_positions", "order_positions.sql"), +] + +# `icon` is a bytea blob of brand logos — megabytes of hex that no question in a +# SQL demo ever asks about. +DROP_COLUMNS = {"labels": {"icon"}} + + +def _copy_block(text: str, table: str): + """Return (columns, rows) for a table's `COPY … FROM stdin` block.""" + needle_variants = ( + f"COPY webshop.{table} (", + f'COPY webshop."{table}" (', + ) + for needle in needle_variants: + start = text.find(needle) + if start != -1: + break + else: + raise SystemExit(f"No COPY block for {table}") + + header_end = text.index("\n", start) + header = text[start:header_end] + columns = [ + c.strip() for c in header[header.index("(") + 1 : header.rindex(")")].split(",") + ] + + body = text[header_end + 1 :] + body = body[: body.index("\n\\.")] + rows = [line.split("\t") for line in body.split("\n") if line] + return columns, rows + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit(__doc__) + data_dir = Path(sys.argv[1]) / "data" + + cache: dict[str, str] = {} + tables = [] + latest = None + + for table, filename in SOURCES: + if filename not in cache: + cache[filename] = (data_dir / filename).read_text(encoding="utf-8") + columns, rows = _copy_block(cache[filename], table) + + drop = DROP_COLUMNS.get(table, set()) + keep = [i for i, c in enumerate(columns) if c not in drop] + columns = [columns[i] for i in keep] + rows = [[r[i] for i in keep] for r in rows] + + for row in rows: + for value in row: + stamp = _parse_timestamp(value) + if stamp and (latest is None or stamp > latest): + latest = stamp + + tables.append((table, columns, rows)) + + # The upstream data was generated in 2018. Left alone, every "last month" + # or "this year" question against the demo returns nothing, which reads as a + # broken product rather than old data. Slide the whole history forward so the + # newest row lands on today, keeping every interval between rows intact. + shift = timedelta(0) + if latest is not None: + shift = datetime.now().replace(microsecond=0) - latest + + with gzip.open(OUT, "wt", encoding="utf-8", compresslevel=9) as fh: + for table, columns, rows in tables: + fh.write(f"COPY {table} ({', '.join(columns)}) FROM stdin;\n") + for row in rows: + fh.write("\t".join(_shift(v, shift) for v in row) + "\n") + fh.write("\\.\n") + + total = sum(len(rows) for _, _, rows in tables) + print( + f"Wrote {OUT} — {len(tables)} tables, {total} rows, shifted by {shift.days} days" + ) + + +def _parse_timestamp(value: str): + """Parse a pg_dump `timestamp with time zone` literal, or return None.""" + if len(value) < 19 or value[4] != "-" or value[7] != "-" or value[10] != " ": + return None + try: + return datetime.strptime(value[:19], "%Y-%m-%d %H:%M:%S") + except ValueError: + return None + + +def _shift(value: str, shift: timedelta) -> str: + stamp = _parse_timestamp(value) + if stamp is None: + return value + return (stamp + shift).strftime("%Y-%m-%d %H:%M:%S") + + +if __name__ == "__main__": + main() diff --git a/backend/sample_db/webshop.dump.gz b/backend/sample_db/webshop.dump.gz new file mode 100644 index 00000000..52927c53 Binary files /dev/null and b/backend/sample_db/webshop.dump.gz differ diff --git a/design.md b/design.md deleted file mode 100644 index 88c1ca1d..00000000 --- a/design.md +++ /dev/null @@ -1,116 +0,0 @@ -# BoloDB — Hallmark Design Audit - -**Date:** 2026-07-16 -**Branch:** hallmark-redesign - -## Anti-Pattern Findings - -| ID | Severity | Pattern | Status | -|---|---|---|---| -| C1 | Critical | AI-default nav (wordmark-left, links-centred, CTA-right) | Fixed — N5 floating pill | -| C2 | Critical | Centred hero layout | Fixed — asymmetric 2-column | -| C3 | Critical | Centred everything (eyebrows + centred text) | Fixed — eyebrows removed, FinalCta left-aligned | -| C4 | Critical | 3-column equal feature grid | Fixed — 2-col + hero card | -| C5 | Critical | AI-default footer (social icons, multi-column) | Fixed — Ft6 letter close | -| C6 | Critical | Fake browser chrome in demo | Fixed — minimal demo-top with badge | -| C7 | Critical | Token improvisation (raw hex in components) | Fixed — lifted to tokens | -| C8 | Critical | Invented metrics (128940) | Fixed — replaced with — placeholder | -| M3 | Major | Centred text in sections | Fixed — left-aligned where needed | -| M4 | Major | GSAP scroll animations (performance) | Fixed — removed from Pipeline, FinalCta, TrustEngine | -| M5 | Major | Eyebrow labels on every section | Fixed — removed all eyebrows | - -## Changes Applied - -### Commit 1: `fix(routing)` -- `connect/+page.svelte`: Shows "currently connected" banner with switch/disconnect options -- `chat/+page.svelte`: Shows "No database connected" empty state instead of blank flash -- `dashboard/+page.svelte`: Shows empty state with "Connect a database" CTA -- `onboard/+page.svelte`: Added loading state while checking connection -- `Navbar.svelte`: Conditionally shows "Switch DB" vs "Connect" based on `appState.dbInfo` - -### Commit 2: `fix(nav)` -- `MarketingNav.svelte`: Full rewrite from full-width sticky nav to floating pill at bottom of viewport -- Pill appears after scrolling past hero (200px threshold) -- Frosted glass backdrop-filter, logo · divider · links · divider · actions -- Mobile: links hidden, full-width pill at bottom - -### Commit 3: `fix(hero)` -- `Hero.svelte`: Replaced centred single-column with asymmetric 2-column grid (1.2fr + 0.8fr) -- Removed eyebrow label, removed `min-height: 90vh`, content-based height -- Mobile: collapses to centred single column at 768px - -### Commit 4: `fix(components)` -- `TrustEngine.svelte`: Break 3-col equal grid → 2-col bottom + full-width hero card -- `LiveDemo.svelte`: Remove fake browser chrome, replace raw hex with tokens, remove eyebrow -- `Flywheel.svelte`: Replace invented metric 128940 with — placeholder, remove eyebrow - -### Commit 5: `fix(footer)` -- `Footer.svelte`: Rewrite as Ft6 — brand left, links below, reassurance right -- `Pipeline.svelte`: Remove eyebrow, remove GSAP spine scroll animation -- `Integrations.svelte`: Remove eyebrow, remove use:reveal -- `FinalCta.svelte`: Left-align panel, remove GSAP scroll animation, remove magnetic action - -### Commit 6: `fix(alembic)` -- Rename duplicate revision 0003 → 0004 for password_reset_tokens migration - -### Commit 7: `fix: reduce card padding, add missing sampleSQL` -- Pipeline cards: 32px → 24px padding -- TrustEngine cards: 36px → 28px padding -- TrustEngine flip card: define sampleSQL (was undefined) - -### Commit 8: `feat: add privacy and terms pages` -- Create `/privacy` and `/terms` routes with legal content -- Footer: add Privacy and Terms links -- Layout: add legal routes to hidden navbar paths - -## Design Tokens - -| Token | Value | Usage | -|---|---|---| -| `--brand` | `#1b9e6b` | Primary accent | -| `--bg` | `#eef1f0` | Page background | -| `--surface` | `#f6f8f7` | Card background | -| `--ink` | `#1a1d1c` | Primary text | -| `--muted` | `#5c6360` | Secondary text | -| `--faint` | `#8c918e` | Tertiary text | -| `--border` | `#d4dbd8` | Borders | -| `--radius-sm` | `8px` | Small radius | -| `--radius-md` | `12px` | Medium radius | -| `--radius-lg` | `16px` | Large radius | -| `--font-display` | `Hanken Grotesk` | Headings | -| `--font-mono` | `JetBrains Mono` | Code/labels | - -## Marketing Page Architecture - -```text -MarketingLayout -├── MarketingNav (floating pill, bottom) -├── Hero (left-biased 2-col) -├── LiveDemo (minimal panel) -├── Pipeline (3-step cards) -├── TrustEngine (2-col + hero card) -├── Flywheel (diagram + stats) -├── Integrations (DB grid + connection strings) -├── FinalCta (left-aligned panel) -└── Footer (Ft6 letter close) -``` - -## Files Modified - -- `frontend/src/lib/marketing/MarketingNav.svelte` -- `frontend/src/lib/marketing/Hero.svelte` -- `frontend/src/lib/marketing/LiveDemo.svelte` -- `frontend/src/lib/marketing/Pipeline.svelte` -- `frontend/src/lib/marketing/TrustEngine.svelte` -- `frontend/src/lib/marketing/Flywheel.svelte` -- `frontend/src/lib/marketing/Integrations.svelte` -- `frontend/src/lib/marketing/FinalCta.svelte` -- `frontend/src/lib/marketing/Footer.svelte` -- `frontend/src/routes/privacy/+page.svelte` -- `frontend/src/routes/terms/+page.svelte` -- `frontend/src/routes/connect/+page.svelte` -- `frontend/src/routes/chat/+page.svelte` -- `frontend/src/routes/dashboard/+page.svelte` -- `frontend/src/routes/onboard/+page.svelte` -- `frontend/src/lib/components/ui/Navbar.svelte` -- `backend/alembic/versions/0004_add_password_reset_tokens.py` diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..616a63cd --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,51 @@ +# ────────────────────────────────────────────── +# BoloDB · Local development overrides +# ────────────────────────────────────────────── +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build +# +# NOT auto-loaded (unlike docker-compose.override.yml), so Coolify / +# production deploys that use only docker-compose.yml get the static +# nginx image without a Vite bind-mount that empties /app. +# ────────────────────────────────────────────── + +services: + backend: + ports: + - "127.0.0.1:4322:4321" + + # Vite HMR for local UI work — overrides the production static nginx build + frontend: + build: + context: ./frontend + dockerfile: DOCKERFILE.dev + container_name: bolodb-frontend + volumes: + - ./frontend:/app + - /app/node_modules + ports: + - "127.0.0.1:5174:5173" + environment: + - BACKEND_URL=http://backend:4321 + - PUBLIC_POSTHOG_PROJECT_TOKEN=${PUBLIC_POSTHOG_PROJECT_TOKEN} + - PUBLIC_POSTHOG_HOST=${PUBLIC_POSTHOG_HOST:-https://us.i.posthog.com} + - VITE_SUPABASE_URL=${SUPABASE_URL:?SUPABASE_URL must be set} + - VITE_SUPABASE_ANON_KEY=${SUPABASE_ANON_KEY:?SUPABASE_ANON_KEY must be set} + depends_on: + backend: + condition: service_healthy + networks: + - bolodb-net + + nginx: + build: + context: ./nginx + dockerfile: DOCKERFILE.dev + container_name: bolodb-nginx + ports: + - "127.0.0.1:8080:80" + depends_on: + frontend: + condition: service_started + backend: + condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index 62079c6c..dae35362 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,86 +1,69 @@ -# ────────────────────────────────────────────── -# BoloDB · Docker Compose -# ────────────────────────────────────────────── -# Usage: -# docker compose up --build # build & start -# docker compose up --build -d # detached -# docker compose down # stop & remove -# ────────────────────────────────────────────── - services: # ── Backend: FastAPI / Uvicorn ── backend: build: context: . dockerfile: backend/DOCKERFILE - container_name: bolodb-backend restart: unless-stopped - ports: - - "4321:4321" + expose: + - "4321" volumes: - - bolodb-data:/home/bolodb/.bolodb - - ./data:/app/data + # The generated sample database is a regenerable cache; the volume keeps + # it across restarts so it isn't rebuilt each time, but nothing here is + # state that has to survive — all real data lives in Postgres. + - bolodb-app-data:/app/data environment: - JWT_SECRET=${JWT_SECRET:?JWT_SECRET must be set} + # Stable secret that encrypts saved connection URLs. Never written to + # disk, so it must be supplied here and kept constant across deploys. + - RECENT_CONNECTIONS_KEY=${RECENT_CONNECTIONS_KEY:?RECENT_CONNECTIONS_KEY must be set} - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:?OPENROUTER_API_KEY must be set} - DATABASE_URL=${DATABASE_URL:? Database URL must be set} - PYTHONUNBUFFERED=1 - RUNNING_IN_DOCKER=true - # - OLLAMA_URL=http://host.docker.internal:11434 + - SKIP_DB_LOCK=true - SUPABASE_URL=${SUPABASE_URL:?SUPABASE_URL must be set} - SUPABASE_ANON_KEY=${SUPABASE_ANON_KEY:?SUPABASE_ANON_KEY must be set} - SUPABASE_JWT_SECRET=${SUPABASE_JWT_SECRET:?SUPABASE_JWT_SECRET must be set} - MYEMAILVERIFIER_API_KEY=${MYEMAILVERIFIER_API_KEY} - RESEND_API_KEY=${RESEND_API_KEY} + healthcheck: + test: ["CMD-SHELL", "python -c \"import httpx; httpx.get('http://localhost:4321/api/health', timeout=3).raise_for_status()\" || exit 1"] + interval: 10s + timeout: 5s + retries: 3 + # Migrations run before the server binds, and a database that needs the + # whole history takes longer than a few seconds — don't call the + # container unhealthy while it is still migrating. + start_period: 90s extra_hosts: - "host.docker.internal:host-gateway" networks: - bolodb-net - # ── Frontend: Svelte (Vite) dev server ── - frontend: - build: - context: ./frontend - dockerfile: DOCKERFILE.dev - container_name: bolodb-frontend - volumes: - - ./frontend:/app - - /app/node_modules - ports: - - "5173:5173" - environment: - - BACKEND_URL=http://backend:4321 - - PUBLIC_POSTHOG_PROJECT_TOKEN=${PUBLIC_POSTHOG_PROJECT_TOKEN} - - PUBLIC_POSTHOG_HOST=${PUBLIC_POSTHOG_HOST:-https://us.i.posthog.com} - - VITE_SUPABASE_URL=${SUPABASE_URL:?SUPABASE_URL must be set} - - VITE_SUPABASE_ANON_KEY=${SUPABASE_ANON_KEY:?SUPABASE_ANON_KEY must be set} - depends_on: - backend: - condition: service_healthy - networks: - - bolodb-net - - # ── Nginx: Reverse proxy + Vite dev server ── + # ── Nginx: static frontend + /api reverse proxy ── nginx: build: - context: ./nginx - dockerfile: DOCKERFILE - container_name: bolodb-nginx + context: . + dockerfile: nginx/DOCKERFILE + args: + PUBLIC_POSTHOG_PROJECT_TOKEN: ${PUBLIC_POSTHOG_PROJECT_TOKEN:-} + PUBLIC_POSTHOG_HOST: ${PUBLIC_POSTHOG_HOST:-https://us.i.posthog.com} restart: unless-stopped expose: - - '80' + - "8080" + labels: # <--- ADD THIS + - "traefik.http.services.${COOLIFY_CONTAINER_NAME:-nginx}.loadbalancer.server.port=8080" depends_on: - frontend: - condition: service_started backend: condition: service_healthy networks: - bolodb-net volumes: - bolodb-data: bolodb-postgres-data: bolodb-postgres-wal: + bolodb-app-data: networks: bolodb-net: diff --git a/docs/01-what-is-bolodb.md b/docs/01-what-is-bolodb.md index 4cb7ee24..cad7aa87 100644 --- a/docs/01-what-is-bolodb.md +++ b/docs/01-what-is-bolodb.md @@ -2,102 +2,94 @@ **BoloDB lets you talk to your database in plain English.** -You connect a database (PostgreSQL, MySQL, SQLite, SQL Server, DuckDB), and -then instead of writing SQL — the technical language databases understand — -you just ask questions the way you'd ask a colleague: +It is a multi-tenant web application. You sign up for an account, create or join a workspace, connect a database (PostgreSQL, MySQL, SQLite, SQL Server, DuckDB), and then instead of writing SQL you just ask questions the way you'd ask a colleague: > "How many orders were completed last month?" > > "Which product category brings in the most revenue?" -BoloDB translates the question into SQL using OpenRouter AI, runs it -against your database, and shows you: +BoloDB translates the question into SQL using OpenRouter AI (deepseek/deepseek-v4-flash), runs it against your database, and shows you: -1. **The answer** — a table of results (and the raw SQL, one tap away). -2. **A restatement** — one plain sentence describing what was actually - computed, e.g. *"Count of orders with status 'completed' created in the - previous calendar month."* This is how you check the AI understood you. -3. **A confidence badge** — High / Medium / Low, based on real signals - (did the query run? does it match something you verified before?), not on - the AI's self-assessment. +1. **The answer** — a table of results (and the raw SQL, one tap away). Many results also show a **chart** — the AI decides whether a bar, line, area, pie, or big-number visualization fits best. +2. **A restatement** — one plain sentence describing what was actually computed. This is how you check the AI understood you. +3. **A confidence badge** — High / Medium / Low, based on real signals (did the query run? does it match something you verified before?), not on the AI's self-assessment. ## The three promises ### 1. Your data is safe -- BoloDB opens your database in **read-only mode**. Every query is checked - twice — once by parsing it into a syntax tree and rejecting anything that - isn't a plain SELECT (code: `backend/app/database.py` → - `_readonly_violation()`), and we recommend connecting with a read-only - database account as a third layer. -- When the AI is asked to write SQL, it is sent your **question** plus the - context BoloDB builds around it: the **database structure** (table and - column names, keys), a few **sample values and sample rows** per table (so - the AI matches your wording to real data), your confirmed **glossary - terms**, previously **verified question→SQL examples**, and the last couple - of conversation turns. The prompt **never** includes bulk table contents, - query results, or credentials — your OpenRouter API key is used only as the - request's authentication header. See chapter 3 for the exact prompt - contents. +- BoloDB opens your database in **read-only mode**. Every query is checked twice — once by parsing it into a syntax tree and rejecting anything that isn't a plain SELECT (code: `backend/app/database.py` → `_readonly_violation()`), and we recommend connecting with a read-only database account as a third layer. +- Database URLs are validated against an allowlist to **prevent SSRF attacks** — no localhost, no loopback, no cloud metadata endpoints (`backend/app/database.py` → `_validate_db_url()`). +- When the AI is asked to write SQL, it is sent your **question** plus context: the **database structure** (table and column names, keys), a few **sample values and sample rows** per table, your confirmed **glossary terms**, the **semantic catalog** (metrics, joins, synonyms, value maps), previously **verified question→SQL examples**, and the last couple of conversation turns. The prompt **never** includes bulk table contents, query results, or credentials. See chapter 3 for the exact prompt contents. +- Connection credentials saved to the workspace are **encrypted at rest** using Fernet symmetric encryption (`backend/app/pgdatabase/connections.py`). ### 2. You can trust the answers — and check them -Non-technical users can't audit SQL, so BoloDB gives you tools that don't -require reading code: +Non-technical users can't audit SQL, so BoloDB gives you tools that don't require reading code: -- The **restatement** (above) tells you in English what was computed. -- The **Explain** feature can translate any SQL back into plain English - (code: `backend/app/llm.py` → `explain_sql()`, exposed at the - `/api/explain` endpoint). -- The **confidence badge** is computed from signals, not vibes (code: - `backend/app/schema_link.py` → `compute_confidence()`). -- Broken SQL never silently produces a wrong answer: every generated query is - validated against your real schema before it runs, and automatically - repaired if it fails (code: `backend/app/repair.py`). +- The **restatement** tells you in English what was computed. +- The **chart** gives you a visual check — if the bar chart looks reasonable, the SQL probably is too. +- The **Explain** feature can translate any SQL back into plain English (code: `backend/app/llm.py` → `explain_sql()`, exposed at the `/api/explain` endpoint). +- The **confidence badge** is computed from signals, not vibes (code: `backend/app/schema_link.py` → `compute_confidence()`). +- Broken SQL never silently produces a wrong answer: every generated query is validated against your real schema before it runs, and automatically repaired if it fails (code: `backend/app/repair.py`). ### 3. It gets better the more you use it -When you click **"Yes, correct"** on an answer, that question-and-SQL pair is -saved into a local knowledge base (code: `backend/app/knowledge.py`). The next -time you — or a colleague — asks something similar, BoloDB shows the AI your -verified example, which is the single strongest accuracy boost we have. Your -trust level climbs from *Supervised* → *Assisted* → *Trusted* as you verify -more answers. - -## What happens the first time you connect (onboarding) - -1. **Set up the AI** — paste a free OpenRouter API key - (get one at https://aistudio.google.com/app/api-keys). Screen: - `frontend/src/lib/components/ConnectScreen.svelte`. -2. **Connect a database** — or click *"Try with sample data"* to explore a - realistic demo store database (`backend/sample_data.py`). -3. **Confirm business terms** — BoloDB reads your schema and guesses the 3 - most important business terms ("revenue", "active customer"...) and what - they mean in *your* database. You confirm or correct them. (Code: - `backend/app/llm.py` → `generate_glossary()`.) -4. **Verify starter questions** — BoloDB proposes 3 example questions with - answers; each one you confirm seeds the knowledge base. (Code: - `backend/app/llm.py` → `generate_starters()`.) - -After onboarding you land in the chat screen -(`frontend/src/lib/components/AskScreen.svelte`) and just ask. +When you click **"Yes, correct"** on an answer, that question-and-SQL pair is saved into the workspace's knowledge base in PostgreSQL (code: `backend/app/pgdatabase/knowledge.py`). The next time you — or a teammate — asks something similar, BoloDB finds your verified example and includes it in the prompt, which is the single strongest accuracy boost we have. + +## Accounts, workspaces, and permissions + +BoloDB is a **multi-tenant** application: + +- **Sign up** with email/password or Google OAuth. +- Create or join **workspaces**. Each workspace has its own databases, knowledge base, dashboards, and members. +- Each member has a **role**: *Owner*, *Admin*, or *Member*. Roles control fine-grained **permissions**: who can connect databases, who can manage the catalog, who can create dashboards, and more. See chapter 10 for the full permission matrix. + +## What happens the first time you log in + +1. **Sign up** — create an account (or continue with Google). +2. **Create or join a workspace** — workspaces are where your databases and knowledge live. +3. **Set up the AI** — paste an OpenRouter API key (get one at https://openrouter.ai/keys). Or if the workspace admin already configured it, skip this step. +4. **Connect a database** — or click *"Try with sample data"* to explore a realistic demo store. +5. **Confirm business terms** — BoloDB reads your schema and suggests business terms and their meanings. You confirm or correct them (`backend/app/llm.py` → `suggest_catalog()`). +6. **Verify starter questions** — BoloDB proposes example questions with answers; each one you confirm seeds the knowledge base (`backend/app/llm.py` → `generate_starters()`). + +After onboarding you land in the chat screen and just ask. ## The cast of components (30-second architecture) ```text Browser (SvelteKit frontend, frontend/src) - │ HTTP calls to /api/... + │ HTTP calls to /api/... (JWT cookie auth) ▼ FastAPI backend (backend/app) │ - ├── controllers/query.py ── the question→answer pipeline (the heart) - ├── llm.py ──────────────── the ONLY file that talks to OpenRouter - ├── schema_link.py ──────── picks which tables the AI sees - ├── sqlvalidate.py ──────── checks generated SQL against the real schema - ├── repair.py ───────────── auto-fixes SQL that fails, with AI feedback - ├── database.py ─────────── connects to YOUR database, read-only execution - ├── knowledge.py ────────── remembers verified answers + glossary (SQLite) - └── config.py ───────────── settings: OpenRouter key + model (~/.bolodb/) + ├── controllers/query.py ─── the question→answer pipeline (the heart) + ├── llm.py ───────────────── the ONLY file that talks to OpenRouter + ├── schema_link.py ───────── picks which tables the AI sees + ├── sqlvalidate.py ───────── checks generated SQL against the real schema + ├── repair.py ────────────── auto-fixes SQL that fails, with AI feedback + ├── database.py ──────────── connects to YOUR database, read-only execution + ├── pgdatabase/knowledge.py ─ remembers verified answers + glossary (PostgreSQL) + └── auth routes ──────────── JWT auth, Google OAuth, email verification + PostgreSQL ────────── users, workspaces, knowledge, dashboards, history +``` + +### Services layer + +```text + Workspace ────────── has members with roles and permissions + │ + ├── Database connection ─── keyed per workspace+db_id + │ └── Schema introspection → schema_link → prompt + │ + ├── Knowledge (PostgreSQL) ─ verified Q&A, glossary, catalog + │ + ├── Saved queries ───────── queries saved for reuse + │ + ├── Dashboards ──────────── panels with ECharts visualizations + │ + └── Activity log ────────── append-only audit trail ``` The next chapter walks a single question through this whole pipeline with diff --git a/docs/02-how-a-question-becomes-an-answer.md b/docs/02-how-a-question-becomes-an-answer.md index 91cf7ec1..d747b36c 100644 --- a/docs/02-how-a-question-becomes-an-answer.md +++ b/docs/02-how-a-question-becomes-an-answer.md @@ -1,146 +1,79 @@ # 2. How a question becomes an answer -This is the most important chapter: it follows **one question** through the -entire system, in order, with the exact code location for every step. If you -ever wonder "where does X happen?" or "why did I get this answer?", start here. +This chapter follows **one question** through the entire BoloDB system, in order, with exact code locations for every step. -Our example question, asked against a demo e-commerce database: +Our example question: > **"How much revenue did we make from laptops last month?"** +--- + ## Step 0 — The browser sends the question -You type into the chat screen and press *Ask*. - -- **Where:** `frontend/src/lib/components/AskScreen.svelte` → the `ask()` - function sends `POST /api/query` with the question, plus the previous couple - of question/SQL pairs (so follow-up questions like "now just 2023" work). -- The request lands in `backend/app/routes/query.py` → `query()`, which calls - the pipeline: `backend/app/controllers/query.py` → `run_query()`. - -Everything below happens inside `run_query()`. The step numbers here match -the numbered comments in that function — you can read them side by side. - -## Step 1 — Look up what BoloDB already knows - -Two lookups against the local knowledge base (a small SQLite file at -`~/.bolodb/knowledge.db`, managed by `backend/app/knowledge.py`): - -1. **Glossary** — the business terms you confirmed during onboarding. - For our question, the glossary might contain - `revenue = sum of total_amount on completed orders`. - *Code:* `KnowledgeBase.get_glossary()`. -2. **Similar verified answers** — up to 3 previously-confirmed - question→SQL pairs that resemble this question, found with a text - similarity score (word overlap + sequence similarity). - *Code:* `KnowledgeBase.retrieve_similar()`, scoring in `_similarity()`. - -Both are later placed into the AI prompt: the glossary as ground-truth term -definitions, the verified answers as worked examples to imitate. - -## Step 2 — Pick which tables the AI gets to see (schema linking) - -A real database may have 40+ tables. Sending them all would cost more AND -confuse the AI. So BoloDB scores every table against the question and keeps -only the relevant ones. - -- **Where:** `backend/app/schema_link.py` → `link_relevant_tables()`. -- For our question, tables named `orders`, `order_items`, `products` score - high (name/column matches on "revenue"→glossary→"total_amount", - "laptops" matches a known value of `products.category`), and tables like - `employees` or `audit_log` score zero and are dropped. -- Foreign keys are then followed so the chosen tables can be JOINed - (details in [chapter 4](04-schema-linking.md)). -- How many tables are allowed depends on the OpenRouter model configured — - `model_budget()` in the same file. -- On big databases (30+ tables), a cheap AI pre-pass shortlists candidate - tables from a names-only catalog first — "two-stage linking", chapter 4. - -The chosen tables are rendered into a compact text form the AI reads — -`compact_schema()` produces lines like: - -```text -orders(id PK, customer_id->customers.id, status[completed,pending,cancelled], total_amount, created_at) ~38104 rows -``` - -## Step 3 — Generate, validate, execute — and self-repair if needed - -This is the loop that actually produces the answer. **Where:** -`backend/app/repair.py` → `run_repair_loop()`, wired up in `run_query()`. - -Each iteration (at most 3, and no new attempt after 60 seconds): - -1. **Generate.** The OpenRouter AI is asked for the SQL. - *Code:* `backend/app/llm.py` → `generate_sql()`; the exact prompt text is - assembled in `build_sql_system_prompt()` in the same file — open it to see - precisely what the AI is told. The AI must reply in a fixed JSON shape - (`SQL_SCHEMA`): the SQL, a plain-English restatement, and any assumptions - it made. -2. **Validate.** Before anything touches your database, the SQL is parsed - and every table and column it mentions is checked against your real - schema. A hallucinated column like `orders.revenue` is caught here. - *Code:* `backend/app/sqlvalidate.py` → `validate_sql()`. -3. **Execute.** The query runs against your database — read-only, with the - safety guard described in [chapter 5](05-safety-validation-and-self-repair.md). - *Code:* `backend/app/database.py` → `DatabaseManager.execute()`. -4. **Repair (only if 2 or 3 failed).** The specific error — "Unknown column: - 'revenue' does not exist in table 'orders'" or the database's own error - message — is appended to the question and the loop starts over. The AI - sees exactly what broke and fixes it. *Code:* the feedback text is built - by `_feedback()` in `repair.py`. And if the failed SQL referenced a real - table that step 2 didn't select, that table is added to the prompt for - the retry ("schema-retry" — chapter 4). - -The response records how many attempts were needed (`attempts` field) and -whether a repair happened (`repaired`) — useful when debugging. - -## Step 4 — Score confidence, log, and answer - -- **Confidence** is computed from real signals — did the query run, did it - return rows, and how closely does the question match something you already - verified. *Code:* `backend/app/schema_link.py` → `compute_confidence()`. - The exact thresholds are in [chapter 6](06-learning-trust-and-confidence.md). -- The query is **logged** to the session log (`backend/app/logbook.py`) and - the per-user history (MongoDB, `backend/app/mongodatabase.py`, called from - `routes/query.py`). -- The JSON response goes back to the browser, which renders it as an answer - card: `frontend/src/lib/components/AnswerCard.svelte`. - -## Step 5 — You give feedback (optional but powerful) - -If you click **"Yes, correct"**, the browser calls `POST /api/feedback` → -`backend/app/controllers/query.py` → `feedback()`, which saves the -question+SQL into the knowledge base (`KnowledgeBase.add_verified()`). -From now on, similar questions retrieve this example in Step 1 and show -higher confidence in Step 4. This loop — ask, verify, improve — is the -product's core flywheel. - -## The full picture - -```text - Question ─▶ [1] knowledge lookup knowledge.py - [2] schema linking schema_link.py - [3] generate ◀────┐ llm.py (OpenRouter) - validate │ sqlvalidate.py - execute │ database.py - (repair) ─────┘ repair.py - [4] confidence + log schema_link.py, logbook.py - Answer ◀── JSON: sql, restatement, rows, confidence, attempts -``` - -## What the response actually contains - -`POST /api/query` returns (shape assembled at the end of `run_query()`): - -| Field | Meaning | -|---|---| -| `sql` | The SQL that was run (empty if generation failed entirely) | -| `restatement` | One plain sentence describing what the query computes | -| `assumptions` | Assumptions the AI made (e.g. "interpreted 'last month' as the previous calendar month") | -| `columns`, `rows` | The results (max 500 rows; `truncated` is true if capped) | -| `confidence`, `confidence_reason` | High/Medium/Low + why | -| `based_on_verified` | Whether a previously verified answer informed this one | -| `tables_used` | Which tables schema linking selected | -| `attempts`, `repaired` | How many generation attempts were needed; whether a self-repair happened | -| `execution_error` | Present only if the final attempt still failed | -| `query_id` | Handle for feedback ("Yes, correct" / "Something's wrong") | +You type into the chat screen (`frontend/src/lib/components/AskScreen.svelte`) and submit. + +- **Request**: Sends `POST /api/query` containing: + - Natural language question + - Conversation context (previous question/SQL pairs for follow-up resolution) + - `X-Workspace-Id` and `X-Db-Id` headers (Workspace and Database selection) + - JWT authentication cookie (`backend/app/dependencies.py` → `get_current_user`) +- **Route**: `backend/app/routes/query.py` → `query()`, which invokes `backend/app/controllers/query.py` → `run_query()`. + +--- + +## Step 1 — Knowledge Lookup & Semantic Layer Injection + +BoloDB queries the PostgreSQL workspace knowledge base (`backend/app/pgdatabase/knowledge.py` → `KnowledgeService`): + +1. **Business Glossary**: Confirmed business term definitions (e.g., `revenue = sum of total_amount on completed orders`). +2. **Semantic Layer Catalog**: Active metrics (`CatalogMetric`), join paths (`CatalogJoin`), synonyms (`CatalogSynonym`), and value mappings (`CatalogValueMapping`). +3. **Verified Answer Examples**: Up to 3 similar verified question→SQL pairs retrieved via similarity scoring (`retrieve_similar()`). + +--- + +## Step 2 — Schema Linking (Table Selection) + +To optimize cost and avoid confusing the AI, BoloDB filters the database schema down to only the relevant tables. + +- **Location**: `backend/app/schema_link.py` → `link_relevant_tables()`. +- Tables matching "revenue" (via glossary `total_amount`), "laptops" (via product category value match), and "orders" are selected. +- Foreign keys and junction tables (e.g. `order_items`) are expanded (`expand_linked_tables()`). +- On large databases (30+ tables), a cheap pre-pass shortlists candidates (`llm.py` → `shortlist_tables()`). +- Selected tables are formatted into compact text representations via `compact_schema()`. + +--- + +## Step 3 — Generation, Validation, Execution & Self-Repair + +Orchestrated by `backend/app/repair.py` → `run_repair_loop()`: + +1. **Generate**: OpenRouter AI is invoked (`backend/app/llm.py` → `generate_sql()`). The system prompt is constructed by `build_sql_system_prompt()`. The response returns structured JSON containing: + - `sql`: The generated query + - `restatement`: Plain English summary of what was computed + - `assumptions`: List of assumptions made by the model + - `chart`: Inferred visualization specification (chart type, x/y columns, title) +2. **Validate**: The SQL is parsed into an AST via `sqlglot` (`backend/app/sqlvalidate.py` → `validate_sql()`) to verify that all referenced tables and columns exist. +3. **Execute**: The query runs in **read-only** mode with statement timeouts and SSRF protection (`backend/app/database.py` → `DatabaseManager.execute()`). +4. **Self-Repair (if step 2 or 3 failed)**: The exact error is appended to the prompt (`_feedback()`), and the loop retries (up to 3 attempts within a 60-second window). + +--- + +## Step 4 — Confidence Scoring, Logging & Visualization + +- **Confidence**: Derived from real runtime signals (`schema_link.py` → `compute_confidence()`). +- **Persistence**: Recorded in PostgreSQL query history (`backend/app/pgdatabase/history.py`) and workspace activity log. +- **Rendering**: Returned as JSON (or streamed via Server-Sent Events). The frontend renders an `AnswerCard.svelte` (`frontend/src/lib/components/AnswerCard.svelte`) featuring: + - Plain English restatement + - Interactive data table + - **ECharts visualization** (`frontend/src/lib/components/ChartPanel.svelte`) based on the inferred `chart` spec + - Confidence pill and SQL view toggle + - **"Save to Dashboard"** button (`frontend/src/lib/components/SaveQueryDialog.svelte`) + +--- + +## Step 5 — Feedback & Verification Flywheel + +If a user clicks **"Yes, correct"** or **"Verify"**: +1. Frontend calls `POST /api/feedback` → `backend/app/routes/query.py` → `controllers/query.py`. +2. The question-and-SQL pair is added to PostgreSQL via `KnowledgeService.add_verified()`. +3. Subsequent similar queries retrieve this verified example during Step 1. diff --git a/docs/03-the-ai-layer-openrouter.md b/docs/03-the-ai-layer-openrouter.md index a9687ae3..1e5cdd8f 100644 --- a/docs/03-the-ai-layer-openrouter.md +++ b/docs/03-the-ai-layer-openrouter.md @@ -1,30 +1,79 @@ -# AI Layer — OpenRouter +# 3. The AI Layer — OpenRouter -BoloDB uses OpenRouter (`DeepSeek V4 Flash`) for all AI operations: -- SQL generation (question → SQL) -- SQL explanation (SQL → plain English) -- Glossary suggestion (onboarding) -- Starter question generation (onboarding) -- Semantic catalog suggestion -- Two-stage schema linking (table shortlisting) +BoloDB relies exclusively on **OpenRouter** (`deepseek/deepseek-v4-flash`) for all language-model operations. -## Configuration +--- -Set `OPENROUTER_API_KEY` in the environment. The key is shared across all -users — there is no per-user API key storage. +## 1. Responsibilities of the AI Layer -## Architecture +All AI interactions flow through a single module: `backend/app/llm.py`. The AI layer is responsible for: +1. **SQL Generation** (`generate_sql`): Translating plain English questions into valid database SQL queries, plain-English restatements, and chart visualization specs. +2. **SQL Explanation** (`explain_sql`): Translating raw SQL back into plain English for auditing. +3. **Semantic Catalog Suggestion** (`suggest_catalog`): Proposing domain-specific business terms, metrics, joins, and value maps based on database schema introspection. +4. **Starter Question Generation** (`generate_starters`): Generating sample questions and SQL pairs during onboarding to seed the knowledge base. +5. **Semantic Layer Enrichment**: Assisting in identifying joins, synonyms, and value mappings. +6. **Two-Stage Schema Linking**: Shortlisting tables on large schemas (30+ tables) before full schema linking. -All AI operations funnel through `backend/app/llm.py`: +--- -- `OpenRouterProvider` — wraps the OpenAI SDK pointed at - `https://openrouter.ai/api/v1`. Uses `deepseek/deepseek-v4-flash` with - `max_retries=2` and `temperature=0.1`. -- `create_provider()` — factory function, called at startup. -- `ProviderManager` — caches a single shared provider instance. +## 2. OpenRouter Architecture -## Structured Output +### Provider & Client (`backend/app/llm.py`) +- **`OpenRouterProvider`**: Wraps the `openai.AsyncOpenAI` SDK configured with base URL `https://openrouter.ai/api/v1`. +- **`ProviderManager`**: Thread-safe manager that maintains and caches the active provider instance. +- **Model**: `deepseek/deepseek-v4-flash` running with low temperature (`temperature=0.1`) for maximum determinism and code precision. +- **Configuration**: The API key is read strictly from the `OPENROUTER_API_KEY` environment variable. -All AI responses use OpenAI's `response_format: { type: "json_schema" }` -with `strict: true`. Schemas are defined as module-level constants in -`llm.py` with the `"name"` wrapper required by the API. +--- + +## 3. Structured Output Contract + +To ensure 100% reliable parsing without markdown wrapping or hallucinated structures, BoloDB uses OpenAI JSON Schema enforcement (`response_format={"type": "json_schema", ...}`). + +### `SQL_SCHEMA` Contract +The model returns a JSON object adhering to the following schema: + +```json +{ + "sql": "SELECT category_name, SUM(total_amount) AS revenue FROM orders JOIN products ON ... GROUP BY 1", + "restatement": "Computes total revenue for each product category from completed orders.", + "assumptions": ["Assumed completed orders means status = 'completed'."], + "chart": { + "type": "bar", + "x_axis": "category_name", + "y_axis": "revenue", + "title": "Revenue by Category", + "reason": "Bar chart compares revenue across product categories." + } +} +``` + +### Tolerant Parsing (`parse_json`) +If a model returns JSON wrapped in markdown code blocks (` ```json ... ``` `) or trailing whitespace, `llm.py`'s `parse_json()` utility strips code fences and extracts clean JSON. + +--- + +## 4. Streaming & Real-Time Responses + +For fast user feedback during query generation: +- The backend serves chunked Server-Sent Events (SSE) streaming responses on `POST /api/query/stream`. +- `frontend/src/lib/api.ts` provides `streamApiCall()`, which reads the SSE response stream chunk by chunk to display thinking state and partial output in real-time (`Thinking.svelte`). + +--- + +## 5. System Prompt Assembly + +The system prompt is assembled dynamically in `build_sql_system_prompt()` (`backend/app/llm.py`). It includes: +1. **Database Dialect**: e.g., PostgreSQL, MySQL, SQLite, DuckDB syntax rules. +2. **Linked Schema**: Compact table definitions produced by `compact_schema()` in `schema_link.py`. +3. **Glossary Terms**: Verified business definitions. +4. **Semantic Layer Context**: Metrics, join paths, synonyms, and value mappings from `pgdatabase/knowledge.py`. +5. **Verified Examples**: Similar past question/SQL pairs retrieved from PostgreSQL knowledge base. +6. **Conversation Context**: Last 2-3 chat turns for follow-up resolution. + +--- + +## 6. Error Handling & Retries + +- **Unified Exception**: All LLM operations raise a unified `LLMError(user_message, detail)` exception on failure. User-facing friendly messages (such as `HIGH_TRAFFIC_MESSAGE` for rate limits or provider downtime) are stored in `user_message`, while raw provider details or redacted errors are preserved in `detail`. +- **Retries**: Network failures or transient API errors trigger up to 2 retries with exponential backoff before raising `LLMError`. diff --git a/docs/05-safety-validation-and-self-repair.md b/docs/05-safety-validation-and-self-repair.md index 8861768b..cbfec20a 100644 --- a/docs/05-safety-validation-and-self-repair.md +++ b/docs/05-safety-validation-and-self-repair.md @@ -1,127 +1,84 @@ -# 5. Safety, validation and self-repair +# 5. Safety, validation, and self-repair -AI-generated SQL fails in two ways: it can be **dangerous** (tries to change -data) or **broken** (references things that don't exist, wrong syntax). -BoloDB handles both with three independent layers. Each layer lives in its -own file and is separately tested. +AI-generated SQL fails in two ways: it can be **dangerous** (tries to change data) or **broken** (references non-existent columns, syntax errors). Furthermore, database connections must be protected against network exploits. + +BoloDB handles safety and reliability through multiple defense-in-depth layers. ```text + Database Connection Attempt + │ + ▼ + [0] SSRF & Network Security ── host allowlist, loopback blocking database.py + │ AI writes SQL │ ▼ - ① Static validation — does every table/column actually exist? sqlvalidate.py + ① Static validation ──────── does every table/column exist? sqlvalidate.py │ pass ▼ - ② Read-only guard — is this a pure SELECT? nothing else runs database.py + ② Read-only guard ──────── is this a pure SELECT statement? database.py │ pass ▼ - ③ Execution — did the database accept it? database.py + ③ Exec & Statement Timeout ── 5s timeout & execution safety database.py │ any failure at ①/③ ▼ - ↻ Self-repair loop — feed the exact error back to the AI repair.py + ↻ Self-repair loop ──────── feed exact error back to AI repair.py ``` +--- + +## Layer 0 — Network Security & SSRF Protection (`backend/app/database.py`) + +**Function:** `_validate_db_url(db_url)` + +Before a database connection is established, the target URL undergoes strict Server-Side Request Forgery (SSRF) validation: +- **Loopback & Private Address Blocking**: Requests targeting `localhost`, `127.0.0.1`, `0.0.0.0`, `::1`, or link-local / cloud metadata IPs (`169.254.169.254`) are strictly blocked. +- **Host Allowlisting**: Connection hosts are validated to prevent internal infrastructure scanning. +- **Encrypted Credentials at Rest**: Connection details stored in PostgreSQL are encrypted using Fernet symmetric keys (`backend/app/pgdatabase/connections.py`). + +--- + ## Layer ① — Static validation (`backend/app/sqlvalidate.py`) **Function:** `validate_sql(sql, schema, dialect)` -Before generated SQL goes anywhere near your database, it is parsed into a -syntax tree (using the `sqlglot` library) and every table and column it -references is checked against your *actual* schema. This catches the most -common AI mistake — confidently inventing a column: +Before generated SQL touches your database, it is parsed into a syntax tree (using `sqlglot`) and every table and column referenced is checked against your *actual* schema. This catches column hallucinations instantly: > `Unknown column: 'revenue' does not exist in table 'orders'.` -That sentence is not just an error — it's written to be **fed back to the AI** -so it can fix itself (see layer ↻ below). - -Design principle stated at the top of the file: the validator is deliberately -*conservative*. It only complains when it is certain something is wrong -(unknown table, unknown column on a known table). Columns that might come -from a subquery, CTE, or **SELECT alias** are left alone — a false rejection -of valid SQL would be worse than missing an error, because execution -(layer ③) is the final backstop anyway. Column aliases from the SELECT list -are collected and treated as valid references in ORDER BY / GROUP BY / HAVING. - -Tests: `tests/test_sqlvalidate.py`. - -## Layer ② — The read-only guard (`backend/app/database.py`) - -**Function:** `DatabaseManager._readonly_violation()`, called from -`execute()` on **every** query — including raw SQL a user runs directly via -`/api/execute`. - -The rules, enforced on the parsed syntax tree (not just text matching): - -- Only a single statement (no stacked `SELECT …; DROP …`). -- The statement root must be SELECT / UNION / EXPLAIN. -- **No modifying node anywhere in the tree** — this catches sneaky cases - like a DELETE hidden inside a CTE (`WITH x AS (DELETE …)`). The node - blocklist is `_MODIFYING_NODES`. -- `SELECT … INTO` (which creates a table) is blocked explicitly. -- If the SQL can't even be parsed, a conservative keyword blocklist - (`WRITE_KEYWORDS`) rejects anything suspicious instead of running it - blindly. - -On top of this, results are capped at 500 rows (`max_rows`) and the README -recommends connecting with a read-only database account — defense in depth. - -Tests: `tests/test_database.py`. - -## Layer ③ — Execution errors become repair fuel - -If the database itself rejects the query (syntax quirk, type error, missing -extension), `execute()` returns `{"error": ""}` -rather than raising — so the error text can be recycled by the repair loop. - -## Layer ↻ — The self-repair loop (`backend/app/repair.py`) - -**Function:** `run_repair_loop(generate, validate, execute, …)` — wired into -the live pipeline in `backend/app/controllers/query.py` → `run_query()`. - -The insight: most wrong SQL is wrong *mechanically*, and the AI can fix it -instantly **if you tell it exactly what broke**. So instead of giving up on -the first failure: - -1. Generate a candidate. -2. If validation (①) or execution (③) fails, build a correction prompt — - `_feedback()` produces: - - ```text - The previous SQL attempt was: - SELECT revenue FROM orders - Problems found: - - Unknown column: 'revenue' does not exist in table 'orders'. - Return a corrected SQL query that fixes these problems. - ``` - -3. Ask again with that feedback appended (it flows into - `generate_sql(..., feedback=...)` in `llm.py`). -4. **Schema-retry:** before regenerating, the failed SQL is checked for - tables that exist in the database but weren't shown to the model — a sign - the *table selection* (not the SQL) was the problem. Any such table is - added to the prompt for the next attempt. Details in - [chapter 4](04-schema-linking.md); code: - `expand_linked_tables()` + the `on_failure` hook. -5. Repeat — **at most 3 attempts total, and no new attempt starts after 60 - seconds** (constants `_MAX_ATTEMPTS` / `_MAX_SECONDS` in - `controllers/query.py`), so a stubborn failure can't burn tokens or leave - the user staring at a spinner forever. - -What you see as a user: usually nothing — the answer just works. The response -does carry the evidence: `attempts` (how many generations were needed) and -`repaired: true` if a fix happened mid-flight. If all attempts fail, you get -a Low-confidence answer with the final error in `execution_error` and a -suggestion to rephrase. - -Tests: `tests/test_repair.py` (the loop in isolation), -`tests/test_query_pipeline.py` (the loop wired into the real pipeline — -including proof that invalid SQL **never reaches the database**). - -## What still can't go wrong silently - -- A query that runs but answers the *wrong question* is the one failure no - validator can catch. That's what the plain-English **restatement**, the - **assumptions list**, the **Explain** button and the **confidence badge** - are for — they make it easy for a human to spot. See - [chapter 6](06-learning-trust-and-confidence.md). +The validator is deliberately conservative: it only complains when certain something is wrong. Columns coming from CTEs, subqueries, or aliases are preserved. + +--- + +## Layer ② — Read-only Guard (`backend/app/database.py`) + +**Function:** `DatabaseManager._readonly_violation()`, called from `execute()` on **every** query. + +Enforced on the parsed syntax tree: +- Single statement restriction (blocks stacked queries like `SELECT ...; DROP ...`). +- Statement root must be `SELECT`, `UNION`, or `EXPLAIN`. +- **No modifying node anywhere in the tree** — catches mutations inside CTEs (`WITH x AS (DELETE ...)`). +- Blocked node types: `INSERT`, `UPDATE`, `DELETE`, `DROP`, `ALTER`, `CREATE`, `TRUNCATE`, `SELECT INTO`. +- Keyword fallback blocklist if AST parsing fails. + +--- + +## Layer ③ — Execution Safety & Statement Timeout + +**Function:** `_apply_statement_timeout(engine, dialect)` in `backend/app/database.py`. + +- **Statement Timeouts**: Every connection applies statement timeouts (e.g., `SET statement_timeout = 5000` on PostgreSQL / MySQL `max_execution_time`) to prevent runaway long-running queries from overwhelming the target database. +- **Row Caps**: Execution results are capped at 500 rows (`max_rows`) to prevent memory exhaustion. +- **Error Recycling**: Execution errors are returned cleanly to supply fuel for self-repair. + +--- + +## Layer ↻ — Self-Repair Loop (`backend/app/repair.py`) + +**Function:** `run_repair_loop()` + +When static validation (Layer ①) or database execution (Layer ③) fails: +1. BoloDB constructs a targeted correction prompt (`_feedback()`). +2. The exact error message is fed back to OpenRouter in `generate_sql(..., feedback=...)`. +3. **Schema-retry**: If the failed SQL references a real table that schema linking missed, that table is added to the context prompt for the next attempt. +4. **Limits**: Maximum 3 attempts, with a hard 60-second total execution deadline. diff --git a/docs/06-learning-trust-and-confidence.md b/docs/06-learning-trust-and-confidence.md index 2d63be3a..297fc0cf 100644 --- a/docs/06-learning-trust-and-confidence.md +++ b/docs/06-learning-trust-and-confidence.md @@ -1,42 +1,30 @@ -# 6. Learning, trust and confidence +# 6. Learning, trust, and confidence -BoloDB's accuracy is not static — it improves with use, per database, because -of a simple loop: **every answer you confirm becomes an example the AI sees -next time.** This chapter covers the knowledge base, the confidence badge, -and the trust level. +BoloDB's accuracy is not static — it improves with use, per workspace and database connection, because of a simple loop: **every answer you confirm becomes an example the AI sees next time.** -## The knowledge base (`backend/app/knowledge.py`) +--- -A small local SQLite file (`~/.bolodb/knowledge.db`, path set in -`backend/app/config.py`) with two tables, both keyed by a database -fingerprint (`db_id`, derived in `backend/app/database.py` → `db_id_for()` — -so knowledge for one connected database never leaks into another): +## 1. The Knowledge Base (`backend/app/pgdatabase/knowledge.py`) -1. **`verified`** — question, SQL, restatement of every answer you confirmed. - Written by `KnowledgeBase.add_verified()` (near-duplicate questions are - skipped — similarity > 0.92). -2. **`glossary`** — your confirmed business terms: - term → plain meaning → SQL hint. Written during onboarding - (`set_glossary()`). +Knowledge is persisted asynchronously in PostgreSQL via `KnowledgeService`. All entries are strictly scoped by **Workspace ID** and **Database ID** (`db_id`), ensuring data from one workspace or database never leaks to another. -### How verified answers are found again +### Managed Entities +1. **Verified Q&A (`VerifiedQA`)**: Question, SQL, restatement, and metadata of every answer confirmed by a workspace member (`add_verified()`). Near-duplicate questions are skipped if similarity > 0.92. +2. **Business Glossary (`Glossary`)**: Workspace-confirmed business terms (`term` → `meaning` → `sql_hint`), managed via `set_glossary()`. +3. **Semantic Layer**: Business metrics, join rules, synonyms, and value mappings (`CatalogMetric`, `CatalogJoin`, `CatalogSynonym`, `CatalogValueMapping`), managed via `set_catalog()`. -When a new question arrives, `retrieve_similar()` scores it against every -stored verified question: +### Retrieval & Similar Question Matching +When a new question arrives, `retrieve_similar()` retrieves up to 3 similar verified Q&A entries using Jaccard word-overlap and string distance algorithms (`_similarity()`): -```text -similarity = 0.6 × word-overlap (Jaccard) + 0.4 × character-sequence ratio -``` +$$\text{similarity} = 0.6 \times \text{word-overlap} + 0.4 \times \text{sequence-similarity}$$ -(code: `_similarity()`). The top 3 above a 0.25 threshold are injected into -the AI prompt as worked examples (`_examples_block()` in -`backend/app/llm.py`) — few-shot examples of *your own verified SQL* are the -strongest accuracy signal the system has. +Matches above a 0.25 threshold are injected directly into the system prompt as worked examples (`_examples_block()` in `backend/app/llm.py`). -## The confidence badge (`backend/app/schema_link.py` → `compute_confidence()`) +--- -The High/Medium/Low badge is computed from **observable signals** — never -from asking the AI how confident it feels. The exact rules: +## 2. The Confidence Badge (`backend/app/schema_link.py` → `compute_confidence()`) + +The High/Medium/Low badge is derived strictly from **observable runtime signals**: | Situation | Badge | Reason shown | |---|---|---| @@ -46,53 +34,36 @@ from asking the AI how confident it feels. The exact rules: | No match, query returned zero rows | **Low** | "no matching rows - the question may not match your data" | | No match, query returned rows | **Medium** | "new question - please confirm it is right" | -Note what this implies: **the only way to see High confidence is to verify -answers.** That's intentional — high confidence is earned from your -confirmations, not claimed. +--- -## The trust level (Supervised → Assisted → Trusted) +## 3. The Trust Level (Supervised → Assisted → Trusted) -A per-database summary of how much verified knowledge exists — computed in -`KnowledgeBase.trust_level()` and mirrored in the frontend -(`frontend/src/lib/data.ts` → `trustFor()`): +A workspace database trust summary computed in `KnowledgeService` and reflected in the UI (`frontend/src/lib/data.ts` → `trustFor()`): | Verified answers | Level | Product behaviour | |---|---|---| -| 0–2 | **Supervised** | Every answer waits for your confirmation while it learns. | -| 3–6 | **Assisted** | Confident answers shown; novel ones get a second look. | -| 7+ | **Trusted** | Answers shown directly; reasoning on tap. | - -Displayed as the pill in the chat top bar -(`frontend/src/lib/components/ui/TrustPill.svelte`) and the meter in -onboarding (`TrustMeter.svelte`). - -## The feedback flow, end to end - -1. You click **"Yes, correct"** on an answer card - (`frontend/src/lib/components/AnswerCard.svelte`). -2. `POST /api/feedback` → `backend/app/controllers/query.py` → - `feedback()`. -3. The verdict is logged (`backend/app/logbook.py` — an append-only JSONL - file per session in `~/.bolodb/sessions/`, useful for auditing what was - asked and answered). -4. If the verdict is "correct", the pair is stored via `add_verified()` and - the new trust level is returned to the UI. -5. Clicking **"Something's wrong"** logs the verdict + reason (wrong numbers / - wrong filter / not what I meant …) without storing the answer. - -## Explain — trust without reading SQL - -For any query, `POST /api/explain` (`backend/app/llm.py` → `explain_sql()`) -returns 2–4 plain-English sentences describing what the SQL actually does — -which data it looks at, how it filters and groups, how it's ordered. It's the -"reverse translation" that lets a non-technical user audit an answer without -learning SQL. - -## Where each artifact lives on disk - -| Artifact | Location | Written by | +| 0–2 | **Supervised** | Every answer prompts for confirmation while learning. | +| 3–6 | **Assisted** | Confident answers displayed; novel ones get a second look. | +| 7+ | **Trusted** | Answers displayed directly; reasoning on tap. | + +--- + +## 4. End-to-End Feedback & Verification Flow + +1. User clicks **"Yes, correct"** or **"Verify"** on an answer card (`frontend/src/lib/components/AnswerCard.svelte`). +2. Frontend issues `POST /api/feedback` or `POST /api/verify` → `backend/app/routes/query.py` → `backend/app/controllers/query.py`. +3. The verified pair is saved into PostgreSQL via `KnowledgeService.add_verified()`. +4. Workspace activity is recorded in `WorkspaceActivityLog`. + +--- + +## 5. Persistence Map in BoloDB v2 + +| Entity | Storage | Managed by | |---|---|---| -| Settings (model, API key — key encrypted at rest) | `~/.bolodb/config.json` + `~/.bolodb/.secret` | `backend/app/config.py` | -| Verified answers + glossary | `~/.bolodb/knowledge.db` | `backend/app/knowledge.py` | -| Session query/feedback log | `~/.bolodb/sessions/session-*.jsonl` | `backend/app/logbook.py` | -| Per-user query history | MongoDB (Docker volume) | `backend/app/mongodatabase.py` | +| Users & Workspaces | PostgreSQL | `backend/app/pgdatabase/users.py`, `workspaces.py` | +| Verified Q&A, Glossary, Catalog | PostgreSQL | `backend/app/pgdatabase/knowledge.py` | +| Encrypted DB Connections | PostgreSQL | `backend/app/pgdatabase/connections.py` | +| Query History | PostgreSQL | `backend/app/pgdatabase/history.py` | +| Dashboards & Saved Queries | PostgreSQL | `backend/app/pgdatabase/dashboards.py`, `saved_queries.py` | +| Audit Trail / Session Log | PostgreSQL | `backend/app/models/activity.py` | diff --git a/docs/07-file-map.md b/docs/07-file-map.md index d4e57f9e..1c547d5b 100644 --- a/docs/07-file-map.md +++ b/docs/07-file-map.md @@ -8,89 +8,217 @@ Use this as an index: find the thing you care about, open the file next to it. | File | What it does | |---|---| -| `backend/main.py` | Command-line entry point. Parses `--db/--port/...`, warns if no OpenRouter key is configured, starts the web server. | -| `backend/app/server.py` | Builds the FastAPI app: creates the shared objects (config, provider manager, database manager, knowledge base, session log) and mounts all routes. | -| `backend/sample_data.py` | Builds the "Try with sample data" demo database (TechStore e-commerce). | +| `backend/app/server.py` | Main FastAPI server entry point — builds the app (`create_app()`), initializes shared objects (config, ProviderManager, DatabaseManager, KnowledgeService, SessionLog), runs Alembic migrations at startup, mounts all route modules, applies CORS + rate limiting + body-limit middleware. | +| `backend/sample_data.py` | Builds the "Try with sample data" demo database (PostgreSQL webshop dataset). | ### The AI + question pipeline (the heart of the product) | File | What it does | |---|---| -| `backend/app/llm.py` | **The only file that talks to an AI.** `OpenRouterProvider` (HTTP calls, retries, error taxonomy), the prompt builders, and the four AI operations: `generate_sql`, `explain_sql`, `generate_glossary`, `generate_starters`. → [chapter 3](03-the-ai-layer-openrouter.md) | -| `backend/app/schema_link.py` | Scores tables against the question and picks which ones the AI sees (`link_relevant_tables`), renders them compactly (`compact_schema`), sets per-model budgets (`model_budget`), computes the confidence badge (`compute_confidence`). → [chapter 4](04-schema-linking.md) | +| `backend/app/llm.py` | **The only file that talks to an AI model.** `OpenRouterProvider` (HTTP calls via openai SDK, retries, error taxonomy, streaming), the structured-output schemas (`SQL_SCHEMA`, `CHART_TYPES`), `suggest_catalog()`, and the AI operations: `generate_sql`, `explain_sql`, `suggest_catalog`, `generate_starters`. Now produces a `chart` spec alongside SQL. → [chapter 3](03-the-ai-layer-openrouter.md) | +| `backend/app/schema_link.py` | Scores tables against the question and picks which ones the AI sees (`link_relevant_tables`), renders them compactly (`compact_schema`), sets per-model budgets (`model_budget`), expands FK/junction tables (`expand_linked_tables`), computes the confidence badge (`compute_confidence`). → [chapter 4](04-schema-linking.md) | | `backend/app/sqlvalidate.py` | Static validation: parses generated SQL and checks every table/column against the real schema before execution. → [chapter 5](05-safety-validation-and-self-repair.md) | | `backend/app/repair.py` | The generate→validate→execute→repair loop that auto-fixes broken SQL by feeding the exact error back to the AI. → [chapter 5](05-safety-validation-and-self-repair.md) | -| `backend/app/knowledge.py` | The local knowledge base (SQLite): verified Q→SQL answers, glossary, similarity retrieval, trust level. → [chapter 6](06-learning-trust-and-confidence.md) | -| `backend/app/database.py` | Connects to *your* database (SQLAlchemy), introspects the schema (tables, columns, keys, sample values, row counts), and executes queries with the read-only guard. | -| `backend/app/config.py` | Settings on disk (`~/.bolodb/config.json`): OpenRouter model + API key, allowed model list, migration from pre-OpenRouter configs, `OPENROUTER_API_KEY` env fallback. | +| `backend/app/controllers/query.py` | **`run_query()`** — orchestrates the whole pipeline: knowledge lookup → schema linking → generate/validate/execute/repair → confidence scoring → chart inference → logging. → [chapter 2](02-how-a-question-becomes-an-answer.md) | +| `backend/app/semantic.py` | Semantic catalog helpers: `suggest_from_schema()` derives joins and value maps from the schema, `merge_catalog_suggestions()` combines them with LLM enrichment, `filter_catalog()` scopes entries to linked tables. → [chapter 12](12-semantic-layer.md) | +| `backend/app/database.py` | `DatabaseManager` — connects to *your* database (SQLAlchemy), introspects schema (tables, columns, PKs, FKs, sample rows, row counts, distinct values), read-only guard via AST parsing, SSRF validation, statement timeout. Keyed per workspace+db_id. | +| `backend/app/config.py` | In-memory only. API key read from `OPENROUTER_API_KEY` env var. Activity log retention settings also live here. | | `backend/app/utils.py` | `_tokens()` — the cached word tokenizer used by schema linking and similarity scoring. | -### HTTP layer +### Routes and controllers -Routes (`backend/app/routes/`) are thin — they parse the request and call a -controller (`backend/app/controllers/`) where the logic lives. +Routes (`backend/app/routes/`) are thin — they parse the request, enforce permissions, and call a controller (`backend/app/controllers/`) where the logic lives. | Route file | Endpoints | Controller | |---|---|---| -| `routes/query.py` | `POST /api/query` (ask a question), `/api/feedback`, `/api/verify`, `/api/execute` (raw SQL), `/api/explain` (SQL → English) | `controllers/query.py` — **`run_query()` is the whole pipeline** | -| `routes/system.py` | `GET /api/state`, `GET /api/health`, `POST /api/config` (model + API key) | `controllers/system.py` | -| `routes/database.py` | `/api/connect`, `/api/connect-sample`, `/api/reconnect`, `/api/disconnect`, `/api/schema` | `controllers/database.py` | -| `routes/onboard.py` | `/api/onboard/glossary`, `/api/onboard/starters`, `/api/onboard/save` | `controllers/onboard.py` | -| `routes/auth.py` | signup/login/logout/refresh/me/change-password (JWT) | `controllers/auth.py` | -| `routes/history.py` | `GET/DELETE /api/history` (per-user query history) | `controllers/history.py` | -| `routes/connections.py` | `GET/DELETE /api/connections` (recent databases) | `controllers/connections.py` | - -### Support +| `routes/query.py` | `POST /api/query`, `POST /api/query/stream`, `/api/feedback`, `/api/verify`, `/api/execute`, `/api/explain` | `controllers/query.py` — `run_query()` is the whole pipeline | +| `routes/system.py` | `GET /api/health`, `GET /api/state`, `GET /api/config`, `POST /api/config`, `POST /api/tour-complete` | `controllers/system.py` | +| `routes/database.py` | `GET /api/databases`, `POST /api/connect`, `POST /api/connect-sample`, `POST /api/disconnect`, `GET /api/schema` | `controllers/database.py` | +| `routes/onboard.py` | `POST /api/onboard/save`, `POST /api/onboard/generate-starters` | `controllers/onboard.py` | +| `routes/auth.py` | `POST /api/auth/signup`, `/login`, `/logout`, `/refresh`, `/api/auth/me`, `/api/auth/change-password`, `/api/auth/supabase-google`, email verification, forgot password | `controllers/auth.py` | +| `routes/history.py` | `GET/DELETE /api/history` (per-user query history) | `controllers/query.py` | +| `routes/connections.py` | `GET/DELETE /api/connections` (recent databases, encrypted at rest) | `controllers/database.py` | +| `routes/catalog.py` | `GET /api/catalog`, `POST /api/catalog`, `POST /api/catalog/suggest` | `controllers/catalog.py` | +| `routes/conversations.py` | `GET/POST/PATCH/DELETE /api/conversations` | `controllers/conversations.py` | +| `routes/workspaces.py` | `GET/POST/PATCH/DELETE /api/workspaces`, invites, members | `controllers/workspaces.py` | +| `routes/dashboards.py` | `GET/POST/PATCH/DELETE /api/dashboards`, panel batch update | `controllers/dashboards.py` | +| `routes/saved_queries.py` | `GET/POST/DELETE /api/saved-queries` | `controllers/dashboards.py` | +| — | Activity logs controller | `controllers/activity.py` | + +### Auth, permissions, and dependencies | File | What it does | |---|---| -| `backend/app/dependencies.py` | FastAPI dependency injection: current user from JWT, shared app-state objects. | -| `backend/app/models/api.py` | Request body shapes (pydantic): `QueryReq`, `ConfigUpdate`, `ConnectReq`, ... | -| `backend/app/models/user.py` | User model for auth. | -| `backend/app/pgdatabase/` | PostgreSQL persistence: user accounts, query history, recent connections, conversations. | -| `backend/app/logbook.py` | Append-only session log (`~/.bolodb/sessions/*.jsonl`) of every query + feedback. | +| `backend/app/dependencies.py` | FastAPI dependency injection: `get_current_user` (JWT cookie), `get_current_workspace` (X-Workspace-Id header + membership check), `get_current_db_id` (X-Db-Id header), `require_role()`, `require_permission()`, `get_db()`, `get_kb()`, `get_providers()`, `get_cfg()`. | +| `backend/app/secrets.py` | Centralized secret management: `get_jwt_secret()`, `get_supabase_jwt_secret()`, `get_resend_api_key()`, `get_frontend_url()`, `get_cookie_secure()`. | +| `backend/app/permissions.py` | RBAC permission registry — 21 fine-grained capabilities across 7 resources (members, connections, catalog, dashboards, queries, activity, workspace_management). `resolve_role_permissions()`, `has_permission()`. | +| `backend/app/ratelimit.py` | Shared slowapi `Limiter` instance with X-Forwarded-For-aware client key. | -## Frontend (`frontend/src/`) — SvelteKit +### PostgreSQL persistence (`backend/app/pgdatabase/`) | File | What it does | |---|---| -| `routes/+page.svelte` | Marketing/landing page. | -| `routes/connect/+page.svelte` → `lib/components/ConnectScreen.svelte` | Step 1: OpenRouter API key setup. Step 2: connect a database (form/URL/sample) + recent databases. | -| `routes/onboard/+page.svelte` → `OnboardScreen.svelte`, `GlossaryStep.svelte`, `StartersStep.svelte`, `ProfileStep.svelte` | First-time onboarding: confirm business terms, verify starter questions. | -| `routes/chat/+page.svelte` → `AskScreen.svelte` | The main chat screen: input, answer feed, sidebar. | -| `lib/components/AnswerCard.svelte` | One answer: restatement, results table, SQL toggle, confidence badge, verify buttons. | -| `lib/components/Sidebar.svelte` | Schema browser, history, trust meter, engine indicator (OpenRouter), Settings button. | +| `pgdatabase/engine.py` | Async SQLAlchemy engine and session factory. `get_engine()`, `DATABASE_URL` env var → `create_async_engine()`. `async_session`, `dispose_db()`. | +| `pgdatabase/knowledge.py` | `KnowledgeService` — verified Q&A storage, glossary, semantic catalog (column descriptions, metrics, joins, synonyms, value maps). All async, all PostgreSQL. `add_verified()`, `retrieve_similar()`, `get_glossary()`, `set_glossary()`, `get_catalog()`, `set_catalog()`. | +| `pgdatabase/users.py` | User CRUD: `create_user()`, `get_user_by_email()`, `update_user()`. | +| `pgdatabase/connections.py` | Recent connections with Fernet encryption at rest. `RECENT_CONNECTIONS_KEY` env var. `save_recent_connection()`, `get_recent_connections()`, `get_recent_connection_by_db_id()`. | +| `pgdatabase/conversations.py` | Conversation CRUD, scoped per workspace+user. | +| `pgdatabase/history.py` | Query history CRUD, scoped per workspace+user+db. | +| `pgdatabase/dashboards.py` | Dashboard and panel CRUD. | +| `pgdatabase/saved_queries.py` | Saved query CRUD. | +| `pgdatabase/otp.py` | One-time password/pin for invite acceptance. | +| `pgdatabase/serialization.py` | UUID helpers and serialization utilities. | + +### Support & Models (`backend/app/models/`) + +| File | What it does | +|---|---| +| `backend/app/models/api.py` | Pydantic request/response models: `QueryReq`, `ConfigUpdate`, `ConnectReq`, `FeedbackReq`, `VerifyReq`, etc. | +| `backend/app/models/workspace_api.py` | Pydantic request/response models for workspace API operations (workspace creation, updating, membership, permission matrix updates). | +| `backend/app/models/user.py` | User Pydantic model. | +| `backend/app/models/orm_user.py` | SQLAlchemy ORM User model. | +| `backend/app/models/workspace.py` | Workspace + WorkspaceMember ORM models. | +| `backend/app/models/workspace_settings.py` | WorkspaceSettings ORM model — role permissions overrides. | +| `backend/app/models/catalog.py` | VerifiedQA, Glossary, CatalogColumn, CatalogMetric, CatalogJoin, CatalogSynonym, CatalogValueMapping ORM models. | +| `backend/app/models/conversation.py` | Conversation ORM model. | +| `backend/app/models/activity.py` | WorkspaceActivityLog ORM model. | +| `backend/app/models/dashboard.py` | Dashboard + DashboardPanel ORM models. | +| `backend/app/models/saved_query.py` | SavedQuery ORM model. | +| `backend/app/models/recent_connection.py` | RecentConnection ORM model. | +| `backend/app/models/auth_token.py` | AuthToken ORM model. | +| `backend/app/models/base.py` | Base ORM model class with `_uuid7()` helper. | +| `backend/app/logbook.py` | `SessionLog` — thin class that mints per-query IDs. Durable records live in PostgreSQL (`query_history`). | +| `backend/app/services/email.py` | Email sending via Resend API. | +| `backend/app/services/email_verification.py` | Email verification + forgot-password token flows. | +| `backend/requirements.txt` | Python dependencies. | +| `backend/alembic/` | Alembic migration files for schema versioning. Automatically runs `upgrade head` on server start. | +| `backend/alembic.ini` | Alembic configuration. | +| `backend/DOCKERFILE` | Backend Docker image. | +| `backend/docker-entrypoint.sh` | Container entrypoint. | + +## Frontend (`frontend/src/`) — SvelteKit 5 + +### Routes + +| Route | Component(s) | What it does | +|---|---|---| +| `frontend/src/routes/(marketing)/+page.svelte` | — | Marketing/landing page (9 sections, GSAP+Lenis animations). Prerendered, heavily SEO-optimised (JSON-LD, OG tags, sitemap). | +| `routes/auth/callback/+page.svelte` | — | Supabase / OAuth callback page. | +| `routes/signup/+page.svelte` | `SignupScreen.svelte` | Sign up with email+password or Google OAuth. | +| `routes/login/+page.svelte` | `LoginScreen.svelte` | Login form. | +| `routes/connect/+page.svelte` | `ConnectScreen.svelte` | Connect a database (form/URL/sample) within a workspace context. | +| `routes/chat/+page.svelte` | `AskScreen.svelte`, `AppShell.svelte` | Main chat screen: input (natural language + SQL), answer feed, sidebar. | +| `routes/dashboards/+page.svelte` | — | Dashboard list view. | +| `routes/dashboards/[id]/+page.svelte` | `DashboardEditor.svelte` | Single dashboard with panels and charts. | +| `routes/dashboards/[id]/edit/+page.svelte` | `DashboardEditor.svelte` | Dashboard edit mode. | +| `routes/workspaces/+page.svelte` | — | Workspace list. | +| `routes/workspaces/setup/+page.svelte` | — | Workspace setup wizard. | +| `routes/profile/+page.svelte` | `ProfileStep.svelte` | Account profile & settings. | +| `routes/verify-email/+page.svelte` | — | Email verification page. | +| `routes/forgot-password/+page.svelte` | — | Forgot password form. | +| `routes/reset-password/+page.svelte` | — | Password reset form. | +| `routes/privacy/+page.svelte` | — | Privacy policy page. | +| `routes/terms/+page.svelte` | — | Terms of service page. | + +### Key components (`frontend/src/lib/components/`) + +| Component | What it does | +|---|---| +| `lib/components/AppShell.svelte` | Main layout wrapper — sidebar + content area. | +| `lib/components/AskScreen.svelte` | Main chat: message input, answer feed, streaming responses. | +| `lib/components/AnswerCard.svelte` | One answer: restatement, results table, chart, SQL toggle, confidence badge, verify buttons, "Save to Dashboard". | +| `lib/components/AuthChoiceModal.svelte` | Modal component for choosing authentication method (email vs Google sign-in). | +| `lib/components/ChartPanel.svelte` | ECharts visualization panel component. | +| `lib/components/DataCatalog.svelte` | Data catalog management component for glossary, metrics, and value mappings. | +| `lib/components/ExitIntentModal.svelte` | Modal overlay for capturing landing page exit intent. | +| `lib/components/Flywheel.svelte` | Interactive animated flywheel component for workflow visual presentation. | +| `lib/components/GoogleSignIn.svelte` | Google OAuth sign-in button component. | +| `lib/components/ProfileStep.svelte` | Profile setup and account details step component. | +| `lib/components/SettingsTab.svelte` | Tabbed settings sub-component for workspace configuration. | +| `lib/components/Sidebar.svelte` | Schema browser, conversation list, database switcher, trust meter, settings button. | | `lib/components/Settings.svelte` | OpenRouter model picker + API key management. | -| `lib/appState.svelte.ts` | Global client state: engine/model, connection info, trust count, navigation flow. | -| `lib/api.ts` | `apiCall()` — fetch wrapper with auth handling. | -| `lib/data.ts` | Static data: the OpenRouter provider entry, trust-level definitions, demo schema, human-friendly error translations (`humanError`). | +| `lib/components/ConnectScreen.svelte` | Database connection form (workspace-aware, split layout). | +| `lib/components/DashboardEditor.svelte` | Dashboard grid with draggable/resizable panels. | +| `lib/components/SaveQueryDialog.svelte` | Modal to save a query to a dashboard. | +| `lib/components/ProductTour.svelte` | Interactive onboarding tour overlay. | +| `lib/components/ActivityLog.svelte` | Workspace activity log viewer. | +| `lib/components/Stepper.svelte` | Step progress component for setup and onboarding flows. | +| `lib/components/Thinking.svelte` | Indicator displaying real-time query generation and reasoning progress. | +| `lib/components/TrustToast.svelte` | Toast notification component for displaying database trust status updates. | +| `lib/components/charts/ChartCard.svelte` | Card container component wrapping chart visualizations. | +| `lib/components/charts/ResultChart.svelte` | High-level chart component for rendering query results using ECharts. | +| `lib/components/charts/chartUtils.ts` | Utilities for ECharts configuration, data formatting, and color palettes. | + +### State, API, and utilities + +| File | What it does | +|---|---| +| `lib/appState.svelte.ts` | Global client state (Svelte 5 runes): user, active workspace, engine/model, connection info, trust count, navigation flow. | +| `lib/api.ts` | `apiCall()` and `streamApiCall()` — fetch wrappers with auth headers (workspace + db). Typed endpoint functions. | +| `lib/data.ts` | Static data: provider entries, trust-level definitions, human-friendly error translations (`humanError`). | | `lib/types.ts` | TypeScript interfaces shared across components. | +| `lib/i18n/` | Localized strings (`typesafe-i18n` with server-detect locale + cookie persistence). | +| `hooks.server.ts` | Server hooks: posthog analytics, locale resolution. | +| `hooks.client.ts` | Client hooks: app initialization, auth redirect. | +| `app.html` | HTML shell. | +| `app.d.ts` | TypeScript declarations. | ## Tests (`tests/`) +The repository includes a 37-file backend unit and integration test suite: + | File | Covers | |---|---| -| `tests/test_openrouter.py` | OpenRouterProvider: request shape, structured output, retries, error taxonomy, health check (fake HTTP — no network). | -| `tests/test_query_pipeline.py` | `run_query()` end-to-end with fakes: repair-on-validation-failure, repair-on-execution-error, failure paths. Proves bad SQL never reaches the database. | -| `tests/test_schema_link.py` | Table scoring, plural matching, glossary expansion, FK/junction expansion, budgets, confidence rules. | -| `tests/test_repair.py` | The repair loop in isolation: feedback content, iteration/time bounds. | -| `tests/test_sqlvalidate.py` | Static SQL validation against a schema. | -| `tests/test_structured_output.py` | The JSON output contract (`SQL_SCHEMA`) and tolerant parsing. | -| `tests/test_llm.py` | `parse_json` fence/noise stripping. | -| `tests/test_config.py` | Config defaults, old-config migration, env-var key fallback, secret never exposed. | -| `tests/test_database.py`, `test_database_controller.py` | Read-only guard, connection handling. | -| `tests/test_knowledge.py` | Knowledge base storage + similarity retrieval. | -| `tests/test_history.py` | History endpoints. | - -Run them all: `pytest tests` from the repository root (install -`backend/requirements.txt` first). +| `tests/test_activity_cleanup.py` | Retention and automated cleanup of workspace activity logs. | +| `tests/test_catalog_controller.py` | Data catalog endpoints (read, save, AI suggest). | +| `tests/test_config.py` | Config defaults, env-var key fallback, secret protection. | +| `tests/test_connection_encryption.py` | Encryption and decryption of database connection secrets at rest. | +| `tests/test_connection_rehydration.py` | Rehydration of saved connection parameters into live database engines. | +| `tests/test_database.py` | `DatabaseManager`: connection pooling, schema introspection, SSRF validation. | +| `tests/test_database_controller.py` | Database controller routes: connect, disconnect, sample database. | +| `tests/test_dependencies.py` | FastAPI dependency injection, auth validation, and header scoping. | +| `tests/test_email_verification.py` | Email verification token generation, verification endpoints, and password resets. | +| `tests/test_history.py` | Query history endpoints and CRUD persistence per user/workspace/db. | +| `tests/test_invite_codes.py` | Workspace invitation PIN/code generation, validation, and acceptance. | +| `tests/test_knowledge_service.py` | `KnowledgeService`: verified Q&A, glossary, semantic catalog, and similarity scoring. | +| `tests/test_llm.py` | `OpenRouterProvider`: JSON fence stripping, response parsing, and error mapping. | +| `tests/test_m2_permissions_adversarial.py` | Adversarial security tests for RBAC permission checks across endpoints. | +| `tests/test_onboard_controller.py` | Onboarding wizard controller routes: auto-glossary, starter generation, connection testing. | +| `tests/test_openrouter.py` | OpenRouter provider requests, structured outputs, retries, and health checks. | +| `tests/test_permissions_and_workspace_settings.py` | RBAC role evaluation and custom workspace permission matrix overrides. | +| `tests/test_pgdatabase_users.py` | User persistence and CRUD operations in PostgreSQL. | +| `tests/test_phase4_e2e_suite.py` | End-to-end integration tests for query execution, repair loop, and catalog lookup. | +| `tests/test_query_db_scoping.py` | Strict scoping of query execution to specified workspace and database IDs. | +| `tests/test_query_feedback_verify.py` | User query feedback logging and verified Q&A promotion endpoints. | +| `tests/test_query_pipeline.py` | End-to-end pipeline execution (`run_query()`), schema linking, and self-repair. | +| `tests/test_query_stream.py` | Server-Sent Events (SSE) streaming query endpoint (`POST /api/query/stream`). | +| `tests/test_repair.py` | Isolated SQL self-repair loop: feedback generation, time and iteration bounds. | +| `tests/test_sample_data.py` | Sample database seeding, schema introspection, and pre-populated Q&A verification. | +| `tests/test_schema_link.py` | Table relevance scoring, FK expansion, schema compression, and confidence badge calculation. | +| `tests/test_semantic.py` | Automatic catalog inference from schema, suggestion merging, and table catalog filtering. | +| `tests/test_sqlvalidate.py` | AST-based static SQL parsing and safety validation against live database schemas. | +| `tests/test_structured_output.py` | JSON output schema validation (`SQL_SCHEMA`) and tolerant LLM payload extraction. | +| `tests/test_supabase_auth.py` | Integration with Supabase JWT authentication and OAuth sign-in. | +| `tests/test_system_controller.py` | System diagnostics, health check, and runtime configuration routes. | +| `tests/test_workspace_api_models.py` | Pydantic validation models for workspace API requests and responses. | +| `tests/test_workspace_defaults_empirical.py` | Default settings, role assignments, and capabilities for newly created workspaces. | +| `tests/test_workspace_lifecycle.py` | Workspace lifecycle management: creation, update, member roles, and deletion. | +| `tests/test_workspace_permission_defaults.py` | Verification of initial default permission matrices across Owner, Admin, and Member roles. | +| `tests/test_workspace_settings_api.py` | API endpoints for updating workspace settings and permission overrides. | +| `tests/test_workspace_settings_empirically.py` | Empirical testing of workspace settings resolution and capability evaluation. | + +Run: `pytest tests` from the repository root (with `backend/requirements.txt` installed). ## Everything else | Path | What it is | |---|---| -| `benchmarks/` | The Spider-based evaluation of schema-linking recall and end-to-end accuracy (`benchmarks/README.md` explains how to run it). | -| `docker-compose.yml`, `nginx/`, `*/DOCKERFILE*` | The Docker deployment: backend + frontend + nginx + PostgreSQL. | -| `data/` | Drop SQLite/DuckDB files here to use them from Docker (`/app/data/...`). | -| `.github/workflows/` | CI (tests/lint), CodeQL, OpenRouter-powered PR review bots. | -| `docs/` | These documents. | +| `benchmarks/` | Spider-based evaluation of schema-linking recall and end-to-end accuracy (`benchmarks/README.md`). | +| `docker-compose.yml`, `nginx/`, `backend/DOCKERFILE`, `frontend/DOCKERFILE`, `DOCKERFILE.render` | Docker deployment: backend + frontend + nginx + PostgreSQL. Port 8080. | +| `docker-compose.dev.yml` | Dev overrides — Vite HMR, source mounts. | +| `data/` | Drop SQLite/DuckDB files here to use from Docker (`/app/data/...`). | +| `.github/workflows/` | CI (tests/lint), CodeQL, OpenRouter-powered PR review workflows. | +| `postgres/` | PostgreSQL init scripts. | +| `nginx.conf.render`, `nginx/` | Nginx configs for production and Coolify. | +| `docs/` | Comprehensive user and developer documentation. | +| `memory/` | Product requirements and design docs. | diff --git a/docs/08-troubleshooting.md b/docs/08-troubleshooting.md index 855baa24..c833bafc 100644 --- a/docs/08-troubleshooting.md +++ b/docs/08-troubleshooting.md @@ -1,173 +1,62 @@ # 8. Troubleshooting — when something goes wrong -This chapter is organised by **symptom**. For each: what it probably means, -where to look in the code, and how to fix it. The first section needs no -technical knowledge; the second is for whoever maintains the deployment. +This chapter is organized by **symptom**. For each issue, it covers the root cause, where to check in the codebase, and how to resolve it. --- -## Part A — For users +## Part A — For Users -### "No OpenRouter API key is configured…" +### "No OpenRouter API Key configured" +- **Cause**: The server is missing the `OPENROUTER_API_KEY` environment variable. +- **Fix**: Set `OPENROUTER_API_KEY` in your `.env` file or environment and restart the backend server. -The AI has no key to work with. -**Fix:** open **Settings** (gear icon in the sidebar) → paste a key → -Save. Get a free key at https://aistudio.google.com/app/api-keys. +### "Authentication Required / 401 Unauthorized" +- **Cause**: Your session JWT cookie has expired or is invalid. +- **Fix**: Log out and log back in at `/login`. -### "…the API key is malformed or not valid" / "…invalid or expired" / "…may lack permission" +### "Workspace Access Denied / 403 Forbidden" +- **Cause**: You are attempting to perform an action (e.g., connect database, edit catalog) without the required role permission in the active workspace. +- **Fix**: Contact your workspace Owner or Admin to adjust your role in Workspace Settings (`/workspaces/[id]/settings`). -The key exists but Google refused it. The message tells you which flavour: +### "Connection Refused / Database Unreachable" +- **Cause**: Network connection to the target database failed, or SSRF protection blocked the host (`backend/app/database.py`). +- **Fix**: Ensure the database host is not loopback (`localhost`, `127.0.0.1`) when running inside Docker, and verify firewall rules allow traffic. -- **"rejected the request as invalid"** (HTTP 400) — usually a typo or a - truncated key. Re-copy it carefully. -- **"was not accepted — invalid or expired"** (HTTP 401) — the key was - revoked or expired. Generate a fresh one. -- **"denied access — may lack permission"** (HTTP 403) — the key is real but - restricted: the Generative Language API isn't enabled for its project, the - key has API restrictions, or your region/Workspace blocks access. Check the - key's settings in Google AI Studio. +### "Could not form a query — try rephrasing" +- **Cause**: The AI attempted up to 3 repair loops but could not generate valid SQL for your schema. +- **Fix**: Rephrase the question using exact column or table names, or add missing business definitions to the Data Catalog (`/api/catalog`). -**Fix:** generate a fresh key in Google AI Studio and re-enter it in -Settings. The moment a valid key is saved, everything works — no restart -needed (saving triggers `providers.reconfigure()` in -`backend/app/controllers/system.py`). - -### "The AI service is busy right now — please try again in a moment." - -Google's API returned a rate-limit or temporary server error, and BoloDB's -automatic retries (3 attempts with backoff) didn't get through. -**Fix:** wait a minute and re-ask. If it persists for hours, check Google's -status page or your quota in AI Studio (free-tier quotas are per-day). - -### "Could not form a query - try rephrasing" / an answer with Low confidence and an error - -The AI tried up to 3 times (including automatic self-repair) and still -couldn't produce SQL your database accepts. -**Fix:** rephrase with more specifics — name the thing you want counted or -summed the way it's called in your business. If a term keeps failing, add it -to the glossary during onboarding or verify a correct answer once — both -directly teach the system (see [chapter 6](06-learning-trust-and-confidence.md)). - -### "no matching rows - the question may not match your data" - -The query ran fine but found nothing. Often the filter is subtly off — e.g. -a date range with no data, or a category spelled differently in your data. -**Fix:** open the SQL toggle or hit Explain and check the filter values; ask -again naming an exact value you know exists. - -### The answer is wrong, but confidently so - -No validator can catch "runs fine, answers the wrong question". Your tools: -read the **restatement** and **assumptions**, hit **Explain**, and click -**"Something's wrong"** — that feedback is logged and keeps the wrong answer -out of the knowledge base. +--- -### "Only read-only SELECT queries are allowed" +## Part B — For Developers and Maintainers -You (or the AI) tried something that would modify data. That's the safety -guard working as intended — BoloDB never writes to your database. Code: -`backend/app/database.py` → `_readonly_violation()`. +### 1. Database Connection Encryption Errors +- **Symptom**: `Fernet InvalidToken` or failure decrypting stored connection credentials. +- **Root Cause**: `RECENT_CONNECTIONS_KEY` env variable changed or is not set properly across deployments. +- **Fix**: Ensure `RECENT_CONNECTIONS_KEY` remains static in your production `.env` file (`backend/app/pgdatabase/connections.py`). -### Can't connect to the database +### 2. Rate Limiting Limits Triggered (`slowapi`) +- **Symptom**: HTTP 429 Too Many Requests response. +- **Root Cause**: Client exceeded endpoint rate limits. +- **Fix**: Inspect `backend/app/ratelimit.py`. Ensure proxy headers (`X-Forwarded-For`) are forwarded correctly by Nginx/Traefik so all users don't share a single IP limit. -The connect screen translates the raw error into a friendly sentence -(`frontend/src/lib/data.ts` → `humanError()`): wrong password, unreachable -host, missing file, etc. If running in Docker and connecting to a database on -your own machine, note that `localhost` is automatically rewritten to reach -the host (`backend/app/database.py` → `connect()`). +### 3. OpenRouter API Errors & Taxonomy +- **Symptom**: `LLMError` in backend logs. +- **Code Reference**: `backend/app/llm.py` → `OpenRouterProvider`. +- **Fix**: + - For `LLMError` (authentication issue): Verify `OPENROUTER_API_KEY` validity at https://openrouter.ai/keys. + - For `LLMError` (context window / budget issue): Check `schema_link.py` budgets — table count budget may need adjustment. ---- +### 4. Stale-Chunk Streaming Interruptions +- **Symptom**: Chat interface hangs or streaming stops abruptly during query generation. +- **Code Reference**: `frontend/src/lib/api.ts` → `streamApiCall()`. +- **Fix**: Verify Nginx reverse proxy buffer settings (`proxy_buffering off;`) to allow SSE chunks to pass through immediately without buffering. -## Part B — For developers / maintainers - -### Golden rule: check the logs first - -- Every LLM failure logs a `WARNING` with the technical detail (the - user-facing message is intentionally vaguer). Search the backend log for - `LLM error during run_query` (from `controllers/query.py`) — the `detail` - field of the `LLMError` carries the HTTP status + response snippet from - Google (`backend/app/llm.py` → `complete()`). -- Every question and its outcome is appended to - `~/.bolodb/sessions/session-.jsonl` — question, SQL, - confidence, error, feedback verdicts, plus `tables_used` (exactly which - tables the AI saw, including schema-retry additions) and `attempts` (how - many generation rounds were needed). This is your replay log; high - `attempts` values are the production signal of linking misses. - -### Diagnosing a bad answer, step by step - -1. **Which tables did the AI see?** The `/api/query` response includes - `tables_used`. If the right table isn't there, the problem is schema - linking — see `link_relevant_tables()` scoring in - `backend/app/schema_link.py`, and enable DEBUG logging to see per-table - scores. Common cause: table/column names that share no words with how - users talk → fix by adding glossary terms. -2. **What was the AI actually told?** The full prompt is assembled in - `build_sql_system_prompt()` (`backend/app/llm.py`). Log or breakpoint - there to inspect it. -3. **Did self-repair fire?** Check `attempts` and `repaired` in the - response. `attempts: 3` + an `execution_error` means the model never - converged — usually a schema-linking miss (step 1) or a dialect quirk - (see `_DIALECT_HINTS` in `llm.py`). -4. **Was the SQL rejected before execution?** Validation errors ("Unknown - column …") come from `backend/app/sqlvalidate.py`; read the conservative - design note at the top of that file before "fixing" it. - -### The schema looks stale (renamed/added columns not showing) - -Schema introspection is cached per connection -(`backend/app/database.py` → `get_schema()`, `_schema_cache`). -Reconnecting the database refreshes it (or call `get_schema(user_id, -refresh=True)` where appropriate). - -### Everything AI-related fails instantly with no network calls - -`OpenRouterProvider.complete()` fails fast when `api_key` is empty — before any -HTTP. Check `GET /api/health`: it reports -`{"provider": {"name": "openrouter", "ok": false, "error": "No OpenRouter API key configured"}}` -in that state. Key resolution order: value saved in Settings beats the -`OPENROUTER_API_KEY` env var (`backend/app/config.py` → `load_config()`). - -### Google returned 404 for the model - -Model names get retired. The allowed list lives in one place — -`ALLOWED_MODELS` in `backend/app/config.py` — and the Settings dropdown in -`frontend/src/lib/components/Settings.svelte` mirrors it. Update both when -Google renames models. - -### The saved API key stopped working after files were moved/restored - -The key in `config.json` is encrypted with the per-install secret in -`~/.bolodb/.secret` (`backend/app/config.py` → `_fernet()`). If `.secret` is -deleted or the config is copied to another machine without it, the stored key -can no longer be decrypted — OpenRouter will reject it as invalid. -**Fix:** re-enter the key in Settings (it gets re-encrypted with the current -secret). When backing up `~/.bolodb`, keep `config.json` and `.secret` -together. - -### Old config from the multi-provider era - -Config files written before the OpenRouter-only switch (provider `ollama` / -`claude` / `openai` / `groq`) are migrated silently on load — -`load_config()` coerces provider+model and drops old vendor keys. Covered by -`tests/test_config.py::test_load_config_migrates_old_provider_config`. - -### Running the test suite +### 5. Running the Backend Test Suite ```bash pip install -r backend/requirements.txt -pytest tests -q +pytest tests -v ``` -All AI tests use a fake HTTP client (`tests/test_openrouter.py`) — the suite -needs **no network and no API key**. If you touched the pipeline, the most -valuable file to keep green is `tests/test_query_pipeline.py`. - -### Quick health checklist - -| Check | Expect | -|---|---| -| `GET /api/health` | `provider.name == "openrouter"`, `ok: true` when a valid key is set | -| `GET /api/state` | `config.api_keys_set.openrouter == "set"`, connection info | -| `~/.bolodb/config.json` | `{"provider": "openrouter", "model": "openrouter-2.5-…", "api_keys": {"openrouter": "…"}}` | -| Backend log | no repeated `OpenRouter transient HTTP …` warnings (would mean quota pressure) | +All unit tests use mock/fake OpenRouter providers and isolated database fixtures (`tests/test_openrouter.py`, `tests/test_query_pipeline.py`). diff --git a/docs/09-cost-optimisation.md b/docs/09-cost-optimisation.md index f714f288..49f4f444 100644 --- a/docs/09-cost-optimisation.md +++ b/docs/09-cost-optimisation.md @@ -1,76 +1,46 @@ -# 9. Cost optimisation — good answers without burning tokens +# 9. Cost Optimization — High Accuracy without Burning Tokens -Design goal, in order: **(1) the user's experience is never compromised, -(2) every avoidable token is avoided.** It turns out these rarely conflict — -most of what makes prompts cheaper (less irrelevant schema, structured -output, verified examples) also makes answers *better*. +BoloDB follows a clear design rule: **(1) the user's experience is never compromised, (2) every avoidable token is avoided.** -## Where AI cost comes from +Because OpenRouter bills per token, optimizing prompt construction and output format directly reduces operational cost while simultaneously improving model accuracy. -OpenRouter bills per token, input and output separately. For BoloDB, a -question's cost ≈ +--- -```text -(prompt: instructions + schema + glossary + examples + context) ← input tokens -+ (the model's reply: SQL + restatement + assumptions) ← output tokens -+ (invisible "thinking" tokens, if enabled) ← output tokens -× (number of attempts, if self-repair fires) -``` +## 1. Where AI Cost Comes From -Every mechanism below trims one of those factors. Each row names the code -that implements it. +Query generation cost is governed by: -## The mechanisms +$$\text{Cost} = \left( \text{Tokens}_{\text{prompt}} + \text{Tokens}_{\text{output}} \right) \times \text{Attempts}_{\text{repair}}$$ -| # | Mechanism | Effect | Code | +Where: +- **Prompt Tokens**: System instructions + compact schema + business glossary + semantic catalog rules + verified worked examples + conversation turns. +- **Output Tokens**: Structured JSON containing SQL, plain-English restatement, assumptions, and chart specification. +- **Attempts**: Number of self-repair retries (1 attempt for clean executions, up to 3 for repaired queries). + +--- + +## 2. Optimization Mechanisms + +| # | Mechanism | Effect | Implementation File | |---|---|---|---| -| 1 | **Schema linking** — only relevant tables are sent, not the whole database | Biggest single saving on input tokens; also *improves* accuracy | `backend/app/schema_link.py` → `link_relevant_tables()` ([chapter 4](04-schema-linking.md)) | -| 2 | **Compact schema rendering** — one dense line per table instead of CREATE TABLE dumps | ~3–5× fewer tokens per table | `schema_link.py` → `compact_schema()` | -| 3 | **Structured output** — the model is constrained to a JSON schema; it cannot ramble, apologise, or wrap answers in prose | Output tokens = exactly the fields we parse | `backend/app/llm.py` → `responseSchema` in `OpenRouterProvider._build_body()` | -| 4 | **Thinking off for simple tasks** — glossary, starters and Explain don't need reasoning tokens | Removes the invisible thinking cost on 3 of 4 operations | `llm.py` → `thinking_budget=0` call sites | -| 5 | **Thinking left dynamic for SQL generation** — the model spends reasoning only when the question is hard | Pay for reasoning only where it buys accuracy (deliberate: experience > cost) | `llm.py` → `generate_sql()` (no `thinking_budget` override) | -| 6 | **Output caps** — `maxOutputTokens: 4096`, temperature 0.1 | Bounds worst-case output cost | `llm.py` → `_build_body()` | -| 7 | **Bounded self-repair** — at most 3 attempts, 60-second budget | A stubborn failure can't loop forever | `backend/app/controllers/query.py` → `_MAX_ATTEMPTS`, `_MAX_SECONDS` | -| 8 | **Retry with backoff, never blindly** — only transient statuses retry; a bad API key fails once, immediately | No token/HTTP waste on hopeless calls | `llm.py` → `_RETRYABLE_STATUS`, `complete()` | -| 9 | **Verified-answer reuse** — similar past answers are injected as examples, so the model converges on attempt 1 instead of needing repairs | Fewer repair rounds over time; the system gets *cheaper as it's used* | `backend/app/knowledge.py` → `retrieve_similar()` | -| 10 | **Per-model budgets** — cheaper models get smaller prompts (fewer tables/samples/examples) | Cost scales with the model tier you chose | `schema_link.py` → `model_budget()` | -| 11 | **Schema introspection cache** — your database is profiled once per connection, not per question | No repeated DB scans; instant prompts | `backend/app/database.py` → `get_schema()` cache | -| 12 | **Two-stage linking pays for itself** — the shortlist pre-pass (big schemas only, 30+ tables) is a names-only prompt with thinking off; picking the right tables up-front avoids repair rounds that cost far more | One tiny call replaces 1–2 full retries | `llm.py` → `shortlist_tables()`, threshold in `controllers/query.py` | -| 13 | **Enrichment cap** — sample rows / known values are collected for at most 40 tables (largest first); structure is still collected for all | Bounded introspection cost on huge databases | `database.py` → `ENRICH_MAX` in `get_schema()` | - -## The one knob you control: the model - -In Settings (persisted via `backend/app/config.py`): - -| Model | Relative cost | Guidance | -|---|---|---| -| `deepseek-v4-flash` | lowest | Small schema, straightforward questions, high volume | -| `deepseek-v4-flash` (default) | low | The right answer for almost everyone | -| `deepseek-v4-flash` | highest | Big schemas, complex analytical questions | - -OpenRouter's API also has a **free tier** — for evaluation and light usage the -cost is simply zero. Current pricing/quotas: https://ai.google.dev/pricing. - -## What we deliberately do NOT do - -- **We don't cut schema below what the question needs.** FK parents and - junction tables are always included even past the table budget - (`_FK_EXTRA_SLOTS`) — a cheap prompt that produces a broken JOIN costs - more after one repair round than the tokens it saved. -- **We don't disable thinking for SQL generation.** Hard questions benefit; - dynamic thinking means simple questions barely spend any. -- **We don't retry non-transient errors.** An invalid key or a blocked - prompt fails fast with a clear message ([chapter 3](03-the-ai-layer-openrouter.md)). - -## Ideas for later (not implemented) - -- **Context caching** for the schema block — OpenRouter can cache repeated - prompt prefixes at a discount; worth it once per-database question volume - is high. Would slot into `OpenRouterProvider._build_body()`. -- **Semantic retrieval** (embeddings) instead of word-overlap for verified - answers — better example reuse at the same token cost. Would replace - `_similarity()` in `knowledge.py`. - -A [Spider](https://github.com/taoyds/spider)-based benchmark for measuring -linking recall and execution accuracy before shipping prompt changes now -exists — see `benchmarks/README.md`. +| 1 | **Schema Linking** | Sends only relevant tables instead of dumping full database schema | `backend/app/schema_link.py` → `link_relevant_tables()` | +| 2 | **Compact Schema Rendering** | Compresses table definitions into 1 line per table | `schema_link.py` → `compact_schema()` | +| 3 | **Structured Output Contract** | Forces exact JSON schema (`SQL_SCHEMA`); eliminates rambling prose | `backend/app/llm.py` → `OpenRouterProvider` | +| 4 | **Streaming Responses** | Streams chunks in real time, giving immediate user feedback without polling overhead | `backend/app/routes/query.py` & `frontend/src/lib/api.ts` | +| 5 | **Bounded Self-Repair** | Enforces max 3 repair attempts and 60-second execution deadline | `backend/app/controllers/query.py` → `_MAX_ATTEMPTS` | +| 6 | **PostgreSQL Verified Answer Reuse** | Injects verified Q&A pairs as few-shot prompt examples, maximizing first-pass success | `backend/app/pgdatabase/knowledge.py` → `retrieve_similar()`, `set_glossary()`, `set_catalog()` | +| 7 | **Two-Stage Linking for Large Schemas** | Pre-filters candidate tables via cheap names-only shortlist on schemas with 30+ tables | `llm.py` → `shortlist_tables()` | +| 8 | **Schema Introspection Caching** | Caches table introspections per workspace database connection | `backend/app/database.py` → `get_schema()` | + +--- + +## 3. Deliberate Trade-Offs + +- **No Over-Trimming**: Foreign key parents and junction tables are retained even beyond the initial table budget (`_FK_EXTRA_SLOTS`) to avoid broken SQL JOINs. +- **Fail Fast**: Non-retryable errors (e.g., authentication or permissions) fail immediately to avoid wasting tokens. + +--- + +## 4. Benchmarking Cost and Accuracy + +The benchmark suite (`benchmarks/README.md`) measures schema linking recall, execution accuracy, and average token consumption against standard benchmark datasets like Spider. diff --git a/docs/10-auth-and-workspaces.md b/docs/10-auth-and-workspaces.md new file mode 100644 index 00000000..be38ffc4 --- /dev/null +++ b/docs/10-auth-and-workspaces.md @@ -0,0 +1,84 @@ +# 10. Authentication and Workspaces + +BoloDB is built as a multi-tenant web application. All data—connections, query history, saved queries, dashboards, and the knowledge base—is isolated per **workspace**. + +--- + +## 1. Authentication + +Authentication is handled via JWT tokens issued by the backend server (`backend/app/routes/auth.py` and `backend/app/controllers/auth.py`). + +### Auth Methods +- **Email + Password**: Users register via `/api/auth/signup` and authenticate via `/api/auth/login`. Password hashes are stored securely using passlib/bcrypt. +- **Google OAuth**: Google authentication flow produces a verified OAuth token handled at `/api/auth/supabase-google`. +- **Email Verification & Password Reset**: Handled via token generation in `backend/app/services/email_verification.py` and dispatched using the Resend API service (`backend/app/services/email.py`). + +### Token Management & Cookies +- Upon successful authentication, the server sets secure HTTP-only cookies containing the JWT access and refresh tokens. +- `backend/app/dependencies.py` exposes `get_current_user`, which validates the token on incoming API calls. +- Centralized secrets management (`backend/app/secrets.py`) retrieves secrets like `JWT_SECRET`, `RESEND_API_KEY`, and `FRONTEND_URL` from the environment. + +--- + +## 2. Workspace Multi-Tenancy + +A **Workspace** is the primary boundary of data isolation. Each workspace contains: +- **Database Connections**: Connections registered within the workspace context (encrypted at rest). +- **Knowledge Base**: Verified Q&A, business glossary, and semantic layer entries (`backend/app/pgdatabase/knowledge.py`). +- **Dashboards & Saved Queries**: Custom dashboards with ECharts visual panels. +- **Activity Log**: Workspace audit log (`backend/app/models/activity.py`). + +### Context Propagation +Client requests include `X-Workspace-Id` and `X-Db-Id` headers. The FastAPI dependencies `get_current_workspace()` and `get_current_db_id()` in `backend/app/dependencies.py` ensure: +1. The requested workspace exists and the authenticated user is an active member of that workspace. +2. The active target database connection context (`X-Db-Id`) is validated and propagated to database services and controllers. + +--- + +## 3. Role-Based Access Control (RBAC) + +BoloDB uses a granular permission registry defined in `backend/app/permissions.py`. + +### Roles +Every workspace member holds one of three roles: +1. **Owner**: Full administrative control over workspace settings, members, permissions, and database connections. +2. **Admin**: Can manage connections, edit the semantic catalog, invite members, and build dashboards. +3. **Member**: Can execute queries, view dashboards, view schemas, and save queries. + +### Permission Registry Matrix + +The system defines 21 fine-grained capabilities across 7 resources: + +| Resource | Capability Key | Description | Default Owner | Default Admin | Default Member | +|---|---|---|---|---|---| +| **members** | `members.view` | View workspace member list and member details | Yes | Yes | Yes | +| | `members.invite` | Invite new members to join the workspace | Yes | Yes | No | +| | `members.update_role` | Change roles of workspace members | Yes | Yes | No | +| | `members.remove` | Remove members from the workspace | Yes | Yes | No | +| **connections** | `connections.view` | View configured database connections | Yes | Yes | Yes | +| | `connections.manage` | Create, edit, or delete database connections | Yes | Yes | No | +| | `connections.view_schema` | View schema and metadata for database connections | Yes | Yes | Yes | +| **catalog** | `catalog.view` | View data catalog, verified Q&A, and metrics | Yes | Yes | Yes | +| | `catalog.manage` | Create or update data catalog definitions | Yes | Yes | No | +| **dashboards** | `dashboards.view` | View dashboards and visualization panels | Yes | Yes | Yes | +| | `dashboards.create` | Create new dashboards and panels | Yes | Yes | No | +| | `dashboards.manage` | Edit or delete existing dashboards | Yes | Yes | No | +| **queries** | `queries.execute` | Execute natural language and SQL queries | Yes | Yes | Yes | +| | `queries.explain` | Generate query explanations and execution plans | Yes | Yes | Yes | +| | `queries.save` | Save queries for workspace access | Yes | Yes | Yes | +| | `queries.delete_saved` | Delete saved queries | Yes | Yes | No | +| **activity** | `activity.view` | View workspace activity logs | Yes | Yes | No | +| | `activity.export` | Export workspace activity logs | Yes | Yes | No | +| **workspace_management** | `workspace.view` | View workspace details and configuration | Yes | Yes | Yes | +| | `workspace.update` | Update workspace basic profile details | Yes | Yes | No | +| | `workspace.settings` | Manage workspace defaults and role permission matrix | Yes | Yes | No | + +Custom role overrides can be saved per workspace in `WorkspaceSettings` (`backend/app/models/workspace_settings.py`). + +--- + +## 4. Member Invites & Onboarding Flow + +1. Workspace owners/admins generate an invitation link or email invite. +2. An invitation pin/token is created (`backend/app/pgdatabase/otp.py`). +3. Invited users accept the invitation during registration or via their workspace switcher in the app. diff --git a/docs/11-dashboards-and-charts.md b/docs/11-dashboards-and-charts.md new file mode 100644 index 00000000..ee69d13c --- /dev/null +++ b/docs/11-dashboards-and-charts.md @@ -0,0 +1,67 @@ +# 11. Dashboards and Charts + +BoloDB provides built-in visualization and dashboarding capabilities, allowing users to save queries and compile them into interactive dashboards powered by **Apache ECharts**. + +--- + +## 1. Visualization & Chart Type Inference + +When BoloDB executes a natural language query, the OpenRouter AI model infers the best visualization format alongside generating the SQL (`backend/app/llm.py`). + +### Supported Chart Types +- **`bar`**: Ideal for categorical comparisons (e.g., revenue per category). +- **`line`**: Ideal for continuous time-series data (e.g., daily active users). +- **`area`**: Ideal for cumulative time-series data. +- **`pie`**: Ideal for proportional breakdown of a single metric. +- **`number`**: Ideal for single scalar metric KPIs (e.g., total sales count). +- **`table`**: Default fallback tabular view for multi-column relational data. + +### Chart Specification Schema +The AI returns a structured chart specification object as part of `SQL_SCHEMA`: + +```json +{ + "chart": { + "type": "bar", + "x_column": "category_name", + "y_columns": ["total_revenue"], + "title": "Revenue by Product Category" + } +} +``` + +The frontend components `frontend/src/lib/components/ChartPanel.svelte`, `frontend/src/lib/components/charts/ResultChart.svelte`, and `frontend/src/lib/components/charts/ChartCard.svelte` consume this specification (using helper utilities from `frontend/src/lib/components/charts/chartUtils.ts`) to dynamic initialize ECharts instances tailored to the data shape. + +--- + +## 2. Saved Queries + +Users can save any query result directly from an Answer Card (`frontend/src/lib/components/AnswerCard.svelte`) using `frontend/src/lib/components/SaveQueryDialog.svelte`. + +### Backend Implementation +- **Route**: `backend/app/routes/saved_queries.py` +- **Database Service**: `backend/app/pgdatabase/saved_queries.py` +- **Data Model**: `backend/app/models/saved_query.py` (`SavedQuery`) + +Saved queries capture: +- SQL query string +- Natural language question & restatement +- Selected chart type and configuration parameters +- Workspace and Database ID scoping + +--- + +## 3. Dashboards & Panels + +Dashboards aggregate multiple saved queries or custom query panels into grid layouts. + +### Architecture & Endpoints +- **Routes**: `backend/app/routes/dashboards.py` +- **Controller**: `backend/app/controllers/dashboards.py` +- **Database Service**: `backend/app/pgdatabase/dashboards.py` +- **Models**: `backend/app/models/dashboard.py` (`Dashboard`, `DashboardPanel`) + +### Key Operations +1. **Dashboard CRUD**: Create, read, update title/description, and delete workspace dashboards. +2. **Panel Batch Update**: `POST /api/dashboards/{id}/panels/batch` allows layout changes (grid coordinates `x, y, w, h`) to be saved atomically as users resize or drag panels in `frontend/src/lib/components/DashboardEditor.svelte`. +3. **Live Query Execution**: Panels execute their stored SQL against the current workspace database connection on load or manual refresh. diff --git a/docs/12-semantic-layer.md b/docs/12-semantic-layer.md new file mode 100644 index 00000000..466b65ff --- /dev/null +++ b/docs/12-semantic-layer.md @@ -0,0 +1,81 @@ +# 12. The Semantic Layer + +The **Semantic Layer** bridges the gap between raw database schemas and business domain concepts. It ensures that business terminology, complex metrics, and implicit table relationships are correctly interpreted by the AI during SQL generation. + +--- + +## 1. Core Concepts & Data Models + +The semantic layer is stored in PostgreSQL via `KnowledgeService` (`set_catalog` / `get_catalog` in `backend/app/pgdatabase/knowledge.py`) and modeled in `backend/app/models/catalog.py`. + +### 1. Metric Definitions (`CatalogMetric`) +Pre-defined SQL expressions for business metrics so the AI doesn't miscalculate key KPIs. +- *Example*: `ARR = SUM(annual_contract_value) WHERE status = 'active'` + +### 2. Join Paths (`CatalogJoin`) +Explicit join rules between tables when foreign keys are missing or when non-standard join conditions are required. +- *Example*: `orders.customer_id = customers.id AND customers.is_deleted = false` + +### 3. Synonyms (`CatalogSynonym`) +Mapping business slang or alternative names to actual database table or column names. +- *Example*: `"client"` → `customers`, `"gross sales"` → `orders.total_amount` + +### 4. Value Mappings (`CatalogValueMapping`) +Translating user concepts into specific column value filters. +- *Example*: `"VIP customers"` → `customers.tier = 'VIP'` or `customers.total_spend > 10000` + +### 5. Column Descriptions (`CatalogColumn`) +Human-readable context and notes attached to specific database columns. + +--- + +## 2. Automatic Inference & Schema Enrichment + +BoloDB automatically assists in building the semantic layer using `backend/app/semantic.py` and `backend/app/llm.py`: + +1. **`suggest_from_schema(schema)`**: Inspects foreign keys, column names, and sample values to propose join paths and value mappings deterministically. +2. **`suggest_catalog(provider, schema_text)`**: Uses LLM enrichment (`backend/app/llm.py`) to generate descriptions, metrics, synonyms, and business labels from schema text. +3. **`merge_catalog_suggestions(existing, suggested)`**: Merges automatic schema suggestions with human overrides. +4. **`filter_catalog(catalog, linked_tables)`**: Scopes the semantic catalog so only relevant metrics, joins, and synonyms are included in the prompt for the tables selected by schema linking. + +--- + +## 3. How the AI Uses the Semantic Layer + +During Step 1 and 2 of the question pipeline (`backend/app/controllers/query.py`), BoloDB fetches the workspace's semantic catalog for the active database connection: + +```text +Question ──▶ Schema Linking (picks relevant tables) + │ + ▼ + filter_catalog() (filters metrics/joins for selected tables) + │ + ▼ + Prompt Assembly (appends Semantic Rules block to OpenRouter prompt) + │ + ▼ + LLM generates accurate, domain-aware SQL +``` + +### Prompt Integration Example +In `backend/app/llm.py`, the semantic context is injected directly into the prompt: + +```text +[SEMANTIC RULES & METRICS] +- When user asks for "ARR", use expression: SUM(annual_contract_value) WHERE status = 'active' +- Always join orders to customers using: orders.customer_id = customers.id +- "VIP customer" means: tier = 'VIP' +``` + +--- + +## 4. Semantic Catalog Management + +Workspace admins and owners can view, edit, and AI-suggest the semantic layer via the Data Catalog endpoints (`GET /api/catalog`, `POST /api/catalog`, `POST /api/catalog/suggest`), handled by `backend/app/routes/catalog.py` and `backend/app/controllers/catalog.py`. + +### Request Context Headers +All catalog endpoints require workspace and database context propagation headers: +- `X-Workspace-Id`: Target workspace identifier. +- `X-Db-Id`: Target database connection identifier. + +Saving catalog entries executes `set_catalog()` on `KnowledgeService` (`backend/app/pgdatabase/knowledge.py`), replacing or updating stored entries for the active workspace and database. Changes take effect immediately for all subsequent queries. diff --git a/docs/README.md b/docs/README.md index 5bbd5738..e3c716e6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,44 +1,31 @@ # BoloDB Documentation -Welcome! These docs explain **what BoloDB does, how it does it, and where in -the code each piece lives**. They are written so that: +Welcome! These docs explain **what BoloDB does, how it does it, and where in the code each piece lives**. -- a **non-technical reader** can follow every chapter top-to-bottom and - understand the product, and -- a **developer** (or a curious founder) can jump from any paragraph straight - into the code, because every mechanism is annotated with the file — and - usually the exact function — where it happens. +- A **non-technical reader** can follow every chapter top-to-bottom and understand the product architecture. +- A **developer** can jump from any paragraph straight into the codebase, annotated with file and function references. -## How to read these docs - -If you are new, read chapters 1 → 2 first. After that, each chapter stands on -its own. Code pointers look like this: - -> `backend/app/llm.py` → `generate_sql()` - -which means: open that file, find that function. +--- ## Chapters | # | Chapter | What it answers | |---|---------|-----------------| -| 1 | [What is BoloDB?](01-what-is-bolodb.md) | The product in plain language. What happens when you connect a database and ask a question. | -| 2 | [How a question becomes an answer](02-how-a-question-becomes-an-answer.md) | The full pipeline, step by step, with the exact code location for every step. **The most important chapter.** | -| 3 | [The AI layer (OpenRouter)](03-the-ai-layer-openrouter.md) | Everything about the AI: what is sent to Google, which models we use, API keys, retries, errors, and how to add another AI vendor later. | -| 4 | [Schema linking — choosing tables](04-schema-linking.md) | How BoloDB decides which tables the AI gets to see, and why that makes answers cheaper *and* more accurate. | -| 5 | [Safety, validation and self-repair](05-safety-validation-and-self-repair.md) | The three safety nets: read-only enforcement, SQL validation before execution, and the automatic repair loop. | -| 6 | [Learning, trust and confidence](06-learning-trust-and-confidence.md) | How BoloDB learns from your confirmations, and how the High/Medium/Low confidence badge is computed. | -| 7 | [File map](07-file-map.md) | Every file in the repository with a one-line description. Use it as an index. | -| 8 | [Troubleshooting](08-troubleshooting.md) | "Something is wrong" → what it probably is → where to look in the code → how to fix it. | -| 9 | [Cost optimisation](09-cost-optimisation.md) | Where AI costs come from and every mechanism BoloDB uses to keep them low without hurting answer quality. | - -## The one-paragraph summary - -BoloDB lets anyone ask questions about a SQL database in plain English. When -you ask a question, BoloDB picks the handful of tables that matter -(chapter 4), sends the question plus that trimmed schema to OpenRouter -(chapter 3), checks the SQL that comes back before running it and -automatically fixes it if it's broken (chapter 5), runs it **read-only** -against your database, and shows the results with a plain-English restatement -and a confidence level (chapter 6). Every answer you confirm as correct is -remembered and makes future answers better. +| 1 | [What is BoloDB?](01-what-is-bolodb.md) | Multi-tenant product introduction, security model, and 30-second architecture overview. | +| 2 | [How a question becomes an answer](02-how-a-question-becomes-an-answer.md) | The full execution pipeline step by step. **The most important chapter.** | +| 3 | [The AI layer (OpenRouter)](03-the-ai-layer-openrouter.md) | OpenRouter provider (`deepseek/deepseek-v4-flash`), structured JSON outputs, streaming SSE, and prompt assembly. | +| 4 | [Schema linking — choosing tables](04-schema-linking.md) | How table scoring, FK expansion, and two-stage linking optimize context windows. | +| 5 | [Safety, validation and self-repair](05-safety-validation-and-self-repair.md) | SSRF defense, read-only AST safety, static SQL validation, statement timeouts, and self-repair. | +| 6 | [Learning, trust and confidence](06-learning-trust-and-confidence.md) | PostgreSQL knowledge base, Jaccard/sequence similarity, and confidence badges. | +| 7 | [File map](07-file-map.md) | Complete codebase index mapping backend, frontend, tests, and configurations. | +| 8 | [Troubleshooting](08-troubleshooting.md) | Symptom-based troubleshooting guide for users and maintainers. | +| 9 | [Cost optimisation](09-cost-optimisation.md) | Token optimization mechanisms, structured output contracts, and budget controls. | +| 10 | [Authentication and Workspaces](10-auth-and-workspaces.md) | JWT auth, Google OAuth, workspace multi-tenancy, and RBAC permission matrix. | +| 11 | [Dashboards and Charts](11-dashboards-and-charts.md) | Interactive ECharts visualizations, saved queries, and dashboard panel CRUD. | +| 12 | [The Semantic Layer](12-semantic-layer.md) | Business metrics, join rules, synonyms, value mappings, and domain context injection. | + +--- + +## The Summary + +BoloDB lets anyone talk to their database in plain English. Built as a multi-tenant FastAPI + SvelteKit web application, it translates questions into SQL using OpenRouter AI (`deepseek/deepseek-v4-flash`), validates and executes queries safely in read-only mode, and renders interactive ECharts visualizations and restatements. Every request is scoped to a workspace and database connection via `X-Workspace-Id` and `X-Db-Id` headers. Every confirmed query enriches the workspace's PostgreSQL knowledge base to improve future accuracy. diff --git a/frontend/DOCKERFILE b/frontend/DOCKERFILE index adf9cb5a..79fa4c63 100644 --- a/frontend/DOCKERFILE +++ b/frontend/DOCKERFILE @@ -10,7 +10,7 @@ WORKDIR /app # ── 1. Install dependencies (cached layer) ── COPY package.json package-lock.json* ./ -RUN npm ci +RUN npm install # ── 2. Copy source & build ── COPY . . diff --git a/frontend/DOCKERFILE.dev b/frontend/DOCKERFILE.dev index b9ce5a3f..908aae1d 100644 --- a/frontend/DOCKERFILE.dev +++ b/frontend/DOCKERFILE.dev @@ -8,7 +8,7 @@ WORKDIR /app # Install dependencies (cached layer) COPY package.json package-lock.json* ./ -RUN npm ci +RUN npm install # Copy source files COPY . . @@ -22,4 +22,4 @@ EXPOSE 5173 # dependencies missing whenever package.json gains a new one — surfacing as # ERR_MODULE_NOT_FOUND for a package that installs fine everywhere else. # Reconcile against the lockfile on every start so the volume can't go stale. -CMD ["sh", "-c", "npm ci && npm run dev -- --host 0.0.0.0"] +CMD ["sh", "-c", "npm install && npm run dev -- --host 0.0.0.0"] diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ce23d40c..c7354eff 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,10 +9,12 @@ "version": "0.0.1", "dependencies": { "driver.js": "^1.8.0", + "echarts": "^5.6.0", "gsap": "^3.15.0", "layerchart": "^2.0.1", "lenis": "^1.3.25", "posthog-js": "^1.404.1", + "svelte-echarts": "^1.0.0", "typesafe-i18n": "^5.27.1" }, "devDependencies": { @@ -1523,6 +1525,22 @@ "integrity": "sha512-+8/IO7h1v14IzWh2GP60N7T3PFZweXwdn5e5POuxRSBoCYUojsBxzqawPeXh3YZIibRy7EehYNEyxe7slwwtdg==", "license": "MIT" }, + "node_modules/echarts": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/echarts/-/echarts-5.6.0.tgz", + "integrity": "sha512-oTbVTsXfKuEhxftHqL5xprgLoc0k7uScAwtryCgWF6hPYFLRwOUHiFmHGCBKP5NPFNkDVopOieyUqYGH8Fa3kA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "2.3.0", + "zrender": "5.6.1" + } + }, + "node_modules/echarts/node_modules/tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "license": "0BSD" + }, "node_modules/enhanced-resolve": { "version": "5.21.6", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.21.6.tgz", @@ -2420,6 +2438,16 @@ "typescript": ">=5.0.0" } }, + "node_modules/svelte-echarts": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/svelte-echarts/-/svelte-echarts-1.0.0.tgz", + "integrity": "sha512-o0VgdxGJt+Km+IrxNo35qTJFqDsvj65p7JwAlIrk7CsWjBKKniJaUV6hKbMDNf0S34Su0idWbWEdymJNPX3anA==", + "license": "MIT", + "peerDependencies": { + "echarts": "^5.0.0", + "svelte": ">=5" + } + }, "node_modules/tailwind-merge": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.6.0.tgz", @@ -2637,6 +2665,21 @@ "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.4.tgz", "integrity": "sha512-B58NGBEoc8Y9MWWCQGl/gq9xBCe4IiKM0a2x7GZdQKOW5Exr8S1W24J6OgM1njK8xCRGvAJIL/MxXHf6SkmQKQ==", "license": "MIT" + }, + "node_modules/zrender": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/zrender/-/zrender-5.6.1.tgz", + "integrity": "sha512-OFXkDJKcrlx5su2XbzJvj/34Q3m6PvyCZkVPHGYpcCJ52ek4U/ymZyfuV1nKE23AyBJ51E/6Yr0mhZ7xGTO4ag==", + "license": "BSD-3-Clause", + "dependencies": { + "tslib": "2.3.0" + } + }, + "node_modules/zrender/node_modules/tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "license": "0BSD" } } } diff --git a/frontend/package.json b/frontend/package.json index ea9c532f..54fade46 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -30,10 +30,12 @@ }, "dependencies": { "driver.js": "^1.8.0", + "echarts": "^5.6.0", "gsap": "^3.15.0", "layerchart": "^2.0.1", "lenis": "^1.3.25", "posthog-js": "^1.404.1", + "svelte-echarts": "^1.0.0", "typesafe-i18n": "^5.27.1" } } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 26926619..e5e34a07 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -2,6 +2,27 @@ import type { StreamEvent } from "$lib/types"; +/** + * Turn a FastAPI error body's `detail` into a readable string. `detail` is + * usually a plain string, but 422 validation errors return it as an array of + * `{ loc, msg, type }` objects — passing that straight to `new Error()` renders + * as the useless "[object Object]". + */ +function extractDetail(data: any, status: number): string { + const detail = data?.detail; + if (typeof detail === "string" && detail) return detail; + if (Array.isArray(detail)) { + const msg = detail + .map((d) => (typeof d === "string" ? d : d?.msg)) + .filter(Boolean) + .join("; "); + if (msg) return msg; + } else if (detail && typeof detail === "object" && detail.msg) { + return detail.msg; + } + return `Request failed: ${status}`; +} + export async function apiCall( path: string, body?: unknown, @@ -10,15 +31,30 @@ export async function apiCall( const opts: RequestInit = { method: method || (body ? "POST" : "GET"), }; + const activeWorkspaceId = + typeof window !== "undefined" + ? localStorage.getItem("bolodb_active_workspace_id") + : null; + const activeDbId = + typeof window !== "undefined" && activeWorkspaceId + ? localStorage.getItem(`bolodb_active_db_id_${activeWorkspaceId}`) + : null; + const headers: Record = {}; + if (activeWorkspaceId) headers["X-Workspace-Id"] = activeWorkspaceId; + if (activeDbId) headers["X-Db-Id"] = activeDbId; + if (body) { - opts.headers = { "Content-Type": "application/json" }; + headers["Content-Type"] = "application/json"; opts.body = JSON.stringify(body); } + if (Object.keys(headers).length > 0) { + opts.headers = headers; + } opts.credentials = "include"; const r = await fetch(path, opts); const data = await r.json().catch(() => ({})); if (!r.ok) { - const error: any = new Error(data.detail || `Request failed: ${r.status}`); + const error: any = new Error(extractDetail(data, r.status)); error.status = r.status; throw error; } @@ -46,16 +82,30 @@ export async function streamApiCall( signal?: AbortSignal, ): Promise { try { + const activeWorkspaceId = + typeof window !== "undefined" + ? localStorage.getItem("bolodb_active_workspace_id") + : null; + const activeDbId = + typeof window !== "undefined" && activeWorkspaceId + ? localStorage.getItem(`bolodb_active_db_id_${activeWorkspaceId}`) + : null; + const headers: Record = { + "Content-Type": "application/json", + }; + if (activeWorkspaceId) headers["X-Workspace-Id"] = activeWorkspaceId; + if (activeDbId) headers["X-Db-Id"] = activeDbId; + const response = await fetch(path, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers, credentials: "include", body: JSON.stringify(body), signal, }); if (!response.ok) { const data = await response.json().catch(() => ({})); - throw new Error(data.detail || `Request failed: ${response.status}`); + throw new Error(extractDetail(data, response.status)); } if (!response.body) throw new Error("Streaming not supported"); @@ -185,6 +235,201 @@ export async function supabaseGoogleLogin(accessToken: string): Promise { }); } +export async function updateProfile(fields: any): Promise { + return apiCall("/api/auth/me", fields, "PATCH"); +} + +// --- Workspaces --- + +export interface WorkspaceSettings { + workspace_id?: string; + default_invite_role?: "member" | "admin" | string; + invite_expiry_days?: number; + activity_retention_days?: number; + role_permissions?: Record; + resolved_matrix?: Record>; + /** The matrix with nothing customised, so the UI can mark what is default. */ + default_matrix?: Record>; +} + +export async function getWorkspaceSettings( + workspaceId: string, +): Promise { + return apiCall(`/api/workspaces/${workspaceId}/settings`); +} + +export async function updateWorkspaceSettings( + workspaceId: string, + data: Partial, +): Promise { + return apiCall(`/api/workspaces/${workspaceId}/settings`, data, "PATCH"); +} + +export async function createWorkspace(name: string): Promise { + return apiCall("/api/workspaces", { name }); +} + +export async function updateWorkspace(id: string, name: string): Promise { + return apiCall(`/api/workspaces/${id}`, { name }, "PATCH"); +} + +export async function deleteWorkspace(id: string): Promise { + return apiCall(`/api/workspaces/${id}`, undefined, "DELETE"); +} + +export async function leaveWorkspace(id: string): Promise { + return apiCall(`/api/workspaces/${id}/leave`, undefined, "POST"); +} + +export async function updateConnectionAlias( + id: string, + aliasName: string, +): Promise { + return apiCall(`/api/connections/${id}`, { alias_name: aliasName }, "PATCH"); +} + +export async function getWorkspaceMembers(id: string): Promise { + return apiCall(`/api/workspaces/${id}/members`); +} + +export async function inviteWorkspaceMember( + id: string, + email: string, + role: string, +): Promise { + return apiCall(`/api/workspaces/${id}/members`, { email, role }, "POST"); +} + +/** Invite many people at once; the response reports a status per address. */ +export async function bulkInviteMembers( + id: string, + emails: string[], + role: string, +): Promise { + return apiCall( + `/api/workspaces/${id}/members/bulk`, + { emails, role }, + "POST", + ); +} + +export async function transferOwnership( + workspaceId: string, + userId: string, +): Promise { + return apiCall( + `/api/workspaces/${workspaceId}/transfer-ownership`, + { user_id: userId }, + "POST", + ); +} + +export async function updateWorkspaceMemberRole( + workspaceId: string, + userId: string, + role: string, +): Promise { + return apiCall( + `/api/workspaces/${workspaceId}/members/${userId}`, + { role }, + "PUT", + ); +} + +export async function removeWorkspaceMember( + workspaceId: string, + userId: string, +): Promise { + return apiCall( + `/api/workspaces/${workspaceId}/members/${userId}`, + undefined, + "DELETE", + ); +} + +/** Invites sent from this workspace that are still awaiting acceptance. */ +export async function getPendingInvites(workspaceId: string): Promise { + return apiCall(`/api/workspaces/${workspaceId}/invites`); +} + +export async function rescindInvite( + workspaceId: string, + inviteId: string, +): Promise { + return apiCall( + `/api/workspaces/${workspaceId}/invites/${inviteId}`, + undefined, + "DELETE", + ); +} + +export async function resendInvite( + workspaceId: string, + inviteId: string, +): Promise { + return apiCall( + `/api/workspaces/${workspaceId}/invites/${inviteId}/resend`, + undefined, + "POST", + ); +} + +export async function acceptWorkspaceInvite(token: string): Promise { + return apiCall(`/api/workspaces/invites/${token}/accept`, undefined, "POST"); +} + +export async function getWorkspaceActivity( + workspaceId: string, + page: number = 1, +): Promise { + const limit = 50; + const offset = (page - 1) * limit; + return apiCall( + `/api/workspaces/${workspaceId}/activity?limit=${limit}&offset=${offset}`, + ); +} + +/** + * Download the workspace activity log as CSV. + * + * Goes through fetch rather than a plain link because the endpoint needs the + * workspace header and session cookie that `apiCall` attaches. + */ +export async function downloadWorkspaceActivity( + workspaceId: string, +): Promise { + const headers: Record = { "X-Workspace-Id": workspaceId }; + const r = await fetch(`/api/workspaces/${workspaceId}/activity/export`, { + credentials: "include", + headers, + }); + if (!r.ok) { + const data = await r.json().catch(() => ({})); + const error: any = new Error(extractDetail(data, r.status)); + error.status = r.status; + throw error; + } + const blob = await r.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `activity-${new Date().toISOString().slice(0, 10)}.csv`; + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); +} + +// --- Databases --- + +export async function listDatabases(): Promise { + return apiCall("/api/databases"); +} + +export async function removeDatabase(): Promise { + return apiCall("/api/disconnect", undefined, "POST"); +} + /** Convert API rows (array of objects) to 2D string arrays for ResultTable */ export function rowsToArrays( columns: string[], diff --git a/frontend/src/lib/appState.svelte.ts b/frontend/src/lib/appState.svelte.ts index a354210f..ee538ee4 100644 --- a/frontend/src/lib/appState.svelte.ts +++ b/frontend/src/lib/appState.svelte.ts @@ -1,6 +1,6 @@ import { trustFor, schemaObjToDisplay } from "$lib/data"; -import { apiCall } from "$lib/api"; -import type { DbInfo, SchemaTable, Toast } from "$lib/types"; +import { apiCall, getConversations, listDatabases } from "$lib/api"; +import type { Conversation, DbInfo, SchemaTable, Toast } from "$lib/types"; import { goto } from "$app/navigation"; import { browser } from "$app/environment"; @@ -14,31 +14,32 @@ class AppState { theme = $state("dark"); openrouterReady = $state(false); activeConversationId = $state(null); + // Set while a database switch is in flight, so any screen showing results can + // put up a loading state — the switch is triggered from the shared shell. + switchingDatabase = $state(false); + workspaces = $state([]); + activeWorkspace = $state(null); + invites = $state([]); tourCompleted = $state(false); - // True while a freshly connected user is walking the onboarding flow — - // stops /onboard's has_knowledge redirect from kicking them out (sample - // databases ship with seeded knowledge, so has_knowledge alone can't - // distinguish "already onboarded" from "just connected"). - onboardingActive = $state(false); + // tourCompleted = $state(false); + + /** + * The workspace's saved databases and conversations, held here rather than in + * the components that show them. Both the sidebar and the database switcher + * are re-created on every route change, and re-fetching each time made moving + * between chat and dashboards flash empty lists over a page that hadn't + * actually changed. + */ + databases = $state([]); + conversations = $state([]); + conversationsLoaded = $state(false); + databasesLoaded = $state(false); constructor() { if (typeof window !== "undefined") { const stored = localStorage.getItem("bolodb_theme"); // Normalize legacy theme names to the two-theme system (dark / light) this.theme = stored === "dark" ? "dark" : stored ? "light" : "dark"; - // Survive a page reload mid-onboarding: sample DBs ship with seeded - // knowledge, so without this /onboard would bounce a refreshed user - // straight to /chat before they finished. - this.onboardingActive = - sessionStorage.getItem("bolodb_onboarding_active") === "1"; - } - } - - private setOnboardingActive(active: boolean) { - this.onboardingActive = active; - if (typeof window !== "undefined") { - if (active) sessionStorage.setItem("bolodb_onboarding_active", "1"); - else sessionStorage.removeItem("bolodb_onboarding_active"); } } @@ -59,7 +60,104 @@ class AppState { return trustFor(this.verifiedCount).key; } + /** + * Invite ids already seen, so a refresh only announces genuinely new ones. + * `null` until the first load — the invitations waiting when you sign in are + * shown by the bell, not thrown at you as a toast. + */ + private seenInviteIds: Set | null = null; + + async loadWorkspaces() { + try { + const previousWorkspaceId = this.activeWorkspace?.id; + this.workspaces = await apiCall("/api/workspaces"); + this.invites = await apiCall("/api/workspaces/invites/me"); + this.announceNewInvites(); + const stored = localStorage.getItem("bolodb_active_workspace_id"); + if (stored && this.workspaces.find((w: any) => w.id === stored)) { + this.activeWorkspace = this.workspaces.find( + (w: any) => w.id === stored, + ); + } else if (this.workspaces.length > 0) { + this.activeWorkspace = this.workspaces[0]; + localStorage.setItem( + "bolodb_active_workspace_id", + this.activeWorkspace.id, + ); + } else { + this.activeWorkspace = null; + localStorage.removeItem("bolodb_active_workspace_id"); + } + if (this.activeWorkspace?.id !== previousWorkspaceId) { + this.resetWorkspaceData(); + } + } catch (e) { + console.error("Failed to load workspaces:", e); + } + } + + /** + * Load the workspace's saved databases. `force` refetches even when a list is + * already held — used after connecting, renaming or removing one. + */ + async loadDatabases(force = false) { + if (this.databasesLoaded && !force) return; + try { + this.databases = await listDatabases(); + this.databasesLoaded = true; + } catch (e) { + console.error("Failed to load databases:", e); + } + } + + async loadConversations(force = false) { + if (this.conversationsLoaded && !force) return; + try { + const res = await getConversations(); + this.conversations = res?.conversations || []; + this.conversationsLoaded = true; + } catch (e) { + console.error("Failed to load conversations:", e); + } + } + + /** Drop cached per-workspace data when the active workspace changes. */ + resetWorkspaceData() { + this.databases = []; + this.databasesLoaded = false; + this.conversations = []; + this.conversationsLoaded = false; + } + + private announceNewInvites() { + const ids = new Set((this.invites || []).map((i: any) => i.id)); + if (this.seenInviteIds === null) { + this.seenInviteIds = ids; + return; + } + const fresh = (this.invites || []).filter( + (i: any) => !this.seenInviteIds!.has(i.id), + ); + this.seenInviteIds = ids; + if (fresh.length === 0) return; + this.showToast({ + title: + fresh.length === 1 + ? "New workspace invitation" + : `${fresh.length} new workspace invitations`, + body: + fresh.length === 1 + ? `You've been invited to ${fresh[0].workspace_name}. Open the bell to accept.` + : "Open the bell in the header to accept them.", + }); + } + async init(redirect: boolean = true) { + await this.loadWorkspaces(); + if (this.activeWorkspace) { + this.loadDatabases(); + this.loadConversations(); + } try { const s = await apiCall("/api/state"); this.openrouterReady = s.openrouter_ready ?? false; @@ -75,16 +173,18 @@ class AppState { console.error("Failed to load schema:", e); } this.isLoaded = true; - if (redirect) { - if (this.dbInfo?.has_knowledge) { - goto("/chat"); - } else { - goto("/onboard"); - } + if (this.workspaces.length === 0) { + goto("/workspaces/setup"); + } else if (redirect) { + goto("/chat"); } } else { this.isLoaded = true; - if (redirect) goto("/connect"); + if (this.workspaces.length === 0) { + goto("/workspaces/setup"); + } else if (redirect) { + goto("/connect"); + } } } catch (e: any) { this.isLoaded = true; @@ -121,10 +221,7 @@ class AppState { posthog.reset(); } try { - this.setOnboardingActive(false); - // Navigate to the landing page BEFORE clearing dbInfo. Clearing it while - // still on /chat (or /dashboard/onboard) would trip those pages' own - // "no database → /connect" redirect effect and race it to /connect. + // Navigate to the landing page BEFORE clearing dbInfo. await goto("/"); } finally { this.dbInfo = null; @@ -133,30 +230,80 @@ class AppState { this.starters = []; this.tourCompleted = false; this.activeConversationId = null; + this.resetWorkspaceData(); } } async setConnect(isSample: boolean, res: DbInfo) { if (res) { + if (typeof window !== "undefined" && res.db_id) { + if (this.activeWorkspace) + localStorage.setItem( + `bolodb_active_db_id_${this.activeWorkspace.id}`, + res.db_id, + ); + } this.dbInfo = res; this.verifiedCount = res.trust?.verified || 0; if (res.starters) this.starters = res.starters; + // The database this just connected has to appear in the switcher and on + // the connect screen straight away. + this.loadDatabases(true); + if (res.save_error) + this.showError(res.save_error, "Connection not saved"); try { const schema = await apiCall("/api/schema"); this.realSchema = schemaObjToDisplay(schema); } catch (e) { console.error("Failed to load schema:", e); } - // Sample databases always walk through onboarding (they ship with - // seeded knowledge, so has_knowledge can't mean "already onboarded"). - // Own databases with existing knowledge (reconnects) skip straight in. - if (!isSample && res.has_knowledge) { - goto("/chat"); - return; + // Always go to chat + goto("/chat"); + } + } + + /** + * Reconnect the workspace to `dbId`. + * + * Switching database normally abandons the open conversation, since its + * answers came from the database being left behind. Opening a conversation + * that belongs to another database is the exception — that switch exists to + * serve the conversation, so it passes `keepConversation`. + */ + async switchDatabase(dbId: string, { keepConversation = false } = {}) { + if (typeof window !== "undefined") { + if (this.activeWorkspace) + localStorage.setItem( + `bolodb_active_db_id_${this.activeWorkspace.id}`, + dbId, + ); + } + this.switchingDatabase = true; + try { + await apiCall("/api/reconnect", { db_id: dbId }); + const s = await apiCall("/api/state"); + if (s.database) { + this.dbInfo = s.database; + } + if (s.starters) { + this.starters = s.starters; } + if (!keepConversation) this.activeConversationId = null; + this.realSchema = null; + this.fetchSchemaAsync(true); + this.loadDatabases(true); + return true; + } catch (e: any) { + console.error("Failed to switch DB:", e); + // The backend explains *why* (unreachable, undecryptable credentials, no + // longer present); a generic message would hide all of that. + this.showError( + e?.message || "Failed to switch database. It may have been deleted.", + ); + return false; + } finally { + this.switchingDatabase = false; } - this.setOnboardingActive(true); - goto("/onboard"); } async setOnboardDone(seedCount: number) { @@ -176,7 +323,6 @@ class AppState { dialect: this.dbInfo?.dialect, }); } - this.setOnboardingActive(false); goto("/chat"); } @@ -224,17 +370,17 @@ class AppState { } async disconnect() { - try { - await apiCall("/api/disconnect", {}); - } catch (e) { - console.error("Failed to disconnect:", e); + if (typeof window !== "undefined") { + if (this.activeWorkspace) + localStorage.removeItem( + `bolodb_active_db_id_${this.activeWorkspace.id}`, + ); } this.dbInfo = null; this.realSchema = null; this.verifiedCount = 0; this.starters = []; this.activeConversationId = null; - this.setOnboardingActive(false); goto("/connect"); } @@ -243,6 +389,16 @@ class AppState { goto(`/chat?id=${id}`); } + async fetchSchemaAsync(forceRefresh = false) { + try { + const qs = forceRefresh ? "?refresh=true" : ""; + const schema = await apiCall(`/api/schema${qs}`); + this.realSchema = schemaObjToDisplay(schema); + } catch (e) { + console.error("Failed to load async schema:", e); + } + } + async fetchStartersAsync() { if (this.starters && this.starters.length > 0) return; try { diff --git a/frontend/src/lib/components/ActivityLog.svelte b/frontend/src/lib/components/ActivityLog.svelte new file mode 100644 index 00000000..0931f627 --- /dev/null +++ b/frontend/src/lib/components/ActivityLog.svelte @@ -0,0 +1,338 @@ + + +
+
+
+

Activity log

+

Workspace events from the last 30 days.

+
+ +
+ + {#if error} +
{error}
+ {/if} + + {#if loading && page === 1} +
+ + Loading activity… +
+ {:else if activities.length === 0} +
+ No activity recorded yet. +
+ {:else} +
+ + + + + + + + + + + {#each activities as a (a.id)} + + + + + + + {/each} + +
TimeActorEventDetails
+ {absoluteTime(a.created_at)} + + {a.actor_email || 'System'} + + {a.event_type} + {describe(a)}
+
+ + {#if hasMore} + + {/if} + {/if} +
+ + diff --git a/frontend/src/lib/components/AnswerCard.svelte b/frontend/src/lib/components/AnswerCard.svelte index 645c0b12..f26ad17b 100644 --- a/frontend/src/lib/components/AnswerCard.svelte +++ b/frontend/src/lib/components/AnswerCard.svelte @@ -9,9 +9,10 @@ import ResultChart from '$lib/components/charts/ResultChart.svelte'; import Thinking from '$lib/components/Thinking.svelte'; import Flywheel from '$lib/components/Flywheel.svelte'; - import { detectChartData } from '$lib/components/charts/chartUtils'; + import SaveQueryDialog from '$lib/components/SaveQueryDialog.svelte'; + import { detectChartData, planChart } from '$lib/components/charts/chartUtils'; - let { turn, onVerify, isLatest, liveArtifacts, onRegenerate, onEditPrompt }: + let { turn, onVerify, isLatest, liveArtifacts, onRegenerate, onEditPrompt, onRerun, rerunning = false }: { turn: Turn; onVerify: (id: string, verdict: string, reason: string | null) => void; @@ -19,6 +20,9 @@ liveArtifacts?: ThinkingArtifact[]; onRegenerate?: (id: string) => void; onEditPrompt?: (id: string, newQuestion: string) => void; + /** Re-execute this turn's SQL — no model call, so no tokens spent. */ + onRerun?: (id: string) => void; + rerunning?: boolean; } = $props(); let showReasons = $state(false); @@ -27,11 +31,35 @@ let editing = $state(false); let editValue = $state(''); let copyFeedback = $state<'response' | 'prompt' | null>(null); + let showSaveDialog = $state(false); + const stringRows = $derived((turn.rows || []).map(r => r.map(String))); + + // The model picks the chart from the SQL it wrote, so trust it when it made a + // call; the local heuristic only covers turns that have no chart spec. + const modelPlan = $derived(planChart(turn.chart, turn.columns || [], stringRows)); const hasChartData = $derived( - detectChartData(turn.columns || [], (turn.rows || []).map(r => r.map(String))) !== null + modelPlan !== null || + detectChartData(turn.columns || [], stringRows) !== null + ); + // "table" is a deliberate choice by the model, not an absent one — respect it. + const modelWantsChart = $derived(!!turn.chart && turn.chart.type !== 'table' && modelPlan !== null); + + // Open on the chart when the model asked for one, without pinning the toggle. + let userPickedView = $state(false); + const effectiveView = $derived( + userPickedView ? viewMode : (modelWantsChart ? 'chart' : 'table') ); + function toggleView() { + // Capture the view actually on screen *before* flipping the flag — once + // userPickedView is true, effectiveView resolves to viewMode, so reading it + // afterwards would ignore the model's chart default and no-op the first click. + const current = effectiveView; + userPickedView = true; + viewMode = current === 'table' ? 'chart' : 'table'; + } + function yes() { justVerified = true; onVerify(turn.id, 'correct', null); setTimeout(() => justVerified = false, 1600); } function no(reason: string) { showReasons = false; onVerify(turn.id, 'wrong', reason); } @@ -103,6 +131,26 @@ } +{#snippet rerunButton()} + {#if onRerun && turn.sql} + {#if turn.lastRunAt} + Updated {formatTime(turn.lastRunAt)} + {/if} + + {/if} +{/snippet} +
@@ -169,6 +217,9 @@
{turn.executionError}
{:else} +
+ {@render rerunButton()} +
{/if} {#if turn.resultTruncated} @@ -219,13 +270,14 @@ {/if} {#if !turn.executionError && turn.sql} -
+
+ {@render rerunButton()} {#if hasChartData} - viewMode = viewMode === 'table' ? 'chart' : 'table'} /> + {/if}
- {#if viewMode === 'chart' && hasChartData} - r.map(String))} /> + {#if effectiveView === 'chart' && hasChartData} + {:else} {/if} @@ -295,6 +347,11 @@ + {#if turn.sql} + + {/if}
+{#if showSaveDialog} + showSaveDialog = false} /> +{/if} + diff --git a/frontend/src/lib/components/AskScreen.svelte b/frontend/src/lib/components/AskScreen.svelte index c60918e6..49da044a 100644 --- a/frontend/src/lib/components/AskScreen.svelte +++ b/frontend/src/lib/components/AskScreen.svelte @@ -5,17 +5,15 @@ Turn, SchemaTable, DbInfo, - Toast, ThinkingArtifact, StreamEvent, Conversation, ConversationTurn, } from "$lib/types"; - import Sidebar from "$lib/components/Sidebar.svelte"; + import { page } from "$app/state"; + import AppShell from "$lib/components/AppShell.svelte"; import AnswerCard from "$lib/components/AnswerCard.svelte"; - import DashboardTab from "$lib/components/DashboardTab.svelte"; import SettingsTab from "$lib/components/SettingsTab.svelte"; - import TrustToast from "$lib/components/TrustToast.svelte"; import Spinner from '$lib/components/ui/Spinner.svelte'; import LoadingScreen from '$lib/components/ui/LoadingScreen.svelte'; import SlashCommandMenu from "$lib/components/ui/SlashCommandMenu.svelte"; @@ -28,7 +26,6 @@ verifiedCount, onVerify, onUpdateStarters, - toast, realSchema, dbInfo, starters, @@ -38,7 +35,6 @@ verifiedCount: number; onVerify: (count?: number) => void; onUpdateStarters: (s: string[]) => void; - toast: Toast | null; realSchema: SchemaTable[] | null; dbInfo: DbInfo | null; starters: string[]; @@ -46,24 +42,22 @@ onActiveConversationChange?: (id: string | null) => void; } = $props(); - const dbLabel = $derived( - dbInfo - ? (dbInfo.url || "").split("/").pop() || dbInfo.dialect || "your database" - : "your database", - ); - const tableCount = $derived(dbInfo ? dbInfo.tables || 0 : 0); let turns: Turn[] = $state([]); let input = $state(""); - let tab = $state<"ask" | "dash" | "settings">("ask"); - let userEmail = $state(""); + // ?tab=settings lets the sidebar land directly on Settings when coming back + // from a route that isn't /chat. + let tab = $state<"ask" | "settings">( + page.url.searchParams.get("tab") === "settings" ? "settings" : "ask", + ); let openCatalogTrigger = $state(0); - onMount(async () => { + onMount(() => { appState.fetchStartersAsync(); - try { - const res = await apiCall("/api/auth/me"); - userEmail = res?.content?.email || ""; - } catch {} + // Arriving from another route (the sidebar on /dashboards, say) carries the + // conversation to open in the URL, since that screen has no feed to load it + // into itself. + const requested = page.url.searchParams.get('conversation'); + if (requested) handleConversationSelect(requested); }); const suggestionChips = $derived( @@ -77,11 +71,38 @@ let conversationTrigger = $state(0); let activeConversationId: string | null = $state(null); let abortController: AbortController | null = $state(null); + /** Turn ids whose SQL is being re-executed right now. */ + let rerunning = $state>(new Set()); let showScrollBtn = $state(false); let lastTurnCount = 0; let convLoadSeq = 0; let convLoading = $state(false); // true only while fetching a past conversation - let mobileNavOpen = $state(false); + + // The database can be switched from the shared shell on any screen, so the + // feed reacts to the active database changing rather than to the click. + let lastDbId: string | null = null; + $effect(() => { + const id = dbInfo?.db_id ?? null; + if (lastDbId !== null && id !== lastDbId) { + abortController?.abort(); + turns = []; + activeConversationId = null; + onActiveConversationChange(null); + } + lastDbId = id; + }); + + /** + * Adopt a database change as already-seen, so the effect above treats it as + * the current state rather than as a switch to clear the feed over. Opening a + * conversation recorded against another database switches to it deliberately; + * without this, the effect fired straight afterwards and wiped the turns that + * had just been restored — which is why past conversations opened blank. + */ + function acknowledgeDbId(id: string | null) { + lastDbId = id; + } + const trust = $derived(trustFor(verifiedCount)); @@ -323,6 +344,7 @@ rows: rowsToArrays(data.columns || [], data.rows || []), confidence: data.confidence || "medium", reason: data.confidence_reason || "", + chart: data.chart || null, basedOn: data.based_on_verified || false, query_id: data.query_id || id, executionError: data.execution_error || null, @@ -426,6 +448,43 @@ requeryAt(idx, turns[idx].question); } + /** + * Re-run the SQL a turn already has, without going back to the model. Same + * question, fresh rows — the point is to refresh a result for free rather + * than spend tokens regenerating identical SQL. + */ + async function rerunSql(turnId: string) { + const turn = turns.find((t) => t.id === turnId); + if (!turn?.sql || rerunning.has(turnId)) return; + rerunning = new Set([...rerunning, turnId]); + try { + const data = await apiCall("/api/execute", { + sql: turn.sql, + save_history: false, + }); + turns = turns.map((t) => + t.id === turnId + ? { + ...t, + columns: data.columns || [], + rows: rowsToArrays(data.columns || [], data.rows || []), + executionError: null, + resultTruncated: false, + lastRunAt: new Date().toISOString(), + } + : t, + ); + } catch (e: any) { + appState.showError( + e?.message || "Couldn't re-run this query against your database.", + ); + } finally { + const next = new Set(rerunning); + next.delete(turnId); + rerunning = next; + } + } + function editAndRequery(turnId: string, newQuestion: string) { const idx = turns.findIndex((t) => t.id === turnId); if (idx === -1) return; @@ -503,6 +562,7 @@ })), confidence: (t.confidence || 'medium').toLowerCase() as "high" | "medium" | "low", reason: '', + chart: t.chart || null, basedOn: false, query_id: t._id, executionError: null, @@ -513,9 +573,6 @@ async function handleConversationSelect(convId: string) { if (convId === activeConversationId) return; - // Cancel any in-flight query stream and stamp this load so that when two - // conversations are clicked in quick succession, only the latest click's - // response is applied (otherwise the slower fetch would win). abortController?.abort(); const seq = ++convLoadSeq; loading = true; @@ -524,6 +581,17 @@ try { const conv = await getConversation(convId); if (seq !== convLoadSeq) return; + + // Auto-switch database if the conversation belongs to a different one + if (conv.database_id && dbInfo?.db_id !== conv.database_id) { + const switched = await appState.switchDatabase(conv.database_id, { + keepConversation: true, + }); + if (seq !== convLoadSeq) return; + if (!switched) return; // switchDatabase already explained why + acknowledgeDbId(conv.database_id); + } + activeConversationId = conv._id; onActiveConversationChange(conv._id); const loaded: Turn[] = []; @@ -549,73 +617,19 @@ e.preventDefault(); ask(); } - - // Close mobile nav when viewport leaves mobile breakpoint - onMount(() => { - if (typeof window !== 'undefined') { - const mediaQuery = window.matchMedia('(max-width: 768px)'); - const handleMediaChange = (e: MediaQueryListEvent | MediaQueryList) => { - if (!e.matches) { - mobileNavOpen = false; - } - }; - // Initial check - handleMediaChange(mediaQuery); - // Listen for changes - mediaQuery.addEventListener('change', handleMediaChange); - return () => mediaQuery.removeEventListener('change', handleMediaChange); - } - }); - -
- (tab = t)} - {verifiedCount} - schema={realSchema} - {dbInfo} - {conversationTrigger} - {activeConversationId} - onConversationSelect={handleConversationSelect} - onNewChat={handleNewConversation} - {userEmail} - theme={appState.theme} - onToggleTheme={() => appState.toggleTheme()} - onLogout={() => appState.logout()} - mobileOpen={mobileNavOpen} - onClose={() => (mobileNavOpen = false)} - /> - - {#if mobileNavOpen} - - {/if} - -
- -
- - BoloDB -
+ (tab = t as "ask" | "settings")} + {dbInfo} + {verifiedCount} + {realSchema} + {conversationTrigger} + {activeConversationId} + onConversationSelect={handleConversationSelect} + onNewChat={handleNewConversation} +> {#if tab === "ask"} - -
-
- 🗄 -
- {dbLabel} - {tableCount > 0 ? `${tableCount} table${tableCount === 1 ? "" : "s"} · ` : ""}read-only -
-
- -
-
{#if convLoading} @@ -624,6 +638,12 @@ submessage="Fetching your previous results" variant="default" /> + {:else if appState.switchingDatabase} + {:else}
{#if turns.length === 0} @@ -658,6 +678,8 @@ liveArtifacts={t.thinking ? currentArtifacts : undefined} onRegenerate={regenerate} onEditPrompt={editAndRequery} + onRerun={rerunSql} + rerunning={rerunning.has(t.id)} /> {/each} {/if} @@ -702,9 +724,6 @@
- {#if toast}{/if} - {:else if tab === "dash"} - {:else} {/if} -
-
+ diff --git a/frontend/src/lib/components/ConnectScreen.svelte b/frontend/src/lib/components/ConnectScreen.svelte index 10cda9ab..a4c605d7 100644 --- a/frontend/src/lib/components/ConnectScreen.svelte +++ b/frontend/src/lib/components/ConnectScreen.svelte @@ -1,10 +1,9 @@ -
+
{#if connecting} {/if} - - -
-

Let's meet your data.

-

- One click to explore, or connect your own database. Everything runs - read-only — nothing can be changed. -

-
- - +
+ - - {#if choice === "own"} - { if (e.key === "Enter") start(); }} - placeholder="postgresql://readonly_user:pass@host:5432/dbname" - data-testid="connect-url-input" - aria-label="Database connection string" - /> - {/if} - - {#if error} - - {/if} - - {#if choice === "own" && !appState.openrouterReady} -
AI not yet ready — set OPENROUTER_API_KEY in the server environment.
- {/if} - - +

+
- {#if recentConnections.length > 0} -
-
Recent databases · reconnect in one click
-
+
+ +
+

Workspace Databases

+ {#if loadingConnections} +
+
+
+
+ + + + +
+
+
+
+
+
+
+
+
+

Loading workspace databases…

+
+ {:else if recentConnections.length > 0} +
{#each recentConnections as conn} -
- - - -
- {conn.display_url?.split("/").pop() || conn.dialect || "Database"} - {DIALECT_LABELS[conn.dialect] || conn.dialect} · {conn.table_count || 0} table{conn.table_count === 1 ? "" : "s"}{conn.connected_at ? ` · ${timeAgo(conn.connected_at)}` : ""} +
+
+ {#if editingAliasId === conn.id} + handleRenameAlias(conn.id)} + onkeydown={(e) => { if (e.key === 'Enter') handleRenameAlias(conn.id); if (e.key === 'Escape') editingAliasId = null; }} + class="alias-edit" + autofocus + /> + {:else} + + {conn.alias_name || conn.display_url?.split('@').pop()?.split('/')[0] || conn.dialect} + + {#if isAdmin} + + {/if} + {/if} +
+
+ {DIALECT_LABELS[conn.dialect] || conn.dialect} + {conn.table_count} tables +
+
+ Last used {timeAgo(conn.connected_at)} +
+ {#if isAdmin} + + {/if} + +
- -
{/each}
+ {:else} +
+
+ + + + +
+

No databases connected

+ {#if isAdmin} +

Use the form on the right to add your first database connection.

+ {:else} +

Your workspace administrators have not connected any databases yet.
Please contact them to add a data source.

+ {/if} +
+ {/if} +
+ + + {#if isAdmin} +
+

Add New Connection

+ +
+ + +
+ + {#if choice === "own"} +
+ { if (e.key === "Enter") start(); }} + placeholder="postgresql://readonly_user:pass@host:5432/dbname" + data-testid="db-url-input" + /> + { if (e.key === "Enter") start(); }} + placeholder="Alias (e.g. Production DB) [Optional]" + /> +
+ {:else} +
+ 🛍️ +
+ Sample Webshop
+ 10 tables • 1,000 customers • 2,000 orders • Instant setup +
+
+ {/if} + + {#if error} +
{error}
+ {/if} + + + +

+ + We only run SELECT queries. Your data is never modified. +

{/if}
- -
READ-ONLY · NO TELEMETRY · YOUR ROWS NEVER LEAVE
diff --git a/frontend/src/lib/components/DashboardEditor.svelte b/frontend/src/lib/components/DashboardEditor.svelte new file mode 100644 index 00000000..7d58138b --- /dev/null +++ b/frontend/src/lib/components/DashboardEditor.svelte @@ -0,0 +1,380 @@ + + +
+ + + Drag panels to rearrange. Layout saves automatically on drop. +
+ +{#if !dashboard.panels?.length} +
+

No panels yet

+

Add a panel from a saved query to start building this dashboard.

+ +
+{:else} +
+ {#each dashboard.panels as panel (panelKey(panel))} + {@const w = panel.position?.w || 4} + {@const h = panel.position?.h || 4} + {@const sqId = String(panel.saved_query_id || '')} +
handleDragStart(e, panel)} + ondragover={handleDragOver} + ondrop={(e) => handleDrop(e, panel)} + ondragend={handleDragEnd} + > + + +
+ +
+
+ {/each} +
+{/if} + +{#if showAddModal} + + +
+
e.stopPropagation()}> +
+

Add chart panel

+ +
+
+ + {#if savedQueries.length === 0} +

No saved queries yet. Save a query from chat first, then add it here.

+ {/if} + + +
+ + +
+
+
+ + +
+
+
+{/if} + + diff --git a/frontend/src/lib/components/DashboardTab.svelte b/frontend/src/lib/components/DashboardTab.svelte deleted file mode 100644 index da5fbc47..00000000 --- a/frontend/src/lib/components/DashboardTab.svelte +++ /dev/null @@ -1,191 +0,0 @@ - - -
-
- - -
-
-

Dashboard

-

Analytics, trust metrics, and schema overview.

-
-
- - {#if dbInfo} - - -
- - -
-

Connection

-
-
- Database - {dbInfo.dialect} -
-
- Status - - - Connected - -
-
-
- - -
-

Total Queries

- {#if statsLoading} -
- {:else} -
{stats?.total_queries ?? 0}
-
queries executed
- {/if} -
- - -
-

Trust Level

- -
-
- - {#if statsError} -
- - Could not load query statistics — the stats API may be unavailable. Charts will update once the connection is restored. -
- {/if} - - -
-
- - {#if stats} - - {/if} - -
- -
- - {#if stats} - - {/if} - -
-
- - -
-
- - {#if stats} - - {/if} - -
- -
- - {#if appState.realSchema} - - {/if} - -
-
- - - {#if appState.realSchema && appState.realSchema.length > 0} -
-

Indexed Schema Details

-
- {#each appState.realSchema as table} -
{ e.currentTarget.style.boxShadow = 'var(--shadow)'; e.currentTarget.style.borderColor = 'var(--brand-tint-2)'; }} onmouseleave={(e) => { e.currentTarget.style.boxShadow = 'none'; e.currentTarget.style.borderColor = 'var(--border)'; }}> -
-

{table.name}

- {table.rows} rows -
-
- {#each table.cols as col} - - {col} - - {/each} -
-
- {/each} -
-
- {/if} - - {:else} -
-
-
- -
-

No database connected

-

Connect a database to view your dashboard analytics.

- -
-
- {/if} -
-
- - diff --git a/frontend/src/lib/components/DataCatalog.svelte b/frontend/src/lib/components/DataCatalog.svelte index e84bb81d..21fe74c5 100644 --- a/frontend/src/lib/components/DataCatalog.svelte +++ b/frontend/src/lib/components/DataCatalog.svelte @@ -3,9 +3,15 @@ import { getCatalog, saveCatalog, suggestCatalog } from "$lib/api"; import Button from "$lib/components/ui/Button.svelte"; import Spinner from "$lib/components/ui/Spinner.svelte"; + import { appState } from "$lib/appState.svelte"; let { onClose }: { onClose: () => void } = $props(); + const isAdmin = $derived( + appState.activeWorkspace?.role === "admin" || + appState.activeWorkspace?.role === "owner" + ); + type Field = { k: string; label: string; @@ -227,6 +233,7 @@ Teach BoloDB your business language. It's sent with every question to answer more accurately.
+ {#if isAdmin} + {/if}
@@ -269,6 +277,7 @@ {/if} {/each} + {#if isAdmin} + {/if}
{/each} + {#if isAdmin} + {/if}
{/each} {/if} @@ -335,6 +349,7 @@ >{totalEntries} {totalEntries === 1 ? "entry" : "entries"}
+ {#if isAdmin} + {:else} + + {/if}
diff --git a/frontend/src/lib/components/ExitIntentModal.svelte b/frontend/src/lib/components/ExitIntentModal.svelte index d3af6549..7a6ea000 100644 --- a/frontend/src/lib/components/ExitIntentModal.svelte +++ b/frontend/src/lib/components/ExitIntentModal.svelte @@ -142,7 +142,7 @@

No credit card, no database of your own needed. We spin up a realistic - TechStore e-commerce dataset so you can see BoloDB answer real questions + webshop dataset so you can see BoloDB answer real questions in under 30 seconds.

diff --git a/frontend/src/lib/components/OnboardScreen.svelte b/frontend/src/lib/components/OnboardScreen.svelte deleted file mode 100644 index 666045b5..00000000 --- a/frontend/src/lib/components/OnboardScreen.svelte +++ /dev/null @@ -1,117 +0,0 @@ - - -
- {#if onChangeDb} - - {/if} - - -
- {#if saving} - - {:else} - - {/if} -
- -
READ-ONLY · NO TELEMETRY · YOUR ROWS NEVER LEAVE
-
- - diff --git a/frontend/src/lib/components/SaveQueryDialog.svelte b/frontend/src/lib/components/SaveQueryDialog.svelte new file mode 100644 index 00000000..55223484 --- /dev/null +++ b/frontend/src/lib/components/SaveQueryDialog.svelte @@ -0,0 +1,420 @@ + + + + + diff --git a/frontend/src/lib/components/SettingsTab.svelte b/frontend/src/lib/components/SettingsTab.svelte index a5124182..9b8c6820 100644 --- a/frontend/src/lib/components/SettingsTab.svelte +++ b/frontend/src/lib/components/SettingsTab.svelte @@ -1,6 +1,9 @@
-
-

Settings

+
+
+
+

Workspace

+

App settings

+

Connection, catalog, and appearance for this session.

+
+ +
-
- CONNECTION -
- {dbLabel} - READ-ONLY ENFORCED +
+
Connection
+
+
+
+ + {dbLabel} +
+
+ {dbInfo ? `${dbInfo.dialect || 'SQL'} · read-only enforced` : 'Connect a database to start asking questions'} +
+
+
+ {#if dbInfo} + + {#if onDisconnect} + + {/if} + {#if isAdmin} + + {/if} + {:else} + + {/if} +
- {#if onDisconnect} - - {/if} -
+ -
- DATA CATALOG -

Teach BoloDB your business terms, metrics and value meanings so answers get more accurate.

- -
+
+
Data catalog
+

+ Teach BoloDB your business terms, metrics, and value meanings so answers get more accurate over time. +

+ {#if isAdmin} + + {:else} +

Only workspace admins can modify the data catalog.

+ {/if} +
-
- APPEARANCE -
- Theme +
+
Appearance
+
+
+
Theme
+
Quick toggle for this device.
+
-
+ + -
- PRIVACY +
+
Privacy

- Knowledge and settings are stored per database. Only your question, schema and a few sample - values are sent to the AI — never rows, results or credentials. + Knowledge and settings are stored per workspace. Only your question, schema, and a few sample + values are sent to the AI — never rows, results, or credentials.

-
+
@@ -73,51 +124,131 @@ {/if} diff --git a/frontend/src/lib/components/Sidebar.svelte b/frontend/src/lib/components/Sidebar.svelte index 8979ede1..b496b8eb 100644 --- a/frontend/src/lib/components/Sidebar.svelte +++ b/frontend/src/lib/components/Sidebar.svelte @@ -1,8 +1,10 @@ - +
diff --git a/frontend/src/lib/components/charts/ConfidenceDonut.svelte b/frontend/src/lib/components/charts/ConfidenceDonut.svelte deleted file mode 100644 index cac82619..00000000 --- a/frontend/src/lib/components/charts/ConfidenceDonut.svelte +++ /dev/null @@ -1,55 +0,0 @@ - - -{#if total === 0} -
- No queries yet -
-{:else} -
-
- CONFIDENCE_COLORS[d.key] ?? 'var(--faint)')} - innerRadius={0.55} - cornerRadius={4} - padAngle={0.02} - labels={{ placement: 'centroid', format: (v: any) => { const n = Number(v); const pct = Math.round((n / total) * 100); return pct >= 5 ? `${pct}%` : ''; } }} - /> -
-
- {#each chartData as item} - {@const pct = Math.round((item.value / total) * 100)} -
-
- {item.label} - {item.value} - {pct}% -
- {/each} -
- Total: {total} queries -
-
-
-{/if} diff --git a/frontend/src/lib/components/charts/QueryTimeline.svelte b/frontend/src/lib/components/charts/QueryTimeline.svelte deleted file mode 100644 index 11eed8b9..00000000 --- a/frontend/src/lib/components/charts/QueryTimeline.svelte +++ /dev/null @@ -1,31 +0,0 @@ - - -{#if data.length === 0} -
- No activity yet -
-{:else} -
- -
-{/if} diff --git a/frontend/src/lib/components/charts/ResultChart.svelte b/frontend/src/lib/components/charts/ResultChart.svelte index a612b735..56fbbc4f 100644 --- a/frontend/src/lib/components/charts/ResultChart.svelte +++ b/frontend/src/lib/components/charts/ResultChart.svelte @@ -1,90 +1,295 @@ + // planChart returns null when the spec is unusable against these columns, so + // the heuristic still covers old turns and non-chartable results. + const plan = $derived( + planChart(spec, columns, rows) ?? + (() => { + const d = detectChartData(columns, rows); + return d && d.type + ? { type: d.type, labelKey: d.labelKey, valueKey: d.valueKey, data: d.data, title: '' } + : null; + })(), + ); + + /** Bars need room to breathe; taller as the category count grows, then capped. */ + const chartHeight = $derived( + plan?.type === 'bar' + ? Math.max(160, Math.min(plan.data.length * 30 + 40, 420)) + : 240, + ); + + let chartDom: HTMLElement | undefined = $state(); + + $effect(() => { + if (!chartDom || !plan || plan.type === 'number') return; + const instance = echarts.init(chartDom); + instance.setOption(buildOption(plan, appState.theme), true); -{#if !detection || !detection.type} -
- This data doesn't have a chartable format. -
-{:else if detection.type === 'bar'} -
- -
-{:else if detection.type === 'pie'} -
-
- { - const n = Number(v); - const total = detection.data.reduce((s: number, item: { value: number }) => s + item.value, 0); - const pct = total > 0 ? Math.round((n / total) * 100) : 0; - return pct >= 5 ? `${pct}%` : ''; + const observer = new ResizeObserver(() => instance.resize()); + observer.observe(chartDom); + return () => { + observer.disconnect(); + instance.dispose(); + }; + }); + + function buildOption(p: NonNullable, theme: string) { + const palette = chartColors(theme); + const ink = cssVar('--ink', theme === 'dark' ? '#f0f0f0' : '#16201b'); + const muted = cssVar('--muted', theme === 'dark' ? '#8b8d91' : '#5c6b63'); + const surface = cssVar('--surface', theme === 'dark' ? '#141518' : '#ffffff'); + const border = cssVar('--border', theme === 'dark' ? '#2c2e33' : '#e3e8e5'); + const labels = p.data.map((d) => d.label); + const values = p.data.map((d) => d.value); + + const base: any = { + color: palette, + animationDuration: 400, + textStyle: { color: muted, fontFamily: 'inherit', fontSize: 11.5 }, + tooltip: { + backgroundColor: surface, + borderColor: border, + borderWidth: 1, + textStyle: { color: ink, fontSize: 12 }, + extraCssText: 'border-radius:8px;box-shadow:0 6px 20px rgba(0,0,0,.14);', + }, + }; + + if (p.type === 'pie') { + const total = values.reduce((s, v) => s + v, 0); + return { + ...base, + tooltip: { + ...base.tooltip, + trigger: 'item', + formatter: (i: any) => + `${i.marker} ${i.name}
${formatNumber(i.value)} · ${i.percent}%`, + }, + legend: { + type: 'scroll', + orient: 'vertical', + right: 0, + top: 'middle', + itemWidth: 9, + itemHeight: 9, + itemGap: 8, + // Identity stays in text ink; the swatch beside it carries the colour. + textStyle: { color: ink, fontSize: 12 }, + }, + series: [ + { + type: 'pie', + radius: ['52%', '78%'], + center: ['32%', '50%'], + // A 2px ring of surface between slices keeps neighbours legible + // without a border colour of their own. + itemStyle: { borderColor: surface, borderWidth: 2, borderRadius: 3 }, + // Only slices with room get a direct label — never one per slice. + label: { + show: true, + color: ink, + fontSize: 11, + formatter: (i: any) => (i.percent >= 8 ? `${Math.round(i.percent)}%` : ''), + }, + labelLine: { show: false }, + data: p.data.map((d) => ({ name: d.label, value: d.value })), }, - }} - /> + ], + }; + } + + if (p.type === 'bar') { + return { + ...base, + grid: { left: 4, right: 24, top: 8, bottom: 4, containLabel: true }, + tooltip: { + ...base.tooltip, + trigger: 'axis', + axisPointer: { type: 'shadow', shadowStyle: { color: `${muted}14` } }, + formatter: (ps: any[]) => + `${ps[0].name}
${formatNumber(ps[0].value)} ${p.valueKey}`, + }, + xAxis: { + type: 'value', + axisLabel: { color: muted, formatter: (v: number) => formatNumber(v) }, + axisLine: { show: false }, + axisTick: { show: false }, + splitLine: { lineStyle: { color: border, type: 'dashed' } }, + }, + yAxis: { + type: 'category', + data: labels, + inverse: true, + axisLabel: { + color: muted, + width: 110, + overflow: 'truncate', + }, + axisLine: { show: false }, + axisTick: { show: false }, + }, + series: [ + { + type: 'bar', + data: values, + barMaxWidth: 18, + // Rounded at the data end only — the baseline end stays square so + // bars read as measured from zero. + itemStyle: { color: palette[0], borderRadius: [0, 4, 4, 0] }, + }, + ], + }; + } + + // line / area + const isArea = p.type === 'area'; + return { + ...base, + grid: { left: 4, right: 12, top: 12, bottom: 4, containLabel: true }, + tooltip: { + ...base.tooltip, + trigger: 'axis', + axisPointer: { type: 'line', lineStyle: { color: border } }, + formatter: (ps: any[]) => + `${ps[0].axisValue}
${formatNumber(ps[0].value)} ${p.valueKey}`, + }, + xAxis: { + type: 'category', + data: labels, + boundaryGap: false, + axisLabel: { color: muted, hideOverlap: true }, + axisLine: { lineStyle: { color: border } }, + axisTick: { show: false }, + }, + yAxis: { + type: 'value', + axisLabel: { color: muted, formatter: (v: number) => formatNumber(v) }, + axisLine: { show: false }, + splitLine: { lineStyle: { color: border, type: 'dashed' } }, + }, + series: [ + { + type: 'line', + data: values, + smooth: false, + symbol: 'circle', + symbolSize: 8, + showSymbol: p.data.length <= 40, + lineStyle: { width: 2, color: palette[0] }, + itemStyle: { color: palette[0], borderColor: surface, borderWidth: 2 }, + areaStyle: isArea + ? { + color: new echarts.graphic.LinearGradient(0, 0, 0, 1, [ + { offset: 0, color: `${palette[0]}59` }, + { offset: 1, color: `${palette[0]}05` }, + ]), + } + : undefined, + }, + ], + }; + } + + +{#if !plan} +
This data doesn't have a chartable format.
+{:else} + {#if plan.title} +
{plan.title}
+ {/if} + + {#if plan.type === 'number'} + +
+
{formatNumber(plan.data[0].value)}
+
{plan.valueKey}
-
- {#each detection.data as item} - {@const total = detection.data.reduce((s: number, d: { value: number }) => s + d.value, 0)} - {@const pct = total > 0 ? Math.round((item.value / total) * 100) : 0} -
-
- {item.label} - {item.value.toLocaleString()} - {pct}% -
- {/each} + {:else} +
+
-
-{:else if detection.type === 'line'} -
- -
+ {/if} {/if} + + diff --git a/frontend/src/lib/components/charts/SchemaOverview.svelte b/frontend/src/lib/components/charts/SchemaOverview.svelte deleted file mode 100644 index 63fe5899..00000000 --- a/frontend/src/lib/components/charts/SchemaOverview.svelte +++ /dev/null @@ -1,46 +0,0 @@ - - -{#if tableData.length === 0} -
- No schema data -
-{:else} -
- {#each tableData as table, i} - {@const pct = Math.round((table.rowCount / maxRows) * 100)} -
-
- {table.name} - {table.rowCount.toLocaleString()} rows · {table.colCount} cols -
-
-
-
-
- {/each} -
-{/if} diff --git a/frontend/src/lib/components/charts/TableUsageBar.svelte b/frontend/src/lib/components/charts/TableUsageBar.svelte deleted file mode 100644 index 0da45ad9..00000000 --- a/frontend/src/lib/components/charts/TableUsageBar.svelte +++ /dev/null @@ -1,35 +0,0 @@ - - -{#if chartData.length === 0} -
- No table usage data -
-{:else} -
- -
-{/if} diff --git a/frontend/src/lib/components/charts/TrustGauge.svelte b/frontend/src/lib/components/charts/TrustGauge.svelte deleted file mode 100644 index d81555f9..00000000 --- a/frontend/src/lib/components/charts/TrustGauge.svelte +++ /dev/null @@ -1,55 +0,0 @@ - - -
-
- -
- -
-
- - {level.label} -
- {#if nextMilestone !== null} -
- {nextMilestone - verifiedCount} more verification{nextMilestone - verifiedCount === 1 ? '' : 's'} to reach {level.key === 'supervised' ? 'Assisted' : 'Trusted'} -
- {:else} -
- Maximum trust level reached -
- {/if} -
-
diff --git a/frontend/src/lib/components/charts/chartUtils.ts b/frontend/src/lib/components/charts/chartUtils.ts index e7520264..fb471d6f 100644 --- a/frontend/src/lib/components/charts/chartUtils.ts +++ b/frontend/src/lib/components/charts/chartUtils.ts @@ -98,17 +98,148 @@ export function detectChartData( return best; } +/** A chart the model asked for, resolved against the columns actually returned. */ +export type ChartPlan = + | { + type: "bar" | "pie" | "line" | "area"; + labelKey: string; + valueKey: string; + data: { label: string; value: number }[]; + title: string; + } + | { + type: "number"; + labelKey: string; + valueKey: string; + data: { label: string; value: number }[]; + title: string; + }; + +/** + * Resolve the model's chart choice against real result columns. + * + * The model picks from the SQL it wrote, so its axis names normally match. When + * they don't — a renamed alias, a repaired query — we fall back to the + * positional heuristic rather than rendering nothing. + */ +export function planChart( + spec: + | { type?: string; x_axis?: string; y_axis?: string; title?: string } + | null + | undefined, + columns: string[], + rows: string[][], +): ChartPlan | null { + const type = spec?.type; + if (!type || type === "table") return null; + if (!columns?.length || !rows?.length) return null; + + const findCol = (name?: string) => { + if (!name) return -1; + const target = name.trim().toLowerCase(); + return columns.findIndex((c) => c.trim().toLowerCase() === target); + }; + + if (type === "number") { + const valIdx = findCol(spec?.y_axis); + const idx = valIdx >= 0 ? valIdx : 0; + const value = parseNumeric(String(rows[0]?.[idx] ?? "")); + if (value === null) return null; + return { + type: "number", + labelKey: "", + valueKey: columns[idx], + data: [{ label: columns[idx], value }], + title: spec?.title || "", + }; + } + + if (type !== "bar" && type !== "pie" && type !== "line" && type !== "area") { + return null; + } + + let labelIdx = findCol(spec?.x_axis); + let valueIdx = findCol(spec?.y_axis); + + if (labelIdx < 0 || valueIdx < 0 || labelIdx === valueIdx) { + const detected = detectChartData(columns, rows); + if (!detected) return null; + labelIdx = columns.indexOf(detected.labelKey); + valueIdx = columns.indexOf(detected.valueKey); + if (labelIdx < 0 || valueIdx < 0) return null; + } + + const data = rows + .map((r) => { + const value = parseNumeric(String(r?.[valueIdx] ?? "")); + return value === null + ? null + : { label: String(r?.[labelIdx] ?? ""), value }; + }) + .filter((d): d is { label: string; value: number } => d !== null); + + if (!data.length) return null; + + return { + type: type as "bar" | "pie" | "line" | "area", + labelKey: columns[labelIdx], + valueKey: columns[valueIdx], + data, + title: spec?.title || "", + }; +} + +/** + * Categorical series colours, in fixed order — slot 1 is always the first + * series, never re-assigned when a filter changes how many series there are. + * + * Light and dark are separate sets rather than one set re-used, because the + * readable lightness band differs per background: the steps below sit inside + * OKLCH L 0.43–0.77 on light and 0.48–0.67 on dark. Both orders were checked + * for colour-vision separation between neighbouring slots — the previous order + * put teal next to pink, which deuteranopes cannot tell apart (ΔE 3.7). + * Re-check with the dataviz skill's `validate_palette.js` before editing. + */ export const CHART_COLORS = [ - "var(--brand)", + "#1b9e6b", "#6366f1", "#f59e0b", + "#ec4899", + "#0ea5e9", "#ef4444", "#8b5cf6", - "#06b6d4", + "#65a30d", +]; + +export const CHART_COLORS_DARK = [ + "#1b9e6b", + "#6366f1", + "#d97706", "#ec4899", - "#14b8a6", + "#0b8ec7", + "#ef4444", + "#8b5cf6", + "#65a30d", ]; +export function chartColors(theme: string): string[] { + return theme === "dark" ? CHART_COLORS_DARK : CHART_COLORS; +} + +/** + * Resolve a CSS custom property to a concrete colour. + * + * ECharts paints to a canvas, where `var(--ink)` means nothing — every colour + * handed to it has to be resolved against the document first. + */ +export function cssVar(name: string, fallback: string): string { + if (typeof window === "undefined") return fallback; + const v = getComputedStyle(document.documentElement) + .getPropertyValue(name) + .trim(); + return v || fallback; +} + export const CONFIDENCE_COLORS: Record = { High: "var(--c-high)", Medium: "var(--c-med)", diff --git a/frontend/src/lib/components/ui/ConfirmDialog.svelte b/frontend/src/lib/components/ui/ConfirmDialog.svelte new file mode 100644 index 00000000..d205c5c3 --- /dev/null +++ b/frontend/src/lib/components/ui/ConfirmDialog.svelte @@ -0,0 +1,271 @@ + + +{#if open} + +{/if} + + diff --git a/frontend/src/lib/components/ui/InviteBell.svelte b/frontend/src/lib/components/ui/InviteBell.svelte new file mode 100644 index 00000000..dab21f74 --- /dev/null +++ b/frontend/src/lib/components/ui/InviteBell.svelte @@ -0,0 +1,190 @@ + + +{#if count > 0} +
+ + + {#if open} +
+
+ Pending invitation{count === 1 ? '' : 's'} +
+ {#each invites as invite (invite.id)} +
+
+
{invite.workspace_name}
+
Invited as {invite.role}
+
+ +
+ {/each} +
+ {/if} +
+{/if} + + diff --git a/frontend/src/lib/components/ui/Navbar.svelte b/frontend/src/lib/components/ui/Navbar.svelte index 3fbda51d..b001b3d8 100644 --- a/frontend/src/lib/components/ui/Navbar.svelte +++ b/frontend/src/lib/components/ui/Navbar.svelte @@ -4,6 +4,7 @@ import { onMount } from 'svelte'; import { apiCall } from '$lib/api'; import Logo from './Logo.svelte'; + import InviteBell from './InviteBell.svelte'; let userEmail = $state(''); let menuOpen = $state(false); @@ -47,6 +48,7 @@
+ {#if menuOpen}
@@ -80,6 +87,15 @@ Profile +
+ +{#if appState.toast} +
+ +
+{/if} + diff --git a/frontend/src/routes/dashboards/[id]/+page.svelte b/frontend/src/routes/dashboards/[id]/+page.svelte new file mode 100644 index 00000000..f24885c3 --- /dev/null +++ b/frontend/src/routes/dashboards/[id]/+page.svelte @@ -0,0 +1,228 @@ + + + +
+ {#if loading && !dashboard} +
+ {:else if error} + + Back to dashboards + {:else if dashboard} +
+
+ ← Dashboards +

{dashboard.name}

+ {#if dashboard.description} +

{dashboard.description}

+ {/if} +
+
+ {#if lastRefreshed} + + Updated {lastRefreshed.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} + + {/if} + + {#if canEdit} + Edit dashboard + {/if} +
+
+ + {#if !dashboard.panels?.length} +
+

This dashboard is empty

+

Add chart panels from saved queries to visualize your workspace data.

+ {#if canEdit} + Open editor + {/if} +
+ {:else} +
+ {#each dashboard.panels as panel (panel.id || panel._id)} + {@const w = panel.position?.w || 4} + {@const h = panel.position?.h || 4} + {@const sqId = String(panel.saved_query_id || '')} +
+ +
+ {/each} +
+ {/if} + {/if} +
+ + + diff --git a/frontend/src/routes/dashboards/[id]/edit/+page.svelte b/frontend/src/routes/dashboards/[id]/edit/+page.svelte new file mode 100644 index 00000000..0847394b --- /dev/null +++ b/frontend/src/routes/dashboards/[id]/edit/+page.svelte @@ -0,0 +1,209 @@ + + + +
+ {#if loading && !dashboard} +
+ {:else if dashboard} +
+
+ ← Done editing +
+ + +
+
+ +
+ + + {/if} +
+ + + diff --git a/frontend/src/routes/layout.css b/frontend/src/routes/layout.css index 522b4210..97981e1a 100644 --- a/frontend/src/routes/layout.css +++ b/frontend/src/routes/layout.css @@ -78,6 +78,10 @@ --shadow-brand: 0 8px 22px -8px rgba(27, 158, 107, 0.5); --ring: rgba(27, 158, 107, 0.16); --nav-bg: rgba(255, 255, 255, 0.7); + --brand-rgb: 27, 158, 107; + --surface-rgb: 255, 255, 255; + --surface-1: var(--surface); + --brand-shadow: rgba(27, 158, 107, 0.35); } /* ---------- THEME: soft (warm/human) ---------- */ @@ -114,6 +118,10 @@ --shadow-brand: 0 10px 28px -8px rgba(43, 162, 119, 0.5); --ring: rgba(43, 162, 119, 0.18); --nav-bg: rgba(255, 255, 255, 0.7); + --brand-rgb: 43, 162, 119; + --surface-rgb: 255, 255, 255; + --surface-1: var(--surface); + --brand-shadow: rgba(43, 162, 119, 0.35); } /* ---------- THEME: dark (premium/linear-like) ---------- */ @@ -177,6 +185,10 @@ --code-bg: var(--surface-2); --link: var(--brand-ink); --glow-rgb: 27, 158, 107; + --brand-rgb: 27, 158, 107; + --surface-rgb: 255, 255, 255; + --surface-1: var(--surface); + --brand-shadow: rgba(27, 158, 107, 0.35); --contrast-bg: #16201b; --contrast-ink: #ffffff; --low: var(--c-low-ink); @@ -198,6 +210,10 @@ --contrast-bg: #f0f0f0; --contrast-ink: #0a0a0a; --glow-rgb: 77, 166, 255; + --brand-rgb: 77, 166, 255; + --surface-rgb: 20, 21, 24; + --surface-1: var(--surface); + --brand-shadow: rgba(77, 166, 255, 0.3); } [data-theme="light"] { @@ -232,6 +248,10 @@ --contrast-bg: #16201b; --contrast-ink: #ffffff; --glow-rgb: 27, 158, 107; + --brand-rgb: 27, 158, 107; + --surface-rgb: 255, 255, 255; + --surface-1: var(--surface); + --brand-shadow: rgba(27, 158, 107, 0.35); --radius-xs: 8px; --radius-sm: 11px; --radius: 14px; @@ -686,3 +706,11 @@ svg text.lc-axis-tick-label { .lc-bars rect:hover { filter: brightness(1.15); } + +/* Compact density preference (account settings) */ +html[data-density="compact"] { + --radius-xs: 6px; + --radius-sm: 8px; + --radius: 10px; + --radius-lg: 14px; +} diff --git a/frontend/src/routes/onboard/+page.svelte b/frontend/src/routes/onboard/+page.svelte deleted file mode 100644 index 76771bbd..00000000 --- a/frontend/src/routes/onboard/+page.svelte +++ /dev/null @@ -1,66 +0,0 @@ - - - - Onboarding — BoloDB - - -
- {#if appState.isLoaded && appState.dbInfo} - appState.setOnboardDone(seedCount)} - onChangeDb={() => appState.disconnect()} - dbInfo={appState.dbInfo} - schema={appState.realSchema} - /> - {:else} -
-
- -

Loading…

-
-
- {/if} -
- - diff --git a/frontend/src/routes/profile/+page.svelte b/frontend/src/routes/profile/+page.svelte index 3ba7f251..56131592 100644 --- a/frontend/src/routes/profile/+page.svelte +++ b/frontend/src/routes/profile/+page.svelte @@ -1,95 +1,412 @@ - Profile — BoloDB + Account Settings — BoloDB -
-

Profile

-

Manage your BoloDB account.

- - {#if loading} -
- Loading your profile… -
- {:else if error} - - {:else if user} -
-
- -
-
Signed in as
-
{user.email}
+{#if loading} + +{:else} +
+
+
+
{initials}
+
+

Account

+

{displayName}

+
+
+ + +
+
-
-
-
Role
-
{user.role || 'user'}
-
- {#if user.created_at} -
-
Member since
-
- {new Date(user.created_at).toLocaleDateString(undefined, { year: 'numeric', month: 'long', day: 'numeric' })} + {#if error} + + {/if} + +
+
+
+
+

Personal information

+

How you appear to teammates across workspaces.

+
+
+
+
+ + Shown in member lists and activity logs. +
+ +
+
+
+ + Used for sign-in and workspace invitations. +
+
- {/if} -
-
Verified answers
-
{appState.verifiedCount}
-
-
-
+ -
-
Account actions
-
- - - +
+
+

Appearance

+

Personalize the product for your day-to-day work.

+
+
+
+
+ + Choose light, dark, or follow the system setting. +
+
+ + + +
+
+
+
+ + Tighten spacing for dense analytical workflows. +
+
+ + +
+
+
+
+ +
- {/if} -
+
+{/if} + + diff --git a/frontend/src/routes/verify-email/+page.svelte b/frontend/src/routes/verify-email/+page.svelte index aa3ca04e..5fa2bda0 100644 --- a/frontend/src/routes/verify-email/+page.svelte +++ b/frontend/src/routes/verify-email/+page.svelte @@ -1,5 +1,6 @@ + + + Workspace Settings — BoloDB + + +{#if loading} + +{:else} +
+
+
+

Administration

+

Workspace settings

+

+ Manage identity, members, and access for + {appState.activeWorkspace?.name || 'your workspace'}. +

+
+ +
+ + {#if error} + + {/if} + + {#if invites.length > 0} + + {/if} + +
+ + +
+ {#if !appState.activeWorkspace && section !== 'workspaces'} +
+

No active workspace

+

Create or switch to a workspace to manage settings.

+ +
+ {:else if section === 'general' && appState.activeWorkspace} +
+
+

General

+

Workspace identity visible to all members.

+
+
+
+
+ + Shown in the sidebar and invite emails. +
+
+
+ + {#if isAdmin && renameError} + {renameError} + {/if} +
+ {#if isAdmin} + + {/if} +
+
+
+
+
+ + Permissions for this workspace. +
+ + {appState.activeWorkspace.role} + +
+
+
+
+ + Use when contacting support. +
+
+ {appState.activeWorkspace.id} + +
+
+
+
+
+ + People with access to this workspace. +
+ {members.length} +
+
+
+ + {#if isOwner} +
+
+

Workspace defaults

+

Default settings for new invitations and activity log retention.

+
+
+
+
+ + Initial role assigned when inviting new members. +
+
+ +
+
+
+
+
+ + Number of days before an invitation expires (1-365). +
+
+ +
+
+
+
+
+ + Number of days activity logs are retained (1-365). +
+
+ +
+
+
+
+ +
+
+
+ {/if} + +
+
+

Data & privacy

+

How BoloDB handles workspace data.

+
+
+
+ Read-only queries + SQL execution is restricted to SELECT statements. +
+
+ Schema-only AI context + Prompts include structure and samples — never bulk rows or credentials. +
+
+ Workspace-scoped knowledge + Catalog terms and verified examples stay inside this workspace. +
+
+
+ +
+
+

Danger zone

+

Irreversible actions. Take care.

+
+
+
+
+ + + {isSoleOwner + ? 'You are the sole owner — transfer ownership or delete the workspace instead.' + : 'Give up your own access. Everything you created stays behind.'} + +
+ +
+ {#if isOwner} +
+
+
+ + + Removes the workspace and every member, connection, dashboard and + conversation inside it. + +
+ +
+ {/if} +
+
+ {:else if section === 'members' && appState.activeWorkspace} +
+
+

Members

+

Control who can ask questions, manage connections, and edit dashboards.

+
+
+ {#each members as member} +
+
+
+ {(member.email || 'U').slice(0, 2).toUpperCase()} +
+
+
{member.email || member.user_id}
+
+ Joined {member.joined_at || member.created_at + ? new Date(member.joined_at || member.created_at).toLocaleDateString() + : '—'} +
+
+
+
+ {#if isAdmin && member.role !== 'owner'} + + {#if isOwner} + + {/if} + + {:else} + {member.role} + {/if} +
+
+ {/each} +
+
+ What each role can do +
+ {#each ROLE_HELP as help} + + + {#if openRoleHelp === help.role} + + {help.summary} +
    + {#each help.can as line}
  • {line}
    • {/each} +
+ + {/if} + + {/each} +
+
+
+ {:else if section === 'invites' && isAdmin && appState.activeWorkspace} +
+
+

Invite teammates

+

+ One address or many — separate them with commas, spaces or new lines. + They’ll join after accepting. +

+
+
+ + +
+ + +
+
+ {#if inviteResults} +
+ {#each inviteResults as r} +
+ {r.email || '(blank)'} + + {r.status === 'invited' + ? 'Invited' + : r.status === 'already_member' + ? 'Already a member' + : r.status === 'duplicate' + ? 'Duplicate' + : r.status === 'invalid' + ? 'Not a valid email' + : r.detail || 'Failed'} + +
+ {/each} +
+ {/if} +
+ +
+
+

Pending invitations

+

Invites that haven’t been accepted yet. Resend to extend the expiry.

+
+ {#if pendingInvites.length === 0} +

No outstanding invitations.

+ {:else} +
+ {#each pendingInvites as invite} +
+
+
{invite.email}
+
+ {invite.role} · expires {new Date( + invite.expires_at, + ).toLocaleDateString()}{invite.invited_by + ? ` · invited by ${invite.invited_by}` + : ''} +
+
+
+ + +
+
+ {/each} +
+ {/if} +
+ {:else if section === 'activity' && isAdmin} +
+
+

Activity log

+

Recent administrative events in this workspace.

+
+ +
+ {:else if section === 'permissions' && isOwner && appState.activeWorkspace} +
+
+
+

Role permissions

+

+ Pick the lowest role that gets each capability — everyone above it + gets it too. Owners always have every capability. +

+

+ {#if customisedCount === 0} + Every capability is on its default. + {:else} + {customisedCount} capability{customisedCount === 1 ? '' : ' changes'} customised. + {/if} +

+
+
+ + +
+
+
+ + + + + + + + + {#each PERMISSION_RESOURCES as res} + + + + {#each PERMISSIONS_LIST.filter(p => p.category === res.id) as perm} + + + + + {/each} + {/each} + +
CapabilityAvailable to
+ {res.label} + {res.desc} +
+
{perm.name}
+
{perm.description}
+ {perm.key} + 		+ + {#if isDefaultMinRole(perm.key)} + Default + {:else} + Customised + {/if} +
+
+
+ +
+
+ {:else if section === 'workspaces'} +
+
+

Your workspaces

+

Switch context or create a new workspace for another team.

+
+
+ {#each appState.workspaces || [] as ws} +
+
+
{ws.name}
+
{ws.role}
+
+ {#if ws.id === appState.activeWorkspace?.id} + Current + {:else} + + {/if} +
+ {/each} +
+
+ + {#if invites.length > 0} +
+
+

Pending invitations

+
+
+ {#each invites as invite} +
+
+
{invite.workspace_name}
+
Invited as {invite.role}
+
+ +
+ {/each} +
+
+ {/if} + +
+
+

Create workspace

+

Start a new isolated space for another team or environment.

+
+
+
+ + {#if createError} + {createError} + {/if} +
+ +
+
+ +
+
+

Join with invite code

+

Paste a token from an invitation email if you weren’t auto-matched.

+
+
+ + +
+
+ {/if} +
+
+
+{/if} + + (confirmation = null)} +/> + + diff --git a/frontend/src/routes/workspaces/setup/+page.svelte b/frontend/src/routes/workspaces/setup/+page.svelte new file mode 100644 index 00000000..6c64dc9d --- /dev/null +++ b/frontend/src/routes/workspaces/setup/+page.svelte @@ -0,0 +1,238 @@ + + + + Setup Workspace — BoloDB + + +
+
+ + + {#if mode === 'select'} +

Welcome to BoloDB

+

To get started, you'll need a workspace for your databases and team.

+ +
+ + +
+ + {:else if mode === 'create'} +

Name your workspace

+

This is where your databases will live.

+ +
+ + {#if nameError}
{nameError}
{/if} + {#if error && !nameError}
{error}
{/if} + + +
+ + {:else if mode === 'join'} +

Join a workspace

+

Enter the invite code you received.

+ +
+ + {#if error}
{error}
{/if} + + +
+ {/if} +
+
+ + diff --git a/frontend/svelte.config.js b/frontend/svelte.config.js index a6fc59a1..ca56fd3f 100644 --- a/frontend/svelte.config.js +++ b/frontend/svelte.config.js @@ -12,7 +12,12 @@ const config = { // server ships a build with different content-hashed chunks. This lets // us force a full-page reload before navigating into a route whose old // lazily-loaded chunk no longer exists (stale-chunk-after-deploy). - pollInterval: 60000, + // + // Only in production. A dev server rebuilds constantly, so polling there + // reports a "new deploy" every minute and turns ordinary in-app + // navigation into full page reloads — which throws away the sidebar and + // refetches conversations on every click. + pollInterval: process.env.NODE_ENV === "production" ? 60000 : 0, }, }, }; diff --git a/nginx/DOCKERFILE b/nginx/DOCKERFILE index 8fa3c41f..0f883486 100644 --- a/nginx/DOCKERFILE +++ b/nginx/DOCKERFILE @@ -1,20 +1,42 @@ # ────────────────────────────────────────────── -# BoloDB · Nginx Reverse Proxy -# Serves Svelte build + proxies /api → backend +# BoloDB · Nginx (production) +# Builds the SvelteKit static frontend, then +# serves it and reverse-proxies /api → backend. # ────────────────────────────────────────────── -# Build context: ./nginx +# Build context: repository root (.) # ────────────────────────────────────────────── +FROM node:26-alpine AS frontend-builder + +WORKDIR /app + +COPY frontend/package.json frontend/package-lock.json* ./ +RUN npm ci + +COPY frontend/ . + +# Workaround for Coolify multi-stage build context missing files in second stage +COPY nginx/nginx.conf ./nginx.conf + + +# SvelteKit static adapter bakes $env/dynamic/public at build time. +ARG PUBLIC_POSTHOG_PROJECT_TOKEN +ARG PUBLIC_POSTHOG_HOST +ENV PUBLIC_POSTHOG_PROJECT_TOKEN=$PUBLIC_POSTHOG_PROJECT_TOKEN \ + PUBLIC_POSTHOG_HOST=$PUBLIC_POSTHOG_HOST + +RUN npm run build + +# ── Runtime: nginx serves static + proxies API ── FROM nginx:1.27-alpine -# Remove default site RUN rm -rf /usr/share/nginx/html/* /etc/nginx/conf.d/default.conf -# Custom config -COPY nginx.conf /etc/nginx/conf.d/default.conf +COPY --from=frontend-builder /app/nginx.conf /etc/nginx/conf.d/default.conf +COPY --from=frontend-builder /app/build /usr/share/nginx/html -EXPOSE 80 +EXPOSE 8080 HEALTHCHECK --interval=30s --timeout=3s --retries=3 \ - CMD wget -q -O /dev/null http://127.0.0.1:80/ || exit 1 + CMD wget -q -O /dev/null http://127.0.0.1:8080/ || exit 1 CMD ["nginx", "-g", "daemon off;"] diff --git a/nginx/DOCKERFILE.dev b/nginx/DOCKERFILE.dev new file mode 100644 index 00000000..13063c19 --- /dev/null +++ b/nginx/DOCKERFILE.dev @@ -0,0 +1,15 @@ +# ────────────────────────────────────────────── +# BoloDB · Nginx (local development) +# Proxies / → Vite frontend, /api → backend +# ────────────────────────────────────────────── +# Build context: ./nginx +# ────────────────────────────────────────────── +FROM nginx:1.27-alpine + +RUN rm -rf /usr/share/nginx/html/* /etc/nginx/conf.d/default.conf + +COPY nginx.dev.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 + +CMD ["nginx", "-g", "daemon off;"] diff --git a/nginx/nginx.conf b/nginx/nginx.conf index ad2c2049..b27eb837 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,19 +1,9 @@ server { - listen 80; + listen 8080; server_name _; - # ── Proxy frontend requests to Vite dev server ── - location / { - proxy_pass http://frontend:5173; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $http_connection; - proxy_buffering off; - } + root /usr/share/nginx/html; + index index.html; # ── Reverse-proxy API calls to the FastAPI backend ── location /api/ { @@ -33,6 +23,23 @@ server { proxy_send_timeout 30s; } + # SvelteKit data requests for client-only routes + location ~ /__data\.json { + try_files $uri @empty_data; + } + + location @empty_data { + add_header Content-Type application/json; + return 200 '{"type":"data","nodes":[null,null,null]}'; + } + + # ── Static SPA (adapter-static) ── + location / { + # /index.html is the prerendered marketing homepage; client-only + # routes fall back to the SPA shell in 200.html. + try_files $uri $uri.html $uri/ /200.html; + } + # ── Gzip compression ── gzip on; gzip_types text/plain text/css application/json application/javascript text/xml; diff --git a/nginx/nginx.dev.conf b/nginx/nginx.dev.conf new file mode 100644 index 00000000..57f76b7b --- /dev/null +++ b/nginx/nginx.dev.conf @@ -0,0 +1,43 @@ +server { + listen 80; + server_name _; + + # ── Proxy frontend requests to Vite dev server ── + # Variable + resolver so nginx starts even if frontend + # is still coming up (avoids "host not found in upstream"). + resolver 127.0.0.11 valid=10s ipv6=off; + set $frontend_upstream frontend:5173; + + location / { + proxy_pass http://$frontend_upstream; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $http_connection; + proxy_buffering off; + } + + location /api/ { + set $backend_upstream backend:4321; + proxy_pass http://$backend_upstream; + proxy_http_version 1.1; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Cookie $http_cookie; + proxy_pass_header Set-Cookie; + + proxy_read_timeout 120s; + proxy_connect_timeout 10s; + proxy_send_timeout 30s; + } + + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml; + gzip_min_length 256; +} diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..d89ebc77 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,10 @@ +# Pin the lint rule set so it does not drift with the ruff version. +# +# The repo previously had no ruff config, so linting used whatever defaults the +# installed ruff shipped. Ruff 0.16.0 broadened its defaults (adding B, UP, I, +# DTZ and more), and because CI installs the latest ruff, the whole backend +# started failing there while an older local ruff still passed. These four are +# ruff's historical defaults — the set this codebase was written against — now +# stated explicitly so every ruff version enforces the same rules. +[lint] +select = ["E4", "E7", "E9", "F"] diff --git a/test_reports/iteration_1.json b/test_reports/iteration_1.json deleted file mode 100644 index a75dda8a..00000000 --- a/test_reports/iteration_1.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "summary": "Comprehensive frontend testing of BoloDB landing page + onboarding (signup/login/connect). All 16 review-request checkpoints verified. All data-testids present, password validation & visibility toggle working, theme switch working, skip-to-content link functional, cross-navigation working, error alerts render with role='alert', mobile responsive (375x800) with no horizontal overflow, Connect page testids fully present. Backend is degraded (Postgres unavailable — expected per env note) so signup POST returns 502; the frontend surfaces this correctly in a role='alert' error message.", - "backend_issues": { - "critical": [], - "minor": [ - { - "endpoint": "/api/auth/signup", - "issue": "Returns 500/502 because PostgreSQL is not available (EXPECTED per environment note, degraded mode)" - }, - { - "endpoint": "/api/health", - "issue": "Returns status=degraded, postgres=disconnected (EXPECTED)" - } - ] - }, - "frontend_issues": { - "ui_bugs": [], - "integration_issues": [], - "design_issues": [ - { - "screen": "Landing hero (mobile 375px)", - "issues": [ - "'SQL Server' label in the trust-strip DB list wraps onto two lines at ~375px due to gap:12px between logos — cosmetic only, still readable; consider reducing gap or shrinking font on <420px to keep the list on a single line." - ] - }, - { - "screen": "MarketingNav.svelte", - "issues": [ - "Unused local variable `const before = appState.theme;` in handleThemeToggle() — dead code, minor cleanup." - ] - } - ] - }, - "test_report_links": [], - "action_items": [ - "(Optional cosmetic) Tighten trust-strip DB list on very narrow screens so 'SQL Server' does not wrap.", - "(Nit) Remove unused `const before = appState.theme` in MarketingNav.svelte handleThemeToggle().", - "No functional fixes needed — landing + onboarding pass all optimization/UX/data-testid/accessibility acceptance criteria in this env." - ], - "critical_code_review_comments": [ - "SignupScreen.svelte and LoginScreen.svelte contain large inline style attributes rather than scoped CSS/classes; while functional, this hurts readability and reuse. Consider extracting shared auth-form styles into a class.", - "ConnectScreen.svelte imports `onMount` from svelte but the file uses `$effect`/runes elsewhere — mixing runes and legacy lifecycle is fine but be consistent for maintainability.", - "Error UI uses hardcoded hex colors (#FFF8ED, #F5D78A, #7A5C0A) instead of theme tokens — will not adapt cleanly to a light/dark theme change on the signup/login error banners.", - "The `handleSupabaseGoogle` in GoogleSignIn.svelte will show 'Google sign-in is not configured' when supabase config is null — good UX; but the error surface uses plain
without role='alert' (login/signup have it). Consider adding role='alert' for consistency.", - "Positive: excellent data-testid coverage across all interactive elements, proper aria-live regions, skip-to-content link, aria-pressed on DB type buttons, autocomplete hints on all form fields." - ], - "updated_files": [], - "success_rate": { - "backend": "N/A (frontend-only testing per request; backend intentionally degraded)", - "frontend": "100% (16/16 checkpoints)" - }, - "test_credentials": "N/A - no auth credentials needed; test_credentials.md was not present.", - "seed_data_creation": "None", - "retest_needed": false, - "main_agent_can_self_test": true, - "context_for_next_testing_agent": "Frontend fully verified against the 16 review-request bullets. All data-testids on landing, signup, login, connect are present. Password validation transitions correctly (green checks). Password toggle switches input type. Theme toggle switches document theme attribute (dark <-> crisp). Skip link is off-screen normally (top:-50px) and moves to top:8px on focus. Backend is degraded because PostgreSQL is not configured in this env — that is expected. No further frontend testing needed; only minor cosmetic/dead-code polish items remain.", - "rca of the issue": "No functional issues found. Signup 502 is due to backend PostgreSQL unavailability which is by design in this preview env, and the frontend handles it gracefully via the role='alert' error banner." -} diff --git a/tests/test_activity_cleanup.py b/tests/test_activity_cleanup.py new file mode 100644 index 00000000..2e7bf581 --- /dev/null +++ b/tests/test_activity_cleanup.py @@ -0,0 +1,113 @@ +"""Tests for the activity-log retention job (``delete_old_activity`` and the +``activity_cleanup_loop`` that drives it from the app lifespan). +""" + +import asyncio +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, Mock + +import pytest + +import backend.app.controllers.activity as activity + + +def _patch_session(monkeypatch, session): + factory = Mock() + factory.return_value = session + session.__aenter__.return_value = session + monkeypatch.setattr(activity, "async_session", factory) + return factory + + +def _session_with_rowcount(monkeypatch, rowcount): + session = AsyncMock() + result = Mock() + result.rowcount = rowcount + session.execute.return_value = result + _patch_session(monkeypatch, session) + return session + + +@pytest.mark.asyncio +async def test_delete_old_activity_returns_rows_removed(monkeypatch): + _session_with_rowcount(monkeypatch, 12) + assert await activity.delete_old_activity() == 12 + + +@pytest.mark.asyncio +async def test_delete_old_activity_handles_a_null_rowcount(monkeypatch): + # Some drivers report -1/None rather than a count; that is "nothing to + # report", not a crash. + _session_with_rowcount(monkeypatch, None) + assert await activity.delete_old_activity() == 0 + + +@pytest.mark.asyncio +async def test_delete_old_activity_cuts_off_at_the_retention_window(monkeypatch): + session = _session_with_rowcount(monkeypatch, 0) + before = datetime.now(timezone.utc) + + await activity.delete_old_activity(retention_days=7) + + stmt = session.execute.await_args.args[0] + # The compiled WHERE clause carries the cutoff as a bound parameter. + cutoff = next(v for v in stmt.compile().params.values() if isinstance(v, datetime)) + expected = before - timedelta(days=7) + assert abs((cutoff - expected).total_seconds()) < 5 + + +@pytest.mark.asyncio +async def test_delete_old_activity_respects_an_explicit_zero(monkeypatch): + """`retention_days=0` means "keep nothing", not "fall back to the default".""" + session = _session_with_rowcount(monkeypatch, 0) + before = datetime.now(timezone.utc) + + await activity.delete_old_activity(retention_days=0) + + stmt = session.execute.await_args.args[0] + cutoff = next(v for v in stmt.compile().params.values() if isinstance(v, datetime)) + assert abs((cutoff - before).total_seconds()) < 5 + + +@pytest.mark.asyncio +async def test_cleanup_loop_survives_a_failing_run(monkeypatch): + """A broken prune must not be able to take the server down.""" + calls = [] + + async def flaky(): + calls.append(1) + if len(calls) == 1: + raise RuntimeError("database is down") + return 0 + + monkeypatch.setattr(activity, "delete_old_activity", flaky) + + # A tiny interval keeps the test fast; the loop runs until cancelled. + task = asyncio.create_task(activity.activity_cleanup_loop(interval_hours=1 / 3600)) + while len(calls) < 3: + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert len(calls) >= 3 + + +@pytest.mark.asyncio +async def test_cleanup_loop_stops_on_cancellation(monkeypatch): + monkeypatch.setattr(activity, "delete_old_activity", AsyncMock(return_value=0)) + + task = asyncio.create_task(activity.activity_cleanup_loop(interval_hours=1)) + await asyncio.sleep(0.02) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task.cancelled() + + +@pytest.mark.asyncio +async def test_cleanup_old_activity_logs_alias(monkeypatch): + _session_with_rowcount(monkeypatch, 5) + assert await activity.cleanup_old_activity_logs(retention_days=10) == 5 + assert await activity.cleanup_old_activity_logs(None, 10) == 5 + assert await activity.cleanup_old_activity_logs(10) == 5 diff --git a/tests/test_catalog_controller.py b/tests/test_catalog_controller.py index 03737ddc..14619c26 100644 --- a/tests/test_catalog_controller.py +++ b/tests/test_catalog_controller.py @@ -1,7 +1,7 @@ """Tests for the semantic-catalog controller and routes (backend/app/ controllers/catalog.py, backend/app/routes/catalog.py). -``get_catalog`` / ``save_catalog`` became async and now pass ``user_id`` +``get_catalog`` / ``save_catalog`` became async and now pass ``workspace_id`` through to the (user, db)-scoped KnowledgeService, and the routes had to start awaiting them. """ @@ -21,12 +21,12 @@ def __init__(self, connected=True): self._connected = connected self.db_id_calls = [] - def connected(self, user_id): + def connected(self, workspace_id, db_id=None): return self._connected - def get_db_id(self, user_id): - self.db_id_calls.append(user_id) - return f"db-{user_id}" + def get_db_id(self, workspace_id, db_id=None): + self.db_id_calls.append((workspace_id, db_id)) + return db_id or f"db-{workspace_id}" class DummyKB: @@ -35,12 +35,12 @@ def __init__(self, catalog=None): self.get_calls = [] self.set_calls = [] - async def get_catalog(self, user_id, db_id): - self.get_calls.append((user_id, db_id)) + async def get_catalog(self, workspace_id, db_id): + self.get_calls.append((workspace_id, db_id)) return self._catalog - async def set_catalog(self, user_id, db_id, catalog): - self.set_calls.append((user_id, db_id, catalog)) + async def set_catalog(self, workspace_id, db_id, catalog): + self.set_calls.append((workspace_id, db_id, catalog)) # ── controllers/catalog.py ────────────────────────────────────────── @@ -50,16 +50,24 @@ def test_get_catalog_raises_409_when_not_connected(): db = DummyDB(connected=False) kb = DummyKB() with pytest.raises(HTTPException) as exc: - asyncio.run(catalog_ctrl.get_catalog("u1", db, kb)) + asyncio.run(catalog_ctrl.get_catalog("w1", db, kb)) assert exc.value.status_code == 409 -def test_get_catalog_passes_user_id_and_db_id_to_kb(): +def test_get_catalog_passes_workspace_id_and_db_id_to_kb(): db = DummyDB() kb = DummyKB(catalog={"metrics": [{"name": "revenue"}]}) - result = asyncio.run(catalog_ctrl.get_catalog("u1", db, kb)) + result = asyncio.run(catalog_ctrl.get_catalog("w1", db, kb)) assert result == {"catalog": {"metrics": [{"name": "revenue"}]}} - assert kb.get_calls == [("u1", "db-u1")] + assert kb.get_calls == [("w1", "db-w1")] + + +def test_get_catalog_forwards_selected_db_id(): + """The active X-Db-Id must scope the catalog, not the workspace default.""" + db = DummyDB() + kb = DummyKB(catalog={"metrics": []}) + asyncio.run(catalog_ctrl.get_catalog("w1", db, kb, db_id="db-selected")) + assert kb.get_calls == [("w1", "db-selected")] def test_save_catalog_raises_409_when_not_connected(): @@ -67,20 +75,20 @@ def test_save_catalog_raises_409_when_not_connected(): kb = DummyKB() payload = CatalogPayload() with pytest.raises(HTTPException) as exc: - asyncio.run(catalog_ctrl.save_catalog("u1", db, kb, payload)) + asyncio.run(catalog_ctrl.save_catalog("w1", db, kb, payload)) assert exc.value.status_code == 409 -def test_save_catalog_persists_payload_with_user_id_and_returns_ok(): +def test_save_catalog_persists_payload_with_workspace_id_and_returns_ok(): db = DummyDB() kb = DummyKB() payload = CatalogPayload(metrics=[{"name": "revenue", "sql_expression": "SUM(x)"}]) - result = asyncio.run(catalog_ctrl.save_catalog("u1", db, kb, payload)) + result = asyncio.run(catalog_ctrl.save_catalog("w1", db, kb, payload)) assert result == {"ok": True} assert len(kb.set_calls) == 1 - user_id, db_id, catalog = kb.set_calls[0] - assert user_id == "u1" - assert db_id == "db-u1" + workspace_id, db_id, catalog = kb.set_calls[0] + assert workspace_id == "w1" + assert db_id == "db-w1" assert catalog["metrics"][0]["name"] == "revenue" @@ -88,65 +96,79 @@ def test_save_catalog_persists_payload_with_user_id_and_returns_ok(): def test_get_catalog_route_awaits_controller_and_returns_value(monkeypatch): - async def fake_get_catalog(user_id, db, kb): - assert user_id == "u1" + async def fake_get_catalog(workspace_id, db, kb, db_id=None): + assert workspace_id == "w1" return {"catalog": {"metrics": []}} monkeypatch.setattr(catalog_routes.ctrl, "get_catalog", fake_get_catalog) result = asyncio.run( - catalog_routes.get_catalog(user_token={"user_id": "u1"}, db="db", kb="kb") + catalog_routes.get_catalog( + workspace={"workspace_id": "w1"}, db="db", kb="kb", db_id=None + ) ) assert result == {"catalog": {"metrics": []}} def test_get_catalog_route_reraises_http_exception(monkeypatch): - async def fake_get_catalog(user_id, db, kb): + async def fake_get_catalog(workspace_id, db, kb, db_id=None): raise HTTPException(409, "No database connected") monkeypatch.setattr(catalog_routes.ctrl, "get_catalog", fake_get_catalog) with pytest.raises(HTTPException) as exc: asyncio.run( - catalog_routes.get_catalog(user_token={"user_id": "u1"}, db="db", kb="kb") + catalog_routes.get_catalog( + workspace={"workspace_id": "w1"}, db="db", kb="kb", db_id=None + ) ) assert exc.value.status_code == 409 def test_get_catalog_route_wraps_generic_exception_as_500(monkeypatch): - async def fake_get_catalog(user_id, db, kb): + async def fake_get_catalog(workspace_id, db, kb, db_id=None): raise RuntimeError("boom") monkeypatch.setattr(catalog_routes.ctrl, "get_catalog", fake_get_catalog) with pytest.raises(HTTPException) as exc: asyncio.run( - catalog_routes.get_catalog(user_token={"user_id": "u1"}, db="db", kb="kb") + catalog_routes.get_catalog( + workspace={"workspace_id": "w1"}, db="db", kb="kb", db_id=None + ) ) assert exc.value.status_code == 500 def test_save_catalog_route_awaits_controller_and_returns_value(monkeypatch): - async def fake_save_catalog(user_id, db, kb, payload): - assert user_id == "u1" + async def fake_save_catalog(workspace_id, db, kb, payload, db_id=None): + assert workspace_id == "w1" assert payload == "payload-obj" return {"ok": True} monkeypatch.setattr(catalog_routes.ctrl, "save_catalog", fake_save_catalog) result = asyncio.run( catalog_routes.save_catalog( - payload="payload-obj", user_token={"user_id": "u1"}, db="db", kb="kb" + payload="payload-obj", + workspace={"workspace_id": "w1"}, + db="db", + kb="kb", + db_id=None, ) ) assert result == {"ok": True} def test_save_catalog_route_wraps_generic_exception_as_500(monkeypatch): - async def fake_save_catalog(user_id, db, kb, payload): + async def fake_save_catalog(workspace_id, db, kb, payload, db_id=None): raise RuntimeError("boom") monkeypatch.setattr(catalog_routes.ctrl, "save_catalog", fake_save_catalog) with pytest.raises(HTTPException) as exc: asyncio.run( catalog_routes.save_catalog( - payload="payload-obj", user_token={"user_id": "u1"}, db="db", kb="kb" + payload="payload-obj", + workspace={"workspace_id": "w1"}, + db="db", + kb="kb", + db_id=None, ) ) assert exc.value.status_code == 500 diff --git a/tests/test_config.py b/tests/test_config.py index 54f25c42..20e37438 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,10 @@ -import json +"""Config is built from the environment and held in memory — no files. + +These tests pin that behaviour: the OpenRouter key comes from the environment +and is never exposed to the frontend, saving is a no-op, and nothing is read +from or written to disk. +""" + from contextlib import contextmanager from unittest.mock import patch @@ -11,82 +17,48 @@ @contextmanager -def _paths(tmp_path, env=None): - config_dir = tmp_path / ".bolodb" - with ( - patch("backend.app.config.CONFIG_DIR", config_dir), - patch("backend.app.config.CONFIG_FILE", config_dir / "config.json"), - patch("backend.app.config.SECRET_FILE", config_dir / ".secret"), - patch.dict("os.environ", {"OPENROUTER_API_KEY": "", **(env or {})}), - ): - yield config_dir +def _env(env=None): + with patch.dict("os.environ", {"OPENROUTER_API_KEY": "", **(env or {})}): + yield def test_default_config_no_api_keys(): cfg = default_config() assert "api_keys" not in cfg - assert "openrouter_key" in cfg assert cfg["openrouter_key"] == "" -def test_load_config_no_file(tmp_path): - with _paths(tmp_path) as config_dir: +def test_load_config_reads_key_from_env(): + with _env({"OPENROUTER_API_KEY": "sk-or-test"}): cfg = load_config() - assert cfg["openrouter_key"] == "" - assert config_dir.exists() + assert cfg["openrouter_key"] == "sk-or-test" -def test_load_config_with_valid_file(tmp_path): - with _paths(tmp_path) as config_dir: - config_dir.mkdir(exist_ok=True) - (config_dir / "config.json").write_text( - json.dumps({"last_db_url": "sqlite:///test.db"}) - ) +def test_load_config_defaults_when_env_unset(): + with _env(): cfg = load_config() - assert cfg["last_db_url"] == "sqlite:///test.db" assert cfg["openrouter_key"] == "" + assert cfg["last_db_url"] == "" -def test_load_config_env_var(tmp_path): - with _paths(tmp_path, env={"OPENROUTER_API_KEY": "sk-or-test"}): - cfg = load_config() - assert cfg["openrouter_key"] == "sk-or-test" +def test_load_config_writes_no_files(tmp_path, monkeypatch): + """The whole point of the change: loading config touches no disk.""" + monkeypatch.chdir(tmp_path) + with _env(): + load_config() + assert list(tmp_path.iterdir()) == [] -def test_save_config_writes_plain_dict(tmp_path): - with _paths(tmp_path) as config_dir: - save_config( - {"openrouter_key": "sk-or-v1-secret", "last_db_url": "sqlite:///test.db"} - ) - saved = json.loads((config_dir / "config.json").read_text()) - assert "openrouter_key" not in saved - assert saved["last_db_url"] == "sqlite:///test.db" +def test_save_config_is_a_noop(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + assert ( + save_config({"openrouter_key": "sk", "last_db_url": "sqlite:///x.db"}) is None + ) + assert list(tmp_path.iterdir()) == [] -def test_public_config(): - cfg = { - "openrouter_key": "sk-or-secret", - "last_db_url": "sqlite:///test.db", - } +def test_public_config_never_exposes_the_key(): + cfg = {"openrouter_key": "sk-or-secret", "last_db_url": "sqlite:///test.db"} pub = public_config(cfg) - assert "openrouter_key" not in pub # never expose the key + assert "openrouter_key" not in pub assert pub["last_db_url"] == "sqlite:///test.db" - - -def test_legacy_data_is_dropped(tmp_path): - """Old configs with api_keys, provider, model fields are cleaned up.""" - with _paths(tmp_path) as config_dir: - config_dir.mkdir(exist_ok=True) - (config_dir / "config.json").write_text( - json.dumps( - { - "provider": "gemini", - "model": "gemini-flash-latest", - "api_keys": {"user-1": {"gemini": "AIza-old"}}, - } - ) - ) - cfg = load_config() - assert "api_keys" not in cfg - assert "provider" not in cfg - assert "model" not in cfg diff --git a/tests/test_connection_encryption.py b/tests/test_connection_encryption.py new file mode 100644 index 00000000..0a2e5b0e --- /dev/null +++ b/tests/test_connection_encryption.py @@ -0,0 +1,87 @@ +"""Tests for recent-connection URL encryption. + +The key comes only from RECENT_CONNECTIONS_KEY — nothing is read from or written +to disk. A stored URL must round-trip under that key, legacy rows written when +JWT_SECRET was the key must still decrypt, and a missing key must fail loudly +rather than silently dropping the save. +""" + +import base64 +import hashlib + +import pytest +from cryptography.fernet import Fernet + +import backend.app.pgdatabase.connections as conns +from backend.app.pgdatabase.connections import ConnectionKeyError + + +def _fernet_for(secret: str) -> Fernet: + return Fernet(base64.urlsafe_b64encode(hashlib.sha256(secret.encode()).digest())) + + +@pytest.fixture(autouse=True) +def _clear_cipher_cache(monkeypatch): + monkeypatch.setattr(conns, "_RECENT_CIPHER", None) + for var in ("RECENT_CONNECTIONS_KEY", "JWT_SECRET"): + monkeypatch.delenv(var, raising=False) + yield + conns._RECENT_CIPHER = None + + +def test_roundtrip_with_env_key(monkeypatch): + monkeypatch.setenv("RECENT_CONNECTIONS_KEY", "a-direct-key") + token = conns._encrypt_connection_url("postgresql://u:p@host/db") + assert token != "postgresql://u:p@host/db" + assert conns._decrypt_connection_url(token) == "postgresql://u:p@host/db" + + +def test_missing_key_raises_connection_key_error(monkeypatch): + """No key set is a server misconfiguration, surfaced as a distinct type.""" + with pytest.raises(ConnectionKeyError, match="RECENT_CONNECTIONS_KEY"): + conns._build_recent_connection_cipher() + + +def test_encrypt_without_key_raises_connection_key_error(monkeypatch): + with pytest.raises(ConnectionKeyError): + conns._encrypt_connection_url("postgresql://u:p@host/db") + + +def test_falls_back_to_jwt_secret_for_legacy_rows(monkeypatch): + """Rows written when JWT_SECRET was the key must still decrypt. + + The current key is set to something else, so only the legacy fallback can + read these — the point being that old saved connections survive the move to + an env-only key. + """ + monkeypatch.setenv("RECENT_CONNECTIONS_KEY", "the-new-key") + monkeypatch.setenv("JWT_SECRET", "the-old-secret") + legacy = _fernet_for("the-old-secret").encrypt(b"mysql://u:p@host/db").decode() + + assert conns._decrypt_connection_url(legacy) == "mysql://u:p@host/db" + + +def test_missing_key_does_not_block_the_legacy_path(monkeypatch): + """Even with no current key, a legacy JWT_SECRET row must still decrypt. + + Building the cipher raises ConnectionKeyError; that must be caught so the + fallback still runs rather than the whole read failing. + """ + monkeypatch.setenv("JWT_SECRET", "the-old-secret") + legacy = _fernet_for("the-old-secret").encrypt(b"mysql://u:p@host/db").decode() + + assert conns._decrypt_connection_url(legacy) == "mysql://u:p@host/db" + + +def test_plaintext_url_passes_through(monkeypatch): + monkeypatch.setenv("RECENT_CONNECTIONS_KEY", "a-direct-key") + assert ( + conns._decrypt_connection_url("postgresql://u:p@host/db") + == "postgresql://u:p@host/db" + ) + + +def test_undecryptable_value_raises(monkeypatch): + monkeypatch.setenv("RECENT_CONNECTIONS_KEY", "a-direct-key") + with pytest.raises(RuntimeError, match="Failed to decrypt"): + conns._decrypt_connection_url("not-a-token-and-not-a-url") diff --git a/tests/test_connection_rehydration.py b/tests/test_connection_rehydration.py new file mode 100644 index 00000000..cb2e9d9f --- /dev/null +++ b/tests/test_connection_rehydration.py @@ -0,0 +1,271 @@ +"""A workspace's database must survive the process that connected it. + +Live engines are held in memory. Before this, a restart, a redeploy, or a +request served by a different worker left none — and every screen read that as +the user having disconnected, bouncing them to the connect screen with their +database apparently gone. The credentials are already stored against the +workspace, so the connection is re-established from them instead. + +These tests drive `ensure_connection` directly and then check the behaviour +`/api/state` depends on, since that endpoint is what decides whether the app +shows a database at all. +""" + +import asyncio + +import pytest + +from backend.app.controllers import database as db_ctrl +from backend.app.controllers import system as system_ctrl + +WORKSPACE = "workspace-1" +DB_ID = "abc123def456" +OTHER_DB_ID = "999888777666" +STORED_URL = "postgresql://reader:secret@db.example.com:5432/analytics" + + +class RestartedDB: + """A manager that has lost its connections, as after a process restart.""" + + def __init__(self, connected_ids=(), connect_ok=True): + self.connected_ids = set(connected_ids) + self.connect_ok = connect_ok + self.connect_calls = [] + + def connected(self, workspace_id, db_id=None): + if db_id: + return db_id in self.connected_ids + return bool(self.connected_ids) + + def get_db_id(self, workspace_id, db_id=None): + return db_id or next(iter(self.connected_ids)) + + def connect(self, workspace_id, url): + self.connect_calls.append((workspace_id, url)) + if not self.connect_ok: + return {"ok": False, "error": "could not connect to server"} + self.connected_ids.add(DB_ID) + return {"ok": True, "dialect": "postgresql", "tables": 12, "db_id": DB_ID} + + def get_info(self, workspace_id, db_id=None): + return { + "url": "postgresql://reader:***@db.example.com:5432/analytics", + "tables": 12, + } + + def get_dialect(self, workspace_id, db_id=None): + return "postgresql" + + +def _stored(db_id=DB_ID, url=STORED_URL, alias="Analytics"): + return { + "id": "row-1", + "db_id": db_id, + "db_url": url, + "display_url": "postgresql://reader:***@db.example.com:5432/analytics", + "alias_name": alias, + "dialect": "postgresql", + "table_count": 12, + } + + +@pytest.fixture +def stored_connection(monkeypatch): + """Install a stored connection and report which lookups were used.""" + seen = {"by_db_id": [], "latest": []} + + def install(by_db_id=None, latest=None, raises=False): + async def fake_by_db_id(workspace_id, db_id): + seen["by_db_id"].append((workspace_id, db_id)) + if raises: + raise RuntimeError("Failed to decrypt stored connection URL") + return by_db_id + + async def fake_latest(workspace_id): + seen["latest"].append(workspace_id) + if raises: + raise RuntimeError("Failed to decrypt stored connection URL") + return latest + + monkeypatch.setattr( + db_ctrl.mdb, "get_recent_connection_by_db_id", fake_by_db_id + ) + monkeypatch.setattr(db_ctrl.mdb, "get_latest_recent_connection", fake_latest) + return seen + + return install + + +def test_live_connection_is_used_without_touching_storage(stored_connection): + seen = stored_connection(by_db_id=_stored()) + db = RestartedDB(connected_ids={DB_ID}) + + resolved = asyncio.run(db_ctrl.ensure_connection(db, WORKSPACE, DB_ID)) + + assert resolved == DB_ID + assert db.connect_calls == [], "a connected database must not be reconnected" + assert seen["by_db_id"] == [], "stored credentials shouldn't be read when connected" + + +def test_requested_database_is_restored_from_stored_credentials(stored_connection): + stored_connection(by_db_id=_stored()) + db = RestartedDB() # nothing connected — as after a restart + + resolved = asyncio.run(db_ctrl.ensure_connection(db, WORKSPACE, DB_ID)) + + assert resolved == DB_ID + assert db.connect_calls == [(WORKSPACE, STORED_URL)], ( + "the stored URL should have been used to reconnect" + ) + + +def test_without_a_requested_id_the_most_recent_connection_is_restored( + stored_connection, +): + seen = stored_connection(latest=_stored()) + db = RestartedDB() + + resolved = asyncio.run(db_ctrl.ensure_connection(db, WORKSPACE, None)) + + assert resolved == DB_ID + assert seen["latest"] == [WORKSPACE] + assert db.connect_calls == [(WORKSPACE, STORED_URL)] + + +def test_no_stored_connection_means_no_database(stored_connection): + stored_connection(by_db_id=None, latest=None) + db = RestartedDB() + + assert asyncio.run(db_ctrl.ensure_connection(db, WORKSPACE, DB_ID)) is None + assert asyncio.run(db_ctrl.ensure_connection(db, WORKSPACE, None)) is None + assert db.connect_calls == [] + + +def test_unreachable_database_reports_no_connection_rather_than_raising( + stored_connection, +): + """A database that is stored but down must not 500 the whole request.""" + stored_connection(by_db_id=_stored()) + db = RestartedDB(connect_ok=False) + + assert asyncio.run(db_ctrl.ensure_connection(db, WORKSPACE, DB_ID)) is None + assert db.connect_calls == [(WORKSPACE, STORED_URL)] + + +def test_undecryptable_credentials_report_no_connection(stored_connection): + stored_connection(raises=True) + db = RestartedDB() + + assert asyncio.run(db_ctrl.ensure_connection(db, WORKSPACE, DB_ID)) is None + assert db.connect_calls == [] + + +def test_missing_workspace_never_reaches_storage(stored_connection): + seen = stored_connection(by_db_id=_stored()) + + assert asyncio.run(db_ctrl.ensure_connection(RestartedDB(), None, DB_ID)) is None + assert seen["by_db_id"] == [] + + +# ── the sample database self-heals without stored credentials ─────── + + +def test_sample_is_rebuilt_when_targeted_with_nothing_stored( + stored_connection, tmp_path, monkeypatch +): + """Requesting the sample after a restart must work with zero config. + + The sample needs no saved credentials — it's regenerable — so even with no + stored row and no encryption key, targeting its db_id rebuilds and connects + it instead of returning "no database connected". + """ + from backend.app.database import DatabaseManager + from backend import sample_data + + monkeypatch.setattr(sample_data, "_DATA_DIR", tmp_path) + monkeypatch.delenv("RECENT_CONNECTIONS_KEY", raising=False) + stored_connection(by_db_id=None, latest=None) + + db = DatabaseManager() + sample_id = db_ctrl._sample_db_id() + + resolved = asyncio.run(db_ctrl.ensure_connection(db, WORKSPACE, sample_id)) + + assert resolved == sample_id, "the sample must reconnect from its own dump" + assert db.connected(WORKSPACE, sample_id) + + +def test_non_sample_db_id_still_reports_no_connection(stored_connection): + """The rebuild path is only for the sample, not any missing database.""" + stored_connection(by_db_id=None, latest=None) + db = RestartedDB() + + assert asyncio.run(db_ctrl.ensure_connection(db, WORKSPACE, DB_ID)) is None + assert db.connect_calls == [] + + +# ── what the app actually sees ────────────────────────────────────── + + +class RecordingKB: + async def count_verified(self, workspace_id, db_id): + return 3 + + async def trust_level(self, workspace_id, db_id): + return {"level": "Supervised", "verified": 3} + + async def get_glossary(self, workspace_id, db_id): + return [] + + async def get_verified(self, workspace_id, db_id): + return [{"question": "How many orders?"}] + + +def test_state_shows_the_database_again_after_a_restart(monkeypatch, stored_connection): + """The regression the user hit: reconnect, then /api/state says connected.""" + stored_connection(by_db_id=_stored()) + + async def fake_user(user_id): + return {"tour_completed": True} + + async def fake_by_db_id(workspace_id, db_id): + return _stored() + + monkeypatch.setattr(system_ctrl.mdb, "get_user_by_id", fake_user) + monkeypatch.setattr( + system_ctrl.mdb, "get_recent_connection_by_db_id", fake_by_db_id + ) + + db = RestartedDB() # the engine this workspace had is gone + state = asyncio.run( + system_ctrl.get_state( + "user-1", WORKSPACE, DB_ID, db, {"last_db_url": "x"}, RecordingKB() + ) + ) + + assert state["connected"] is True, "a restart must not look like a disconnect" + assert state["database"]["db_id"] == DB_ID + assert state["database"]["alias_name"] == "Analytics", ( + "the header labels the database from this alias" + ) + assert db.connect_calls == [(WORKSPACE, STORED_URL)] + + +def test_state_reports_disconnected_when_nothing_is_stored( + monkeypatch, stored_connection +): + stored_connection(by_db_id=None, latest=None) + + async def fake_user(user_id): + return {"tour_completed": False} + + monkeypatch.setattr(system_ctrl.mdb, "get_user_by_id", fake_user) + + state = asyncio.run( + system_ctrl.get_state( + "user-1", WORKSPACE, None, RestartedDB(), {}, RecordingKB() + ) + ) + + assert state["connected"] is False + assert "database" not in state diff --git a/tests/test_database_controller.py b/tests/test_database_controller.py index 8be77c5e..0351faf8 100644 --- a/tests/test_database_controller.py +++ b/tests/test_database_controller.py @@ -6,11 +6,11 @@ class DummyKB: - async def trust_level(self, user_id, db_id): + async def trust_level(self, workspace_id, db_id): """Return the trust level associated with a database. Parameters: - user_id: The user requesting the trust level. + workspace_id: The user requesting the trust level. db_id: The database identifier. Returns: @@ -18,11 +18,11 @@ async def trust_level(self, user_id, db_id): """ return f"trust:{db_id}" - async def get_glossary(self, user_id, db_id): + async def get_glossary(self, workspace_id, db_id): """Return glossary entries for a database. Parameters: - user_id: The user requesting the glossary. + workspace_id: The user requesting the glossary. db_id: The database identifier. Returns: @@ -30,11 +30,11 @@ async def get_glossary(self, user_id, db_id): """ return [{"db_id": db_id}] - async def count_verified(self, user_id, db_id): + async def count_verified(self, workspace_id, db_id): """Count the verified entries for a database. Parameters: - user_id: Identifier of the user requesting the count. + workspace_id: Identifier of the user requesting the count. db_id: Identifier of the database. Returns: @@ -42,7 +42,7 @@ async def count_verified(self, user_id, db_id): """ return 1 - async def get_verified(self, user_id, db_id): + async def get_verified(self, workspace_id, db_id): """Return a verified starter question for the specified database. Parameters: @@ -53,7 +53,7 @@ async def get_verified(self, user_id, db_id): """ return [{"question": f"starter:{db_id}"}] - async def seed_sample(self, user_id, db_id): + async def seed_sample(self, workspace_id, db_id): return @@ -62,8 +62,8 @@ class DummyDB: def __init__(self): self.args = None - def connect(self, user_id, url): - self.args = (user_id, url) + def connect(self, workspace_id, url): + self.args = (workspace_id, url) return { "ok": True, "dialect": "sqlite", @@ -84,9 +84,9 @@ async def fake_save(**kwargs): cfg = {} req = SimpleNamespace(db_url="sqlite:///tmp.db") - result = asyncio.run(database_ctrl.connect(db, kb, cfg, req, user_id="u1")) + result = asyncio.run(database_ctrl.connect(db, kb, cfg, req, workspace_id="w1")) - assert db.args == ("u1", "sqlite:///tmp.db") + assert db.args == ("w1", "sqlite:///tmp.db") assert result["trust"] == "trust:db-123" assert saved["dialect"] == "sqlite" assert saved["db_id"] == "db-123" @@ -95,7 +95,7 @@ async def fake_save(**kwargs): def test_connect_sample_uses_result_metadata_for_recent_connection(monkeypatch): class DummyDB: - def connect(self, user_id, url): + def connect(self, workspace_id, url): return { "ok": True, "dialect": "sqlite", @@ -104,7 +104,7 @@ def connect(self, user_id, url): "url": "sqlite:///sample.db", } - def get_db_id(self, user_id): + def get_db_id(self, workspace_id): return "sample-db" saved = {} @@ -118,7 +118,7 @@ async def fake_save(**kwargs): monkeypatch.setattr(database_ctrl.mdb, "save_recent_connection", fake_save) result = asyncio.run( - database_ctrl.connect_sample(DummyDB(), DummyKB(), {}, user_id="u1") + database_ctrl.connect_sample(DummyDB(), DummyKB(), {}, workspace_id="w1") ) assert result["is_sample"] is True @@ -127,17 +127,19 @@ async def fake_save(**kwargs): assert saved["table_count"] == 9 -def test_schema_route_passes_user_id_to_controller(monkeypatch): +def test_schema_route_passes_workspace_id_to_controller(monkeypatch): called = {} - async def fake_get_schema(user_id, db, refresh): - called["args"] = (user_id, db, refresh) + async def fake_get_schema(workspace_id, db, refresh, db_id=None): + called["args"] = (workspace_id, db, refresh) return {"ok": True} monkeypatch.setattr(database_routes.ctrl, "get_schema", fake_get_schema) res = asyncio.run( - database_routes.schema(refresh=True, user_token={"user_id": "u1"}, db="db-ref") + database_routes.schema( + refresh=True, workspace={"workspace_id": "w1"}, db="db-ref" + ) ) assert res == {"ok": True} - assert called["args"] == ("u1", "db-ref", True) + assert called["args"] == ("w1", "db-ref", True) diff --git a/tests/test_invite_codes.py b/tests/test_invite_codes.py new file mode 100644 index 00000000..d4900a68 --- /dev/null +++ b/tests/test_invite_codes.py @@ -0,0 +1,57 @@ +"""Tests for workspace invite code generation and normalisation.""" + +import re + +from backend.app.controllers.workspaces import ( + generate_invite_code, + normalize_invite_code, +) + +_FORMAT = re.compile(r"^BOLO-[0-9A-HJKMNP-TV-Z]{4}-[0-9A-HJKMNP-TV-Z]{4}$") + + +def test_generated_code_matches_format(): + for _ in range(50): + assert _FORMAT.match(generate_invite_code()) + + +def test_generated_code_is_short(): + assert len(generate_invite_code()) == 14 + + +def test_ambiguous_letters_are_excluded(): + # I/L/O/U are omitted from the random body so a code survives being read + # aloud or hand-copied. (The fixed BOLO prefix is exempt — it's a word.) + bodies = "".join(generate_invite_code()[5:] for _ in range(200)) + assert not set("ILOU") & set(bodies) + + +def test_codes_are_unique_enough(): + assert len({generate_invite_code() for _ in range(200)}) == 200 + + +def test_normalize_is_case_insensitive(): + assert normalize_invite_code("bolo-k7m2-9xq4") == "BOLO-K7M2-9XQ4" + + +def test_normalize_strips_whitespace_and_regroups(): + assert normalize_invite_code(" BOLOK7M29XQ4 ") == "BOLO-K7M2-9XQ4" + assert normalize_invite_code("BOLO K7M2 9XQ4") == "BOLO-K7M2-9XQ4" + assert normalize_invite_code("BOLO--K7M2--9XQ4\n") == "BOLO-K7M2-9XQ4" + + +def test_normalize_round_trips_a_generated_code(): + code = generate_invite_code() + assert normalize_invite_code(code.lower()) == code + + +def test_legacy_uuid_token_is_left_alone(): + # Pre-existing invites hold lowercase UUIDs, where hyphens are significant. + uuid_token = "a0000000-0000-4000-8000-000000000000" + assert normalize_invite_code(uuid_token) == uuid_token + assert normalize_invite_code(f" {uuid_token} ") == uuid_token + + +def test_normalize_handles_empty_input(): + assert normalize_invite_code("") == "" + assert normalize_invite_code(None) == "" diff --git a/tests/test_knowledge_service.py b/tests/test_knowledge_service.py index a30b6061..d2e5d078 100644 --- a/tests/test_knowledge_service.py +++ b/tests/test_knowledge_service.py @@ -81,7 +81,7 @@ async def test_add_verified_inserts(kbs): session.add.assert_called_once() session.commit.assert_awaited_once() args = session.add.call_args[0][0] - assert args.user_id == USER_ID + assert args.workspace_id == USER_ID assert args.db_id == DB_ID assert args.question == "test question?" diff --git a/tests/test_m2_permissions_adversarial.py b/tests/test_m2_permissions_adversarial.py new file mode 100644 index 00000000..052b2e41 --- /dev/null +++ b/tests/test_m2_permissions_adversarial.py @@ -0,0 +1,433 @@ +"""Adversarial test suite for BoloDB Phase 4 Milestone 2. + +Focuses on: +1. Route permission enforcement for default role capabilities. +2. Custom matrix overrides (disabling/enabling capabilities per role like dashboards.create or queries.execute). +3. Verification that disabling a permission for Member or Admin results in HTTP 403. +4. Settings update authorization (Admin and Member get HTTP 403 on PATCH /api/workspaces/{id}/settings). +5. Owner non-editable bypass verification across routes. +6. Edge case & malformed input handling in permissions dependency. +""" + +import uuid +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from fastapi import HTTPException +from fastapi.testclient import TestClient + +from backend.app.server import create_app +from backend.app.dependencies import ( + get_current_user, + get_current_workspace, + require_permission, +) +from backend.app.models.workspace_settings import WorkspaceSettings + +WORKSPACE_ID = str(uuid.uuid4()) +OWNER_ID = str(uuid.uuid4()) +ADMIN_ID = str(uuid.uuid4()) +MEMBER_ID = str(uuid.uuid4()) + + +def _mock_db_result(value): + res = Mock(name="QueryResult") + res.scalar_one_or_none = Mock(return_value=value) + return res + + +@pytest.fixture +def app_instance(): + """Create a FastAPI app instance with rate limiting / lifespan dependencies disabled for test simplicity.""" + with patch("backend.app.server.lifespan"): + app = create_app(readonly=True) + # Clear rate limiters if any to prevent 429 during fast test loops + app.state.limiter.enabled = False + return app + + +# ===================================================================== +# 1. ROUTE PERMISSION ENFORCEMENT & SETTINGS AUTHORIZATION (HTTP TESTS) +# ===================================================================== + + +def test_update_settings_admin_returns_403(app_instance, monkeypatch): + """Task 3: Verify that updating settings as Admin returns HTTP 403.""" + client = TestClient(app_instance) + + # Override get_current_workspace to simulate Admin role context + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "admin", + "user_id": ADMIN_ID, + } + + payload = {"default_invite_role": "admin", "invite_expiry_days": 14} + res = client.patch(f"/api/workspaces/{WORKSPACE_ID}/settings", json=payload) + + assert res.status_code == 403 + assert "Owner role required" in res.json()["detail"] + app_instance.dependency_overrides.clear() + + +def test_update_settings_member_returns_403(app_instance, monkeypatch): + """Task 3: Verify that updating settings as Member returns HTTP 403.""" + client = TestClient(app_instance) + + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + + payload = {"default_invite_role": "member", "invite_expiry_days": 30} + res = client.patch(f"/api/workspaces/{WORKSPACE_ID}/settings", json=payload) + + assert res.status_code == 403 + assert "Owner role required" in res.json()["detail"] + app_instance.dependency_overrides.clear() + + +def test_update_settings_owner_succeeds(app_instance, monkeypatch): + """Verify that updating settings as Owner succeeds with HTTP 200.""" + client = TestClient(app_instance) + + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "owner", + "user_id": OWNER_ID, + } + + mock_update = AsyncMock( + return_value={ + "workspace_id": WORKSPACE_ID, + "default_invite_role": "admin", + "invite_expiry_days": 14, + "activity_retention_days": 60, + "role_permissions": {}, + "resolved_matrix": {}, + } + ) + monkeypatch.setattr( + "backend.app.controllers.workspaces.update_workspace_settings", mock_update + ) + + payload = {"default_invite_role": "admin", "invite_expiry_days": 14} + res = client.patch(f"/api/workspaces/{WORKSPACE_ID}/settings", json=payload) + + assert res.status_code == 200 + assert res.json()["default_invite_role"] == "admin" + app_instance.dependency_overrides.clear() + + +def test_member_disabled_dashboards_create_returns_403(app_instance, monkeypatch): + """Task 2: Verify that default or disabled dashboards.create for Member returns HTTP 403.""" + client = TestClient(app_instance) + + # Mock DB session in require_permission to return standard default settings (role_permissions={}) + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), role_permissions={} + ) + session = AsyncMock() + session.execute.return_value = _mock_db_result(settings) + + factory = Mock(return_value=session) + session.__aenter__.return_value = session + monkeypatch.setattr("backend.app.dependencies.async_session", factory) + + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + app_instance.dependency_overrides[get_current_user] = lambda: { + "user_id": MEMBER_ID, + "sub": MEMBER_ID, + } + + res = client.post("/api/dashboards", json={"name": "Forbidden Dashboard"}) + assert res.status_code == 403 + assert "Permission 'dashboards.create' required" in res.json()["detail"] + app_instance.dependency_overrides.clear() + + +def test_member_disabled_queries_execute_via_override_returns_403( + app_instance, monkeypatch +): + """Task 2: Verify that disabling queries.execute for Member via custom matrix override returns HTTP 403.""" + client = TestClient(app_instance) + + # Custom matrix override disabling queries.execute for member + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": {"queries.execute": False}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_db_result(settings) + + factory = Mock(return_value=session) + session.__aenter__.return_value = session + monkeypatch.setattr("backend.app.dependencies.async_session", factory) + + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + + # 1. Check POST /api/query + res_query = client.post("/api/query", json={"question": "SELECT 1"}) + assert res_query.status_code == 403 + assert "Permission 'queries.execute' required" in res_query.json()["detail"] + + # 2. Check POST /api/execute + res_exec = client.post("/api/execute", json={"sql": "SELECT 1"}) + assert res_exec.status_code == 403 + assert "Permission 'queries.execute' required" in res_exec.json()["detail"] + + # 3. Check POST /api/feedback + res_fb = client.post("/api/feedback", json={"query_id": "123", "rating": 5}) + assert res_fb.status_code == 403 + assert "Permission 'queries.execute' required" in res_fb.json()["detail"] + + app_instance.dependency_overrides.clear() + + +def test_admin_disabled_dashboards_create_via_override_returns_403( + app_instance, monkeypatch +): + """Task 1 & 2: Custom matrix override disabling dashboards.create for Admin returns HTTP 403.""" + client = TestClient(app_instance) + + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"admin": {"dashboards.create": False}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_db_result(settings) + + factory = Mock(return_value=session) + session.__aenter__.return_value = session + monkeypatch.setattr("backend.app.dependencies.async_session", factory) + + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "admin", + "user_id": ADMIN_ID, + } + app_instance.dependency_overrides[get_current_user] = lambda: { + "user_id": ADMIN_ID, + "sub": ADMIN_ID, + } + + res = client.post("/api/dashboards", json={"name": "Admin Denied Dashboard"}) + assert res.status_code == 403 + assert "Permission 'dashboards.create' required" in res.json()["detail"] + + app_instance.dependency_overrides.clear() + + +def test_member_granted_dashboards_create_via_override_succeeds( + app_instance, monkeypatch +): + """Task 1: Custom matrix override granting dashboards.create to Member allows access.""" + client = TestClient(app_instance) + + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": {"dashboards.create": True}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_db_result(settings) + + factory = Mock(return_value=session) + session.__aenter__.return_value = session + monkeypatch.setattr("backend.app.dependencies.async_session", factory) + + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + app_instance.dependency_overrides[get_current_user] = lambda: { + "user_id": MEMBER_ID, + "sub": MEMBER_ID, + } + + mock_create = AsyncMock(return_value={"id": "dash-1", "name": "Allowed Dashboard"}) + monkeypatch.setattr( + "backend.app.controllers.dashboards.create_dashboard", mock_create + ) + + res = client.post("/api/dashboards", json={"name": "Allowed Dashboard"}) + assert res.status_code == 200 + assert res.json()["name"] == "Allowed Dashboard" + + app_instance.dependency_overrides.clear() + + +def test_owner_bypasses_custom_matrix_overrides_on_all_routes( + app_instance, monkeypatch +): + """Task 1: Owner role unconditionally bypasses custom matrix overrides trying to disable permissions.""" + client = TestClient(app_instance) + + # Database settings attempt to disable all permissions for Owner (which update_workspace_settings strips, but test DB value here) + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={ + "owner": {"dashboards.create": False, "queries.execute": False} + }, + ) + session = AsyncMock() + session.execute.return_value = _mock_db_result(settings) + + factory = Mock(return_value=session) + session.__aenter__.return_value = session + monkeypatch.setattr("backend.app.dependencies.async_session", factory) + + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "owner", + "user_id": OWNER_ID, + } + app_instance.dependency_overrides[get_current_user] = lambda: { + "user_id": OWNER_ID, + "sub": OWNER_ID, + } + + mock_create = AsyncMock( + return_value={"id": "dash-owner", "name": "Owner Dashboard"} + ) + monkeypatch.setattr( + "backend.app.controllers.dashboards.create_dashboard", mock_create + ) + + res = client.post("/api/dashboards", json={"name": "Owner Dashboard"}) + assert res.status_code == 200 + assert res.json()["name"] == "Owner Dashboard" + + app_instance.dependency_overrides.clear() + + +# ===================================================================== +# 2. ADVERSARIAL & EDGE CASE DEPENDENCY TESTS +# ===================================================================== + + +@pytest.mark.asyncio +async def test_require_permission_missing_settings_uses_default_matrix(monkeypatch): + """When WorkspaceSettings row is missing from DB, require_permission uses default role permissions.""" + session = AsyncMock() + session.execute.return_value = _mock_db_result(None) # No settings record + + factory = Mock(return_value=session) + session.__aenter__.return_value = session + monkeypatch.setattr("backend.app.dependencies.async_session", factory) + + dep_create = require_permission("dashboards.create") + dep_exec = require_permission("queries.execute") + + member_ctx = {"workspace_id": WORKSPACE_ID, "role": "member", "user_id": MEMBER_ID} + admin_ctx = {"workspace_id": WORKSPACE_ID, "role": "admin", "user_id": ADMIN_ID} + + # Member lacks dashboards.create by default -> 403 + with pytest.raises(HTTPException) as exc1: + await dep_create(workspace=member_ctx) + assert exc1.value.status_code == 403 + + # Member has queries.execute by default -> passes + res_exec = await dep_exec(workspace=member_ctx) + assert res_exec == member_ctx + + # Admin has dashboards.create by default -> passes + res_admin = await dep_create(workspace=admin_ctx) + assert res_admin == admin_ctx + + +@pytest.mark.asyncio +async def test_require_permission_invalid_or_malformed_overrides(monkeypatch): + """Test require_permission behavior when role_permissions contains invalid types or malformed structures.""" + malformed_cases = [ + "not_a_dict", + 12345, + [True, False], + {"member": "invalid_inner_string"}, + {"member": {"queries.execute": "yes_string_not_bool"}}, + ] + + for malformed in malformed_cases: + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), role_permissions=malformed + ) + session = AsyncMock() + session.execute.return_value = _mock_db_result(settings) + + factory = Mock(return_value=session) + session.__aenter__.return_value = session + monkeypatch.setattr("backend.app.dependencies.async_session", factory) + + dep_exec = require_permission("queries.execute") + member_ctx = { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + + # Member should still fall back to default queries.execute=True + res = await dep_exec(workspace=member_ctx) + assert res == member_ctx, ( + f"Failed for malformed role_permissions case: {malformed}" + ) + + +@pytest.mark.asyncio +async def test_require_permission_nonexistent_capability_key(monkeypatch): + """Non-existent permission capability keys always raise HTTP 403 for non-owner roles.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"admin": {"fake.key": True}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_db_result(settings) + + factory = Mock(return_value=session) + session.__aenter__.return_value = session + monkeypatch.setattr("backend.app.dependencies.async_session", factory) + + dep_fake = require_permission("fake.capability.key") + admin_ctx = {"workspace_id": WORKSPACE_ID, "role": "admin", "user_id": ADMIN_ID} + + with pytest.raises(HTTPException) as exc: + await dep_fake(workspace=admin_ctx) + assert exc.value.status_code == 403 + assert "Permission 'fake.capability.key' required" in exc.value.detail + + +@pytest.mark.asyncio +async def test_delete_workspace_and_transfer_ownership_authorization( + app_instance, monkeypatch +): + """Verify delete workspace and transfer ownership endpoints require Owner role.""" + client = TestClient(app_instance) + + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "admin", + "user_id": ADMIN_ID, + } + + # Delete workspace as Admin -> 403 + res_del = client.delete(f"/api/workspaces/{WORKSPACE_ID}") + assert res_del.status_code == 403 + assert "Owner role required" in res_del.json()["detail"] + + # Transfer ownership as Admin -> 403 + res_xfer = client.post( + f"/api/workspaces/{WORKSPACE_ID}/transfer-ownership", + json={"user_id": MEMBER_ID}, + ) + assert res_xfer.status_code == 403 + assert "Owner role required" in res_xfer.json()["detail"] + + app_instance.dependency_overrides.clear() diff --git a/tests/test_migration_0005.py b/tests/test_migration_0005.py deleted file mode 100644 index 52191556..00000000 --- a/tests/test_migration_0005.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Tests for the alembic migration that adds the knowledge tables and the -users.tour_completed column (backend/alembic/versions/0005_add_knowledge_tables.py). - -The migration module is loaded directly via importlib (its filename starts -with a digit, so it can't be imported with a normal ``import`` statement), -and ``alembic.op`` is monkeypatched so ``upgrade()``/``downgrade()`` can be -exercised without a real database connection. -""" - -import importlib - -import pytest - -pytest.importorskip("alembic") - -from unittest.mock import Mock - -MODULE_PATH = "backend.alembic.versions.0005_add_knowledge_tables" - -NEW_TABLES = [ - "verified_qas", - "glossary_terms", - "catalog_columns", - "catalog_metrics", - "catalog_joins", - "catalog_synonyms", - "catalog_value_mappings", -] - -NEW_INDEXES = [ - ("ix_verified_qa_user_db", "verified_qas"), - ("ix_glossary_user_db", "glossary_terms"), - ("ix_cat_col_user_db", "catalog_columns"), - ("ix_cat_met_user_db", "catalog_metrics"), - ("ix_cat_join_user_db", "catalog_joins"), - ("ix_cat_syn_user_db", "catalog_synonyms"), - ("ix_cat_val_user_db", "catalog_value_mappings"), -] - - -@pytest.fixture -def migration(): - return importlib.import_module(MODULE_PATH) - - -@pytest.fixture -def fake_op(monkeypatch, migration): - op = Mock() - monkeypatch.setattr(migration, "op", op) - return op - - -def test_revision_metadata(migration): - assert migration.revision == "0005" - assert migration.down_revision == "0004" - - -def test_upgrade_adds_tour_completed_column(fake_op, migration): - migration.upgrade() - - add_column_calls = fake_op.add_column.call_args_list - assert len(add_column_calls) == 1 - table_name, column = add_column_calls[0][0] - assert table_name == "users" - assert column.name == "tour_completed" - assert column.nullable is False - - -def test_upgrade_creates_all_new_tables(fake_op, migration): - migration.upgrade() - - created = [call.args[0] for call in fake_op.create_table.call_args_list] - assert created == NEW_TABLES - - -def test_upgrade_creates_all_composite_indexes_on_user_and_db(fake_op, migration): - migration.upgrade() - - created = [ - (call.args[0], call.args[1], call.args[2]) - for call in fake_op.create_index.call_args_list - ] - for (name, table), (created_name, created_table, cols) in zip(NEW_INDEXES, created): - assert created_name == name - assert created_table == table - assert cols == ["user_id", "db_id"] - assert len(created) == len(NEW_INDEXES) - - -def test_upgrade_creates_one_index_per_new_table(fake_op, migration): - migration.upgrade() - - indexed_tables = {call.args[1] for call in fake_op.create_index.call_args_list} - assert indexed_tables == set(NEW_TABLES) - - -def test_downgrade_drops_new_tables_in_reverse_order(fake_op, migration): - migration.downgrade() - - dropped_tables = [call.args[0] for call in fake_op.drop_table.call_args_list] - assert dropped_tables == list(reversed(NEW_TABLES)) - - -def test_downgrade_drops_tour_completed_column(fake_op, migration): - migration.downgrade() - - fake_op.drop_column.assert_called_once_with("users", "tour_completed") - - -def test_downgrade_drops_column_after_all_tables(fake_op, migration): - """The users.tour_completed column has no FK dependency on the new - tables, but downgrade should still finish table cleanup first.""" - migration.downgrade() - - # Calls to any child of `fake_op` (drop_table, drop_column, ...) are - # automatically recorded on fake_op.mock_calls in chronological order. - call_names = [c[0] for c in fake_op.mock_calls] - assert call_names[-1] == "drop_column" - assert call_names[0] == "drop_constraint" - assert call_names[1:-1] == ["drop_table"] * len(NEW_TABLES) diff --git a/tests/test_onboard_controller.py b/tests/test_onboard_controller.py index 324f7833..ca8afeb5 100644 --- a/tests/test_onboard_controller.py +++ b/tests/test_onboard_controller.py @@ -1,7 +1,7 @@ """Tests for the onboarding controller's ``save`` step (backend/app/ controllers/onboard.py), which now routes every knowledge-base call through the (user, db)-scoped KnowledgeService — every ``kb.*`` call must carry -``user_id`` as the first argument and be awaited.""" +``workspace_id`` as the first argument and be awaited.""" import asyncio from types import SimpleNamespace @@ -17,13 +17,13 @@ def __init__(self, connected=True, schema=None): self._connected = connected self._schema = schema or {} - def connected(self, user_id): + def connected(self, workspace_id): return self._connected - def get_db_id(self, user_id): - return f"db-{user_id}" + def get_db_id(self, workspace_id): + return f"db-{workspace_id}" - def get_schema(self, user_id): + def get_schema(self, workspace_id): return self._schema @@ -35,20 +35,20 @@ def __init__(self, catalog_empty=True): self.catalog_calls = [] self.set_catalog_calls = [] - async def set_glossary(self, user_id, db_id, terms): - self.glossary_calls.append((user_id, db_id, terms)) + async def set_glossary(self, workspace_id, db_id, terms): + self.glossary_calls.append((workspace_id, db_id, terms)) - async def add_verified(self, user_id, db_id, question, sql, restatement): - self.verified_calls.append((user_id, db_id, question, sql, restatement)) + async def add_verified(self, workspace_id, db_id, question, sql, restatement): + self.verified_calls.append((workspace_id, db_id, question, sql, restatement)) - async def catalog_is_empty(self, user_id, db_id): - self.catalog_calls.append((user_id, db_id)) + async def catalog_is_empty(self, workspace_id, db_id): + self.catalog_calls.append((workspace_id, db_id)) return self._catalog_empty - async def set_catalog(self, user_id, db_id, catalog): - self.set_catalog_calls.append((user_id, db_id, catalog)) + async def set_catalog(self, workspace_id, db_id, catalog): + self.set_catalog_calls.append((workspace_id, db_id, catalog)) - async def trust_level(self, user_id, db_id): + async def trust_level(self, workspace_id, db_id): return {"level": "Supervised", "verified": 0, "pct": 8, "note": ""} @@ -69,25 +69,25 @@ def test_save_raises_409_when_not_connected(): db = DummyDB(connected=False) kb = DummyKB() with pytest.raises(HTTPException) as exc: - asyncio.run(onboard_ctrl.save("u1", db, kb, _req())) + asyncio.run(onboard_ctrl.save("w1", db, kb, _req())) assert exc.value.status_code == 409 -def test_save_sets_glossary_with_user_id_and_db_id(): +def test_save_sets_glossary_with_workspace_id_and_db_id(): db = DummyDB() kb = DummyKB() req = _req( glossary=[{"term": "Revenue", "maps_to": "orders.total", "sql_hint": ""}] ) - asyncio.run(onboard_ctrl.save("u1", db, kb, req)) + asyncio.run(onboard_ctrl.save("w1", db, kb, req)) assert len(kb.glossary_calls) == 1 - user_id, db_id, terms = kb.glossary_calls[0] - assert user_id == "u1" - assert db_id == "db-u1" + workspace_id, db_id, terms = kb.glossary_calls[0] + assert workspace_id == "w1" + assert db_id == "db-w1" assert terms == [{"term": "Revenue", "maps_to": "orders.total", "sql_hint": ""}] -def test_save_adds_each_starter_as_verified_with_user_id(): +def test_save_adds_each_starter_as_verified_with_workspace_id(): db = DummyDB() kb = DummyKB() req = _req( @@ -104,18 +104,18 @@ def test_save_adds_each_starter_as_verified_with_user_id(): }, ] ) - asyncio.run(onboard_ctrl.save("u1", db, kb, req)) + asyncio.run(onboard_ctrl.save("w1", db, kb, req)) assert len(kb.verified_calls) == 2 assert kb.verified_calls[0] == ( - "u1", - "db-u1", + "w1", + "db-w1", "How many orders?", "SELECT COUNT(*) FROM orders", "count", ) assert kb.verified_calls[1] == ( - "u1", - "db-u1", + "w1", + "db-w1", "Total revenue?", "SELECT SUM(x) FROM orders", "sum", @@ -128,20 +128,20 @@ def test_save_seeds_catalog_backbone_when_catalog_empty(monkeypatch): monkeypatch.setattr( onboard_ctrl, "suggest_from_schema", lambda schema: {"joins": ["fake-join"]} ) - asyncio.run(onboard_ctrl.save("u1", db, kb, _req())) - assert kb.catalog_calls == [("u1", "db-u1")] + asyncio.run(onboard_ctrl.save("w1", db, kb, _req())) + assert kb.catalog_calls == [("w1", "db-w1")] assert len(kb.set_catalog_calls) == 1 - user_id, db_id, catalog = kb.set_catalog_calls[0] - assert user_id == "u1" - assert db_id == "db-u1" + workspace_id, db_id, catalog = kb.set_catalog_calls[0] + assert workspace_id == "w1" + assert db_id == "db-w1" assert catalog == {"joins": ["fake-join"]} def test_save_does_not_reseed_catalog_when_not_empty(): db = DummyDB() kb = DummyKB(catalog_empty=False) - asyncio.run(onboard_ctrl.save("u1", db, kb, _req())) - assert kb.catalog_calls == [("u1", "db-u1")] + asyncio.run(onboard_ctrl.save("w1", db, kb, _req())) + assert kb.catalog_calls == [("w1", "db-w1")] assert kb.set_catalog_calls == [] @@ -149,11 +149,11 @@ def test_save_returns_trust_level_scoped_to_user_and_db(): db = DummyDB() class TrustKB(DummyKB): - async def trust_level(self, user_id, db_id): + async def trust_level(self, workspace_id, db_id): return {"level": "Trusted", "verified": 9, "pct": 100, "note": "n"} kb = TrustKB() - result = asyncio.run(onboard_ctrl.save("u1", db, kb, _req())) + result = asyncio.run(onboard_ctrl.save("w1", db, kb, _req())) assert result == { "ok": True, "trust": {"level": "Trusted", "verified": 9, "pct": 100, "note": "n"}, diff --git a/tests/test_permissions_and_workspace_settings.py b/tests/test_permissions_and_workspace_settings.py new file mode 100644 index 00000000..c90111e9 --- /dev/null +++ b/tests/test_permissions_and_workspace_settings.py @@ -0,0 +1,343 @@ +import uuid +import pytest +from backend.app.permissions import ( + PERMISSIONS, + DEFAULT_ROLE_PERMISSIONS, + resolve_role_permissions, + has_permission, +) +from backend.app.models.workspace_settings import WorkspaceSettings +from backend.app.models import WorkspaceSettings as ImportedWorkspaceSettings + + +def test_permissions_registry_structure(): + """Verify PERMISSIONS dictionary contains 21 capabilities across 7 resources.""" + assert len(PERMISSIONS) == 21, f"Expected 21 permissions, got {len(PERMISSIONS)}" + + expected_categories = { + "members", + "connections", + "catalog", + "dashboards", + "queries", + "activity", + "workspace_management", + } + categories_found = {perm["category"] for perm in PERMISSIONS.values()} + assert categories_found == expected_categories + + # Verify per-category count + category_counts = {} + for perm in PERMISSIONS.values(): + cat = perm["category"] + category_counts[cat] = category_counts.get(cat, 0) + 1 + + assert category_counts["members"] == 4 + assert category_counts["connections"] == 3 + assert category_counts["catalog"] == 2 + assert category_counts["dashboards"] == 3 + assert category_counts["queries"] == 4 + assert category_counts["activity"] == 2 + assert category_counts["workspace_management"] == 3 + + # Check structure of each entry + for key, spec in PERMISSIONS.items(): + assert spec["key"] == key + assert "name" in spec and isinstance(spec["name"], str) + assert "category" in spec and isinstance(spec["category"], str) + assert "description" in spec and isinstance(spec["description"], str) + assert "default_roles" in spec and isinstance(spec["default_roles"], list) + + +def test_default_role_permissions(): + """Verify default permission maps for owner, admin, and member.""" + # Owner: all 21 True + owner_perms = DEFAULT_ROLE_PERMISSIONS["owner"] + assert len(owner_perms) == 21 + assert all(val is True for val in owner_perms.values()) + + # Admin: all 21 True + admin_perms = DEFAULT_ROLE_PERMISSIONS["admin"] + assert len(admin_perms) == 21 + assert all(val is True for val in admin_perms.values()) + + # Member: exactly 9 True, 12 False + member_perms = DEFAULT_ROLE_PERMISSIONS["member"] + assert len(member_perms) == 21 + + expected_true_member_keys = { + "members.view", + "connections.view", + "connections.view_schema", + "catalog.view", + "dashboards.view", + "queries.execute", + "queries.explain", + "queries.save", + "workspace.view", + } + + actual_true_member_keys = {key for key, val in member_perms.items() if val is True} + assert actual_true_member_keys == expected_true_member_keys + assert len(actual_true_member_keys) == 9 + + false_member_keys = {key for key, val in member_perms.items() if val is False} + assert len(false_member_keys) == 12 + + +def test_resolve_role_permissions_defaults(): + """Test resolve_role_permissions without overrides.""" + resolved_owner = resolve_role_permissions("owner") + assert all(resolved_owner.values()) + + resolved_admin = resolve_role_permissions("admin") + assert all(resolved_admin.values()) + + resolved_member = resolve_role_permissions("member") + assert resolved_member["queries.execute"] is True + assert resolved_member["members.invite"] is False + + resolved_unknown = resolve_role_permissions("guest") + assert all(val is False for val in resolved_unknown.values()) + + +def test_resolve_role_permissions_with_overrides(): + """Test sparse overrides for roles.""" + # Direct dictionary override + overrides = {"members.invite": True, "queries.execute": False} + resolved = resolve_role_permissions("member", overrides) + assert resolved["members.invite"] is True + assert resolved["queries.execute"] is False + assert resolved["catalog.view"] is True # unchanged default + + # Nested role-keyed override + role_keyed_overrides = {"member": {"members.invite": True, "catalog.manage": True}} + resolved_nested = resolve_role_permissions("member", role_keyed_overrides) + assert resolved_nested["members.invite"] is True + assert resolved_nested["catalog.manage"] is True + assert resolved_nested["members.remove"] is False + + +def test_owner_permission_bypass(): + """Owner permissions are non-editable and always remain True.""" + overrides = {"queries.execute": False, "workspace.update": False} + resolved = resolve_role_permissions("owner", overrides) + assert all(val is True for val in resolved.values()) + + assert has_permission("owner", overrides, "queries.execute") is True + assert has_permission("owner", "queries.execute") is True + + +def test_has_permission_utility(): + """Test has_permission helper with positional and keyword arguments.""" + # Single permission check (no overrides) + assert has_permission("member", "queries.execute") is True + assert has_permission("member", "members.invite") is False + assert has_permission("admin", "members.invite") is True + + # With overrides passed as dict + overrides = {"members.invite": True} + assert ( + has_permission("member", overrides=overrides, permission_key="members.invite") + is True + ) + assert has_permission("member", overrides, "members.invite") is True + + # Non-existent permission key + assert has_permission("member", "nonexistent.capability") is False + assert has_permission("owner", "nonexistent.capability") is False + + +def test_workspace_settings_model(): + """Verify WorkspaceSettings model schema and defaults.""" + assert WorkspaceSettings.__tablename__ == "workspace_settings" + assert ImportedWorkspaceSettings is WorkspaceSettings + + dummy_id = uuid.uuid4() + settings = WorkspaceSettings(workspace_id=dummy_id) + + assert settings.workspace_id == dummy_id + assert settings.default_invite_role == "member" + assert settings.invite_expiry_days == 7 + assert settings.activity_retention_days == 30 + assert settings.role_permissions == {} + + +def test_unknown_role_edge_cases(): + """Test resolution behavior when unknown or invalid role strings are passed.""" + # Unknown role names without overrides resolve to all-False boolean map + for unknown_role in [ + "guest", + "operator", + "auditor", + "", + "ADMIN", + " owner ", + "MEMBER", + ]: + resolved = resolve_role_permissions(unknown_role) + assert len(resolved) == 21 + assert all(val is False for val in resolved.values()), ( + f"Role '{unknown_role}' did not resolve to all-False" + ) + assert has_permission(unknown_role, "queries.execute") is False + assert has_permission(unknown_role, "catalog.view") is False + + # Flat overrides passed for unknown role grant specified capabilities + resolved_flat = resolve_role_permissions( + "guest", {"catalog.view": True, "queries.execute": True} + ) + assert resolved_flat["catalog.view"] is True + assert resolved_flat["queries.execute"] is True + assert resolved_flat["members.invite"] is False + + # Role-keyed nested overrides matching unknown role grant specified capabilities + resolved_nested_match = resolve_role_permissions( + "guest", {"guest": {"catalog.view": True}} + ) + assert resolved_nested_match["catalog.view"] is True + assert resolved_nested_match["members.invite"] is False + + # Role-keyed nested overrides not matching unknown role yield all-False + resolved_nested_mismatch = resolve_role_permissions( + "guest", {"admin": {"catalog.view": False}} + ) + assert all(val is False for val in resolved_nested_mismatch.values()) + + +def test_unknown_permission_key_edge_cases(): + """Test behavior when unknown or non-existent permission keys are queried.""" + unknown_keys = [ + "unknown.key", + "members.delete_all", + "catalog.admin", + "DATABASE.DROP", + "", + " ", + "queries.execute.sub", + ] + + for role in ["owner", "admin", "member", "guest"]: + for key in unknown_keys: + # has_permission must return False for unknown permission keys across all roles + assert has_permission(role, key) is False, ( + f"has_permission('{role}', '{key}') returned True, expected False" + ) + assert has_permission(role, {"catalog.view": True}, key) is False + assert ( + has_permission( + role, overrides={"catalog.view": True}, permission_key=key + ) + is False + ) + + # Check non-string permission keys + assert has_permission("member", None) is False + assert has_permission("owner", None) is False + assert has_permission("member", 12345) is False + + +def test_overrides_formats_empty_invalid_nested(): + """Test empty, invalid, flat vs nested role override dictionaries.""" + # 1. Empty dict and None overrides + assert resolve_role_permissions("admin", {}) == DEFAULT_ROLE_PERMISSIONS["admin"] + assert ( + resolve_role_permissions("member", None) == DEFAULT_ROLE_PERMISSIONS["member"] + ) + + # 2. Flat overrides vs Role-nested overrides comparison + flat_override = {"catalog.manage": False} + resolved_flat = resolve_role_permissions("admin", flat_override) + assert resolved_flat["catalog.manage"] is False + assert resolved_flat["catalog.view"] is True + + nested_override = {"admin": {"catalog.manage": False}} + resolved_nested = resolve_role_permissions("admin", nested_override) + assert resolved_nested["catalog.manage"] is False + assert resolved_nested["catalog.view"] is True + + # 3. Multi-role nested overrides dict + multi_role_overrides = { + "admin": {"catalog.manage": False}, + "member": {"queries.execute": False, "members.invite": True}, + } + resolved_admin = resolve_role_permissions("admin", multi_role_overrides) + assert resolved_admin["catalog.manage"] is False + assert resolved_admin["queries.execute"] is True # preserved admin default + + resolved_member = resolve_role_permissions("member", multi_role_overrides) + assert resolved_member["queries.execute"] is False # overridden to False + assert resolved_member["members.invite"] is True # overridden to True + assert resolved_member["catalog.manage"] is False # preserved member default + + # 4. Mismatched nested role override (overrides present for 'member', queried for 'admin') + mismatched_overrides = {"member": {"catalog.manage": False}} + resolved_mismatched = resolve_role_permissions("admin", mismatched_overrides) + assert resolved_mismatched["catalog.manage"] is True # admin default untouched + + # 5. Non-boolean override values (strings, integers, None) are ignored + invalid_val_overrides = { + "catalog.manage": "false", # string 'false' should not count as boolean False + "members.invite": 1, # int 1 should not count as boolean True for override + "queries.execute": 0, # int 0 should not count as boolean False + "dashboards.view": None, # None should not override + } + resolved_invalid_vals = resolve_role_permissions("member", invalid_val_overrides) + assert resolved_invalid_vals["catalog.manage"] is False # member default preserved + assert resolved_invalid_vals["members.invite"] is False # member default preserved + assert resolved_invalid_vals["queries.execute"] is True # member default preserved + assert resolved_invalid_vals["dashboards.view"] is True # member default preserved + + # 6. Unknown permission keys in overrides dictionary are ignored + overrides_with_unknown = { + "fake.permission": True, + "members.invite": True, + } + resolved_unknown_in_dict = resolve_role_permissions( + "member", overrides_with_unknown + ) + assert "fake.permission" not in resolved_unknown_in_dict + assert resolved_unknown_in_dict["members.invite"] is True + + +def test_owner_bypass_immutability_thorough(): + """Verify Owner role permissions cannot be overridden under any scenario.""" + # Flat overrides set all permissions to False + all_false_flat = {perm_key: False for perm_key in PERMISSIONS} + resolved_owner_flat = resolve_role_permissions("owner", all_false_flat) + assert len(resolved_owner_flat) == 21 + assert all(val is True for val in resolved_owner_flat.values()) + + # Nested role overrides set all permissions to False + all_false_nested = {"owner": {perm_key: False for perm_key in PERMISSIONS}} + resolved_owner_nested = resolve_role_permissions("owner", all_false_nested) + assert len(resolved_owner_nested) == 21 + assert all(val is True for val in resolved_owner_nested.values()) + + # Check has_permission for owner with overrides + for perm_key in PERMISSIONS: + assert has_permission("owner", all_false_flat, perm_key) is True + assert has_permission("owner", all_false_nested, perm_key) is True + assert has_permission("owner", perm_key) is True + + +def test_non_dict_overrides_handling(): + """Test behavior when non-dict objects are passed as overrides parameter.""" + # 1. Dual invocation format with has_permission (string passed as 2nd arg without 3rd arg) + assert has_permission("member", "queries.execute") is True + assert has_permission("member", "members.invite") is False + + # 2. Non-dict overrides in has_permission with permission_key specified + assert has_permission("member", 12345, "queries.execute") is True + assert has_permission("member", ["catalog.view"], "queries.execute") is True + + # 3. Direct call to resolve_role_permissions with non-dict truthy overrides + # Empirical observation: resolve_role_permissions raises AttributeError or TypeError when passed non-dict truthy overrides + with pytest.raises(AttributeError): + resolve_role_permissions("member", "non_dict_string_override") + + with pytest.raises(TypeError): + resolve_role_permissions("member", 12345) + + with pytest.raises(AttributeError): + resolve_role_permissions("member", ["catalog.view", True]) diff --git a/tests/test_pgdatabase_users.py b/tests/test_pgdatabase_users.py index ab44415a..74200ded 100644 --- a/tests/test_pgdatabase_users.py +++ b/tests/test_pgdatabase_users.py @@ -25,6 +25,10 @@ def _fake_user(**overrides): email_verified=False, tour_completed=False, created_at=None, + first_name=None, + last_name=None, + avatar_url=None, + metadata_={}, ) defaults.update(overrides) return SimpleNamespace(**defaults) diff --git a/tests/test_phase4_e2e_suite.py b/tests/test_phase4_e2e_suite.py new file mode 100644 index 00000000..53ce7e34 --- /dev/null +++ b/tests/test_phase4_e2e_suite.py @@ -0,0 +1,1470 @@ +""" +Phase 4 E2E Test Track Suite (Tiers 1-4) for BoloDB Workspace Defaults & Configurable RBAC Permission Matrix. + +Features Tested: +1. Central Permissions Registry & Default Role Matrix (Tier 1 & Tier 2) +2. workspace_settings Table & Model Persistence (Tier 1 & Tier 2) +3. require_permission Route Access Control & Owner Bypass (Tier 1 & Tier 2) +4. Settings GET/PATCH Endpoints & Sparse JSONB Overrides (Tier 1 & Tier 2) +5. Workspace Defaults Integration (Invites & Activity Log Retention) (Tier 1 & Tier 2) +6. Frontend Settings & Permissions Toggle Matrix UI Integration (Tier 1 & Tier 2) +7. Cross-Feature Pairwise Combinations (Tier 3) +8. Real-World Application Scenarios (Tier 4) + +Total Test Cases: 71 (Tier 1: 30, Tier 2: 30, Tier 3: 6, Tier 4: 5) +""" + +import uuid +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from fastapi import HTTPException +from fastapi.testclient import TestClient +from pydantic import ValidationError + +import backend.app.controllers.workspaces as ws_ctrl +import backend.app.controllers.activity as activity_ctrl +from backend.app import dependencies +from backend.app.dependencies import get_current_workspace, require_permission +from backend.app.models.workspace_settings import WorkspaceSettings +from backend.app.models.workspace_api import WorkspaceSettingsUpdate +from backend.app.permissions import ( + PERMISSIONS, + DEFAULT_ROLE_PERMISSIONS, + MEMBER_DEFAULT_PERMISSIONS, + resolve_role_permissions, + has_permission, +) +from backend.app.server import create_app + +WORKSPACE_ID = str(uuid.uuid4()) +OWNER_ID = str(uuid.uuid4()) +ADMIN_ID = str(uuid.uuid4()) +MEMBER_ID = str(uuid.uuid4()) + + +def _mock_result(value): + res = Mock(name="QueryResult") + res.scalar_one_or_none = Mock(return_value=value) + return res + + +def _patch_session(monkeypatch, session, target_module): + factory = Mock() + factory.return_value = session + session.__aenter__.return_value = session + monkeypatch.setattr(target_module, "async_session", factory) + return factory + + +@pytest.fixture +def app_instance(): + """Create FastAPI app with disabled lifespan/rate limiting for testing endpoints.""" + with patch("backend.app.server.lifespan"): + app = create_app(readonly=True) + if hasattr(app.state, "limiter"): + app.state.limiter.enabled = False + return app + + +# ===================================================================== +# TIER 1: FEATURE COVERAGE (30 TEST CASES) +# ===================================================================== + + +class TestTier1Feature1CentralPermissionsRegistry: + """Feature 1: Central Permissions Registry & Default Role Matrix (5 cases).""" + + def test_tier1_f1_registry_total_count_and_categories(self): + """1. Verify registry defines exactly 21 capabilities across 7 resource categories.""" + assert len(PERMISSIONS) == 21, ( + f"Expected 21 permissions, got {len(PERMISSIONS)}" + ) + + expected_categories = { + "members": 4, + "connections": 3, + "catalog": 2, + "dashboards": 3, + "queries": 4, + "activity": 2, + "workspace_management": 3, + } + category_counts = {} + for perm in PERMISSIONS.values(): + cat = perm["category"] + category_counts[cat] = category_counts.get(cat, 0) + 1 + + assert category_counts == expected_categories + + def test_tier1_f1_owner_default_role_matrix(self): + """2. Verify owner default role matrix grants all 21 permissions.""" + owner_matrix = DEFAULT_ROLE_PERMISSIONS["owner"] + assert len(owner_matrix) == 21 + assert all(val is True for val in owner_matrix.values()) + + def test_tier1_f1_admin_default_role_matrix(self): + """3. Verify admin default role matrix grants all 21 permissions.""" + admin_matrix = DEFAULT_ROLE_PERMISSIONS["admin"] + assert len(admin_matrix) == 21 + assert all(val is True for val in admin_matrix.values()) + + def test_tier1_f1_member_default_role_matrix(self): + """4. Verify member default role matrix grants 9 True and 12 False permissions.""" + member_matrix = DEFAULT_ROLE_PERMISSIONS["member"] + assert len(member_matrix) == 21 + true_keys = {k for k, v in member_matrix.items() if v is True} + assert true_keys == MEMBER_DEFAULT_PERMISSIONS + assert len(true_keys) == 9 + assert sum(1 for v in member_matrix.values() if v is False) == 12 + + def test_tier1_f1_resolve_role_permissions_defaults(self): + """5. Verify resolve_role_permissions resolves defaults correctly for standard roles.""" + resolved_owner = resolve_role_permissions("owner") + assert all(resolved_owner.values()) + + resolved_admin = resolve_role_permissions("admin") + assert all(resolved_admin.values()) + + resolved_member = resolve_role_permissions("member") + assert resolved_member["queries.execute"] is True + assert resolved_member["members.invite"] is False + + +class TestTier1Feature2WorkspaceSettingsPersistence: + """Feature 2: workspace_settings Table & Model Persistence (5 cases).""" + + def test_tier1_f2_settings_model_defaults(self): + """6. Verify WorkspaceSettings ORM model default field values.""" + wid = uuid.uuid4() + settings = WorkspaceSettings(workspace_id=wid) + assert settings.workspace_id == wid + assert settings.default_invite_role == "member" + assert settings.invite_expiry_days == 7 + assert settings.activity_retention_days == 30 + assert settings.role_permissions == {} + + @pytest.mark.asyncio + async def test_tier1_f2_get_workspace_settings_creates_defaults(self, monkeypatch): + """7. Verify get_workspace_settings initializes missing settings row in DB.""" + session = AsyncMock() + session.execute.return_value = _mock_result(None) + saved = [] + session.add = lambda obj: saved.append(obj) + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.get_workspace_settings(WORKSPACE_ID) + assert res["workspace_id"] == WORKSPACE_ID + assert res["default_invite_role"] == "member" + assert res["invite_expiry_days"] == 7 + assert res["activity_retention_days"] == 30 + assert len(saved) == 1 + + @pytest.mark.asyncio + async def test_tier1_f2_update_workspace_settings_scalar_fields(self, monkeypatch): + """8. Verify update_workspace_settings persists scalar configuration updates.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="member", + invite_expiry_days=7, + activity_retention_days=30, + role_permissions={}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + payload = { + "default_invite_role": "admin", + "invite_expiry_days": 14, + "activity_retention_days": 90, + } + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, payload, actor_id=OWNER_ID + ) + assert res["default_invite_role"] == "admin" + assert res["invite_expiry_days"] == 14 + assert res["activity_retention_days"] == 90 + + @pytest.mark.asyncio + async def test_tier1_f2_update_workspace_settings_role_permissions( + self, monkeypatch + ): + """9. Verify update_workspace_settings persists sparse JSONB permissions overrides.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + payload = { + "role_permissions": { + "member": {"dashboards.create": True}, + "admin": {"members.remove": False}, + } + } + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, payload, actor_id=OWNER_ID + ) + assert res["role_permissions"] == payload["role_permissions"] + assert res["resolved_matrix"]["member"]["dashboards.create"] is True + assert res["resolved_matrix"]["admin"]["members.remove"] is False + + @pytest.mark.asyncio + async def test_tier1_f2_update_workspace_settings_owner_exclusion( + self, monkeypatch + ): + """10. Verify attempted owner permission overrides are stripped on update.""" + existing = WorkspaceSettings(workspace_id=uuid.UUID(WORKSPACE_ID)) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + payload = { + "role_permissions": { + "owner": {"queries.execute": False}, + "member": {"dashboards.create": True}, + } + } + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, payload, actor_id=OWNER_ID + ) + assert "owner" not in res["role_permissions"] + assert res["resolved_matrix"]["owner"]["queries.execute"] is True + + +class TestTier1Feature3RequirePermissionDependency: + """Feature 3: require_permission Dependency & Route Access Control (5 cases).""" + + @pytest.mark.asyncio + async def test_tier1_f3_owner_bypass_without_db_lookup(self): + """11. Verify owner bypasses permissions check directly without DB query.""" + dep = require_permission("dashboards.create") + ws_context = { + "workspace_id": WORKSPACE_ID, + "role": "owner", + "user_id": OWNER_ID, + } + res = await dep(workspace=ws_context) + assert res == ws_context + + @pytest.mark.asyncio + async def test_tier1_f3_admin_default_permission_allowed(self, monkeypatch): + """12. Verify admin passes default permission check.""" + session = AsyncMock() + session.execute.return_value = _mock_result(None) + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("dashboards.create") + ws_context = { + "workspace_id": WORKSPACE_ID, + "role": "admin", + "user_id": ADMIN_ID, + } + res = await dep(workspace=ws_context) + assert res == ws_context + + @pytest.mark.asyncio + async def test_tier1_f3_member_default_permission_allowed(self, monkeypatch): + """13. Verify member passes allowed default permission check (queries.execute).""" + session = AsyncMock() + session.execute.return_value = _mock_result(None) + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("queries.execute") + ws_context = { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + res = await dep(workspace=ws_context) + assert res == ws_context + + @pytest.mark.asyncio + async def test_tier1_f3_member_restricted_permission_raises_403(self, monkeypatch): + """14. Verify member gets 403 for restricted permission (members.invite).""" + session = AsyncMock() + session.execute.return_value = _mock_result(None) + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("members.invite") + ws_context = { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + with pytest.raises(HTTPException) as exc_info: + await dep(workspace=ws_context) + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_tier1_f3_sparse_override_revocation_raises_403(self, monkeypatch): + """15. Verify custom override revoking member permission raises 403.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": {"queries.execute": False}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(settings) + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("queries.execute") + ws_context = { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + with pytest.raises(HTTPException) as exc_info: + await dep(workspace=ws_context) + assert exc_info.value.status_code == 403 + + +class TestTier1Feature4SettingsEndpointsAndOverrides: + """Feature 4: Settings GET/PATCH Endpoints & Sparse Overrides (5 cases).""" + + def test_tier1_f4_get_settings_endpoint_owner_success( + self, app_instance, monkeypatch + ): + """16. Verify GET /api/workspaces/{id}/settings succeeds for Owner.""" + client = TestClient(app_instance) + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "owner", + "user_id": OWNER_ID, + } + mock_settings = { + "workspace_id": WORKSPACE_ID, + "default_invite_role": "member", + "invite_expiry_days": 7, + "activity_retention_days": 30, + "role_permissions": {}, + "resolved_matrix": DEFAULT_ROLE_PERMISSIONS, + } + monkeypatch.setattr( + "backend.app.routes.workspaces.ctrl.get_workspace_settings", + AsyncMock(return_value=mock_settings), + ) + + response = client.get(f"/api/workspaces/{WORKSPACE_ID}/settings") + assert response.status_code == 200 + assert response.json()["workspace_id"] == WORKSPACE_ID + app_instance.dependency_overrides.clear() + + def test_tier1_f4_patch_settings_endpoint_owner_success( + self, app_instance, monkeypatch + ): + """17. Verify PATCH /api/workspaces/{id}/settings succeeds for Owner.""" + client = TestClient(app_instance) + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "owner", + "user_id": OWNER_ID, + } + updated_mock = { + "workspace_id": WORKSPACE_ID, + "default_invite_role": "admin", + "invite_expiry_days": 14, + "activity_retention_days": 60, + "role_permissions": {"member": {"dashboards.create": True}}, + "resolved_matrix": DEFAULT_ROLE_PERMISSIONS, + } + monkeypatch.setattr( + "backend.app.routes.workspaces.ctrl.update_workspace_settings", + AsyncMock(return_value=updated_mock), + ) + + payload = {"default_invite_role": "admin", "invite_expiry_days": 14} + response = client.patch( + f"/api/workspaces/{WORKSPACE_ID}/settings", json=payload + ) + assert response.status_code == 200 + assert response.json()["default_invite_role"] == "admin" + app_instance.dependency_overrides.clear() + + def test_tier1_f4_patch_settings_endpoint_admin_forbidden(self, app_instance): + """18. Verify PATCH /api/workspaces/{id}/settings returns 403 for Admin.""" + client = TestClient(app_instance) + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "admin", + "user_id": ADMIN_ID, + } + + response = client.patch( + f"/api/workspaces/{WORKSPACE_ID}/settings", json={"invite_expiry_days": 14} + ) + assert response.status_code == 403 + assert response.json()["detail"] == "Owner role required" + app_instance.dependency_overrides.clear() + + def test_tier1_f4_patch_settings_endpoint_member_forbidden(self, app_instance): + """19. Verify PATCH /api/workspaces/{id}/settings returns 403 for Member.""" + client = TestClient(app_instance) + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + + response = client.patch( + f"/api/workspaces/{WORKSPACE_ID}/settings", json={"invite_expiry_days": 14} + ) + assert response.status_code == 403 + assert response.json()["detail"] == "Owner role required" + app_instance.dependency_overrides.clear() + + @pytest.mark.asyncio + async def test_tier1_f4_patch_settings_sparse_permission_merge(self, monkeypatch): + """20. Verify sparse PATCH merges role permissions without wiping unreferenced roles.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"admin": {"dashboards.manage": False}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + update_payload = { + "role_permissions": { + "admin": {"dashboards.manage": False}, + "member": {"dashboards.create": True}, + } + } + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, update_payload, actor_id=OWNER_ID + ) + assert res["role_permissions"]["admin"]["dashboards.manage"] is False + assert res["role_permissions"]["member"]["dashboards.create"] is True + + +class TestTier1Feature5WorkspaceDefaultsIntegration: + """Feature 5: Workspace Defaults Integration (Invites & Activity Log Retention) (5 cases).""" + + @pytest.mark.asyncio + async def test_tier1_f5_invite_uses_workspace_default_role(self, monkeypatch): + """21. Verify invite creation uses default_invite_role setting when role is None.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="admin", + invite_expiry_days=7, + ) + session = AsyncMock() + session.execute.side_effect = [ + _mock_result(settings), + _mock_result(None), + _mock_result(None), + _mock_result(None), + ] + session.get.return_value = SimpleNamespace(name="Acme Corp") + monkeypatch.setattr( + ws_ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "newuser@example.com", None, OWNER_ID + ) + assert invite["role"] == "admin" + + @pytest.mark.asyncio + async def test_tier1_f5_invite_uses_workspace_expiry_days(self, monkeypatch): + """22. Verify invite creation uses invite_expiry_days setting when expiry_days is None.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="member", + invite_expiry_days=14, + ) + session = AsyncMock() + session.execute.side_effect = [ + _mock_result(settings), + _mock_result(None), + _mock_result(None), + _mock_result(None), + ] + session.get.return_value = SimpleNamespace(name="Acme Corp") + monkeypatch.setattr( + ws_ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "newuser@example.com", None, OWNER_ID + ) + expected_expiry = datetime.now(timezone.utc) + timedelta(days=14) + diff = abs((invite["expires_at"] - expected_expiry).total_seconds()) + assert diff < 60 + + @pytest.mark.asyncio + async def test_tier1_f5_invite_explicit_role_overrides_default(self, monkeypatch): + """23. Verify passing explicit role overrides default_invite_role setting.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="member", + ) + session = AsyncMock() + session.execute.side_effect = [ + _mock_result(settings), + _mock_result(None), + _mock_result(None), + _mock_result(None), + ] + session.get.return_value = SimpleNamespace(name="Acme Corp") + monkeypatch.setattr( + ws_ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "newuser@example.com", "admin", OWNER_ID + ) + assert invite["role"] == "admin" + + @pytest.mark.asyncio + async def test_tier1_f5_activity_retention_days_query(self, monkeypatch): + """24. Verify activity log query uses activity_retention_days setting.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + activity_retention_days=10, + ) + session = AsyncMock() + res_activity = Mock() + res_activity.all.return_value = [] + session.execute.side_effect = [_mock_result(settings), res_activity] + _patch_session(monkeypatch, session, activity_ctrl) + + logs = await activity_ctrl.get_workspace_activity(WORKSPACE_ID, limit=50) + assert logs == [] + + @pytest.mark.asyncio + async def test_tier1_f5_activity_cleanup_controller_retention(self, monkeypatch): + """25. Verify delete_old_activity prunes logs using retention window.""" + session = AsyncMock() + del_result = Mock() + del_result.rowcount = 3 + session.execute.return_value = del_result + _patch_session(monkeypatch, session, activity_ctrl) + + count = await activity_ctrl.delete_old_activity(retention_days=15) + assert count == 3 + + +class TestTier1Feature6FrontendSettingsMatrixUI: + """Feature 6: Frontend Settings & Permissions Toggle Matrix UI Integration (5 cases).""" + + @pytest.mark.asyncio + async def test_tier1_f6_resolved_matrix_format_structure(self, monkeypatch): + """26. Verify GET settings response contains complete resolved_matrix structure.""" + session = AsyncMock() + session.execute.return_value = _mock_result(None) + session.add = Mock() + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.get_workspace_settings(WORKSPACE_ID) + assert "resolved_matrix" in res + matrix = res["resolved_matrix"] + assert "owner" in matrix and "admin" in matrix and "member" in matrix + + def test_tier1_f6_matrix_payload_all_keys_presence(self): + """27. Verify all 21 permission keys exist in resolved_matrix for every role.""" + matrix = { + role: resolve_role_permissions(role) + for role in ["owner", "admin", "member"] + } + for role in ["owner", "admin", "member"]: + assert len(matrix[role]) == 21 + for key in PERMISSIONS: + assert key in matrix[role] + + @pytest.mark.asyncio + async def test_tier1_f6_matrix_toggle_patch_roundtrip(self, monkeypatch): + """28. Verify setting matrix toggle via update_workspace_settings reflects in resolved_matrix.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), role_permissions={} + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + payload = {"role_permissions": {"member": {"dashboards.create": True}}} + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, payload, actor_id=OWNER_ID + ) + assert res["resolved_matrix"]["member"]["dashboards.create"] is True + + @pytest.mark.asyncio + async def test_tier1_f6_matrix_reset_overrides_empty_dict(self, monkeypatch): + """29. Verify updating role_permissions to {} resets resolved_matrix back to default RBAC.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": {"dashboards.create": True}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + payload = {"role_permissions": {}} + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, payload, actor_id=OWNER_ID + ) + assert res["role_permissions"] == {} + assert res["resolved_matrix"]["member"]["dashboards.create"] is False + + @pytest.mark.asyncio + async def test_tier1_f6_matrix_invalid_permission_keys_ignored(self, monkeypatch): + """30. Verify invalid permission keys in patch payload are filtered out during resolution.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), role_permissions={} + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + payload = { + "role_permissions": { + "member": {"invalid.perm.key": True, "dashboards.create": True} + } + } + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, payload, actor_id=OWNER_ID + ) + assert "invalid.perm.key" not in res["resolved_matrix"]["member"] + assert res["resolved_matrix"]["member"]["dashboards.create"] is True + + +# ===================================================================== +# TIER 2: BOUNDARY & CORNER CASES (30 TEST CASES) +# ===================================================================== + + +class TestTier2Feature1CentralPermissionsRegistryBoundaries: + """Feature 1 Boundaries (5 cases).""" + + def test_tier2_f1_unknown_role_resolution_all_false(self): + """31. Verify resolve_role_permissions for unknown role returns all False.""" + res = resolve_role_permissions("guest") + assert len(res) == 21 + assert all(v is False for v in res.values()) + + def test_tier2_f1_has_permission_non_existent_key(self): + """32. Verify has_permission for non-existent permission key returns False for all roles.""" + assert has_permission("admin", "invalid.key") is False + assert has_permission("owner", "non_existent_capability") is False + + def test_tier2_f1_has_permission_dual_signature_formats(self): + """33. Verify has_permission dual signature invocation forms.""" + # 2-arg signature + assert has_permission("member", "queries.execute") is True + # 3-arg signature with overrides dict + overrides = {"member": {"queries.execute": False}} + assert has_permission("member", overrides, "queries.execute") is False + + def test_tier2_f1_has_permission_none_and_empty_args(self): + """34. Verify has_permission with None or empty string parameters.""" + assert has_permission(None, "queries.execute") is False + assert has_permission("member", "") is False + assert has_permission("member", None) is False + + def test_tier2_f1_all_categories_exhaustive_check(self): + """35. Verify all 21 permission keys resolve boolean for member role.""" + resolved = resolve_role_permissions("member") + for key in PERMISSIONS: + assert isinstance(resolved[key], bool) + + +class TestTier2Feature2WorkspaceSettingsPersistenceBoundaries: + """Feature 2 Boundaries (5 cases).""" + + @pytest.mark.asyncio + async def test_tier2_f2_expiry_days_boundary_values(self, monkeypatch): + """36. Verify update_workspace_settings handles boundary values for invite_expiry_days.""" + existing = WorkspaceSettings(workspace_id=uuid.UUID(WORKSPACE_ID)) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, {"invite_expiry_days": 1}, actor_id=OWNER_ID + ) + assert res["invite_expiry_days"] == 1 + + res_max = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, {"invite_expiry_days": 365}, actor_id=OWNER_ID + ) + assert res_max["invite_expiry_days"] == 365 + + @pytest.mark.asyncio + async def test_tier2_f2_retention_days_boundary_values(self, monkeypatch): + """37. Verify update_workspace_settings handles boundary values for activity_retention_days.""" + existing = WorkspaceSettings(workspace_id=uuid.UUID(WORKSPACE_ID)) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, {"activity_retention_days": 3650}, actor_id=OWNER_ID + ) + assert res["activity_retention_days"] == 3650 + + def test_tier2_f2_invalid_default_invite_role_fallback(self): + """38. Verify WorkspaceSettingsUpdate rejects invalid default_invite_role via Pydantic.""" + with pytest.raises(ValidationError): + WorkspaceSettingsUpdate(default_invite_role="superman") + + @pytest.mark.asyncio + async def test_tier2_f2_workspace_id_string_and_uuid_types(self, monkeypatch): + """39. Verify get_workspace_settings handles UUID object or UUID string workspace_id.""" + session = AsyncMock() + session.execute.return_value = _mock_result(None) + session.add = Mock() + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.get_workspace_settings(WORKSPACE_ID) + assert res["workspace_id"] == WORKSPACE_ID + + def test_tier2_f2_corrupted_jsonb_overrides_resilience(self): + """40. Verify resolve_role_permissions handles corrupted non-dict or invalid type overrides.""" + assert resolve_role_permissions("member", None)["queries.execute"] is True + assert resolve_role_permissions("member", {})["queries.execute"] is True + assert ( + resolve_role_permissions("member", {"member": "invalid_string"})[ + "queries.execute" + ] + is True + ) + assert ( + resolve_role_permissions("member", {"member": 12345})["queries.execute"] + is True + ) + + +class TestTier2Feature3RequirePermissionDependencyBoundaries: + """Feature 3 Boundaries (5 cases).""" + + @pytest.mark.asyncio + async def test_tier2_f3_missing_workspace_id_header(self): + """41. Verify get_current_workspace raises 400 if X-Workspace-Id header is missing.""" + from backend.app.dependencies import get_current_workspace + + with pytest.raises(HTTPException) as exc_info: + await get_current_workspace(x_workspace_id=None, user={"user_id": OWNER_ID}) + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_tier2_f3_invalid_workspace_id_format(self): + """42. Verify get_current_workspace raises 400 if X-Workspace-Id format is invalid.""" + from backend.app.dependencies import get_current_workspace + + with pytest.raises(HTTPException) as exc_info: + await get_current_workspace( + x_workspace_id="invalid-uuid", user={"user_id": OWNER_ID} + ) + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_tier2_f3_non_member_user_access(self, monkeypatch): + """43. Verify get_current_workspace raises 403 if user is not a member of workspace.""" + session = AsyncMock() + session.execute.return_value = _mock_result(None) + _patch_session(monkeypatch, session, dependencies) + + with pytest.raises(HTTPException) as exc_info: + await dependencies.get_current_workspace( + x_workspace_id=WORKSPACE_ID, user={"user_id": OWNER_ID} + ) + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_tier2_f3_unauthenticated_request(self): + """44. Verify get_current_user raises 401 if access_token cookie is missing.""" + from backend.app.dependencies import get_current_user + + with pytest.raises(HTTPException) as exc_info: + await get_current_user(access_token=None) + assert exc_info.value.status_code == 401 + + @pytest.mark.asyncio + async def test_tier2_f3_require_permission_missing_settings_row_fallback( + self, monkeypatch + ): + """45. Verify require_permission falls back to defaults when settings row does not exist.""" + session = AsyncMock() + session.execute.return_value = _mock_result(None) + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("queries.execute") + ws_context = { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + res = await dep(workspace=ws_context) + assert res == ws_context + + +class TestTier2Feature4SettingsEndpointsBoundaries: + """Feature 4 Boundaries (5 cases).""" + + @pytest.mark.asyncio + async def test_tier2_f4_patch_settings_empty_body(self, monkeypatch): + """46. Verify update_workspace_settings with empty dict leaves settings unchanged.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="admin", + invite_expiry_days=14, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, {}, actor_id=OWNER_ID + ) + assert res["default_invite_role"] == "admin" + assert res["invite_expiry_days"] == 14 + + @pytest.mark.asyncio + async def test_tier2_f4_patch_settings_partial_update(self, monkeypatch): + """47. Verify partial patch updates only specified field without resetting others.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="admin", + invite_expiry_days=7, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, {"invite_expiry_days": 30}, actor_id=OWNER_ID + ) + assert res["default_invite_role"] == "admin" + assert res["invite_expiry_days"] == 30 + + @pytest.mark.asyncio + async def test_tier2_f4_patch_settings_sequential_sparse_updates(self, monkeypatch): + """48. Verify sequential sparse updates merge role permissions incrementally.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": {"dashboards.create": True}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + payload = { + "role_permissions": { + "member": {"dashboards.create": True}, + "admin": {"dashboards.manage": False}, + } + } + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, payload, actor_id=OWNER_ID + ) + assert res["role_permissions"]["member"]["dashboards.create"] is True + assert res["role_permissions"]["admin"]["dashboards.manage"] is False + + def test_tier2_f4_patch_settings_invalid_permission_type(self): + """49. Verify WorkspaceSettingsUpdate rejects invalid role_permissions type via Pydantic.""" + with pytest.raises(ValidationError): + WorkspaceSettingsUpdate(role_permissions="invalid_type") + + def test_tier2_f4_get_settings_member_access_forbidden( + self, app_instance, monkeypatch + ): + """50. Verify GET settings returns 403 for member without workspace.settings permission.""" + client = TestClient(app_instance) + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + session = AsyncMock() + session.execute.return_value = _mock_result(None) + _patch_session(monkeypatch, session, dependencies) + + response = client.get(f"/api/workspaces/{WORKSPACE_ID}/settings") + assert response.status_code == 403 + assert "workspace.settings" in response.json()["detail"] + app_instance.dependency_overrides.clear() + + +class TestTier2Feature5WorkspaceDefaultsIntegrationBoundaries: + """Feature 5 Boundaries (5 cases).""" + + @pytest.mark.asyncio + async def test_tier2_f5_invite_expiry_timestamp_precision(self, monkeypatch): + """51. Verify invite expires_at timestamp accuracy for custom expiry_days.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), invite_expiry_days=21 + ) + session = AsyncMock() + session.execute.side_effect = [ + _mock_result(settings), + _mock_result(None), + _mock_result(None), + _mock_result(None), + ] + session.get.return_value = SimpleNamespace(name="Acme Corp") + monkeypatch.setattr( + ws_ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "test@example.com", None, OWNER_ID + ) + expected = datetime.now(timezone.utc) + timedelta(days=21) + diff = abs((invite["expires_at"] - expected).total_seconds()) + assert diff < 60 + + @pytest.mark.asyncio + async def test_tier2_f5_activity_cleanup_zero_logs_deleted(self, monkeypatch): + """52. Verify delete_old_activity returns 0 when no logs exceed retention threshold.""" + session = AsyncMock() + del_result = Mock() + del_result.rowcount = 0 + session.execute.return_value = del_result + _patch_session(monkeypatch, session, activity_ctrl) + + count = await activity_ctrl.delete_old_activity(retention_days=30) + assert count == 0 + + @pytest.mark.asyncio + async def test_tier2_f5_activity_cleanup_boundary_timestamp(self, monkeypatch): + """53. Verify activity cleanup logic executes valid cutoff SQL query.""" + session = AsyncMock() + del_result = Mock() + del_result.rowcount = 5 + session.execute.return_value = del_result + _patch_session(monkeypatch, session, activity_ctrl) + + count = await activity_ctrl.delete_old_activity(retention_days=1) + assert count == 5 + + @pytest.mark.asyncio + async def test_tier2_f5_resend_invite_uses_current_settings(self, monkeypatch): + """54. Verify resending an invite updates expiration with current settings.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), invite_expiry_days=10 + ) + existing_invite = SimpleNamespace( + id=uuid.uuid4(), + email="test@example.com", + role="member", + token="old_token", + invited_by=uuid.UUID(OWNER_ID), + expires_at=datetime.now(timezone.utc), + accepted_at=None, + ) + session = AsyncMock() + session.execute.side_effect = [ + _mock_result(settings), + _mock_result(None), + _mock_result(None), + _mock_result(existing_invite), + ] + session.get.return_value = SimpleNamespace(name="Acme Corp") + monkeypatch.setattr( + ws_ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "test@example.com", None, OWNER_ID + ) + expected = datetime.now(timezone.utc) + timedelta(days=10) + diff = abs((invite["expires_at"] - expected).total_seconds()) + assert diff < 60 + + @pytest.mark.asyncio + async def test_tier2_f5_activity_retention_uninitialized_settings( + self, monkeypatch + ): + """55. Verify activity query falls back to 30 days when settings row is missing.""" + session = AsyncMock() + res_activity = Mock() + res_activity.all.return_value = [] + session.execute.side_effect = [_mock_result(None), res_activity] + _patch_session(monkeypatch, session, activity_ctrl) + + logs = await activity_ctrl.get_workspace_activity(WORKSPACE_ID, limit=50) + assert logs == [] + + +class TestTier2Feature6FrontendSettingsMatrixUIBoundaries: + """Feature 6 Boundaries (5 cases).""" + + @pytest.mark.asyncio + async def test_tier2_f6_matrix_ui_toggle_all_member_perms_true(self, monkeypatch): + """56. Verify resolved matrix when member role permissions are all set to True.""" + all_true_member = {k: True for k in PERMISSIONS} + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": all_true_member}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, {}, actor_id=OWNER_ID + ) + assert all(res["resolved_matrix"]["member"].values()) + + @pytest.mark.asyncio + async def test_tier2_f6_matrix_ui_toggle_all_admin_perms_false(self, monkeypatch): + """57. Verify resolved matrix when admin role permissions are all set to False.""" + all_false_admin = {k: False for k in PERMISSIONS} + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"admin": all_false_admin}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, {}, actor_id=OWNER_ID + ) + assert all(val is False for val in res["resolved_matrix"]["admin"].values()) + + @pytest.mark.asyncio + async def test_tier2_f6_matrix_ui_owner_always_all_true_despite_payload( + self, monkeypatch + ): + """58. Verify owner role in resolved_matrix remains 100% True despite payload overrides.""" + all_false_owner = {k: False for k in PERMISSIONS} + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), role_permissions={} + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, + {"role_permissions": {"owner": all_false_owner}}, + actor_id=OWNER_ID, + ) + assert all(res["resolved_matrix"]["owner"].values()) + + def test_tier2_f6_sparse_override_diff_computation(self): + """59. Verify sparse override diff computation against default RBAC matrix.""" + overrides = {"member": {"dashboards.create": True}} + resolved = resolve_role_permissions("member", overrides) + assert resolved["dashboards.create"] is True + # Original default was False + assert DEFAULT_ROLE_PERMISSIONS["member"]["dashboards.create"] is False + + def test_tier2_f6_matrix_category_grouping_response(self): + """60. Verify resolved matrix keys map cleanly to registered categories.""" + matrix = resolve_role_permissions("member") + for key in matrix: + assert key in PERMISSIONS + assert "category" in PERMISSIONS[key] + + +# ===================================================================== +# TIER 3: CROSS-FEATURE PAIRWISE COMBINATIONS (6 TEST CASES) +# ===================================================================== + + +class TestTier3CrossFeaturePairwise: + """Tier 3: Pairwise Cross-Feature Integration Combinations (6 cases).""" + + @pytest.mark.asyncio + async def test_tier3_p1_perm_override_and_route_enforcement(self, monkeypatch): + """61. Pairwise 1: Permission override + route enforcement (queries.execute revoked for Member -> 403).""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": {"queries.execute": False}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(settings) + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("queries.execute") + ws_context = { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + with pytest.raises(HTTPException) as exc_info: + await dep(workspace=ws_context) + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_tier3_p2_setting_override_and_invite_defaults(self, monkeypatch): + """62. Pairwise 2: Setting override + invite defaults (default_role=admin, expiry=14 -> invite created).""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="admin", + invite_expiry_days=14, + ) + session = AsyncMock() + session.execute.side_effect = [ + _mock_result(settings), + _mock_result(None), + _mock_result(None), + _mock_result(None), + ] + session.get.return_value = SimpleNamespace(name="Acme Corp") + monkeypatch.setattr( + ws_ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "test@example.com", None, OWNER_ID + ) + assert invite["role"] == "admin" + expected = datetime.now(timezone.utc) + timedelta(days=14) + assert abs((invite["expires_at"] - expected).total_seconds()) < 60 + + @pytest.mark.asyncio + async def test_tier3_p3_activity_retention_and_cleanup_job(self, monkeypatch): + """63. Pairwise 3: Activity retention setting + cleanup job (retention=7 days -> cleanup executes cutoff query).""" + session = AsyncMock() + del_result = Mock() + del_result.rowcount = 12 + session.execute.return_value = del_result + _patch_session(monkeypatch, session, activity_ctrl) + + count = await activity_ctrl.delete_old_activity(retention_days=7) + assert count == 12 + + @pytest.mark.asyncio + async def test_tier3_p4_owner_bypass_and_permission_revocation_attempt( + self, monkeypatch + ): + """64. Pairwise 4: Owner bypass + permissions update (Attempt owner override -> ignored, owner executes route).""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), role_permissions={} + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + # Attempt to revoke owner permission in settings + payload = {"role_permissions": {"owner": {"queries.execute": False}}} + res = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, payload, actor_id=OWNER_ID + ) + assert "owner" not in res["role_permissions"] + + # Test require_permission for owner + dep = require_permission("queries.execute") + ws_context = { + "workspace_id": WORKSPACE_ID, + "role": "owner", + "user_id": OWNER_ID, + } + out = await dep(workspace=ws_context) + assert out == ws_context + + @pytest.mark.asyncio + async def test_tier3_p5_sparse_jsonb_update_and_matrix_resolution( + self, monkeypatch + ): + """65. Pairwise 5: Sparse JSONB update + matrix resolution (Sequential patches accumulate in resolved_matrix).""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), role_permissions={} + ) + session = AsyncMock() + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + # Patch 1: Member dashboard create + res1 = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, + {"role_permissions": {"member": {"dashboards.create": True}}}, + actor_id=OWNER_ID, + ) + assert res1["resolved_matrix"]["member"]["dashboards.create"] is True + + # Patch 2: Admin dashboard manage & preserve Member dashboard create + res2 = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, + { + "role_permissions": { + "member": {"dashboards.create": True}, + "admin": {"dashboards.manage": False}, + } + }, + actor_id=OWNER_ID, + ) + assert res2["resolved_matrix"]["member"]["dashboards.create"] is True + assert res2["resolved_matrix"]["admin"]["dashboards.manage"] is False + + @pytest.mark.asyncio + async def test_tier3_p6_non_owner_role_permission_modification_and_route_access( + self, monkeypatch + ): + """66. Pairwise 6: Non-owner role permission modification + route access (Admin dashboards.create revoked -> 403).""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"admin": {"dashboards.create": False}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(settings) + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("dashboards.create") + ws_context = { + "workspace_id": WORKSPACE_ID, + "role": "admin", + "user_id": ADMIN_ID, + } + with pytest.raises(HTTPException) as exc_info: + await dep(workspace=ws_context) + assert exc_info.value.status_code == 403 + + +# ===================================================================== +# TIER 4: REAL-WORLD APPLICATION SCENARIOS (5 TEST CASES) +# ===================================================================== + + +class TestTier4RealWorldScenarios: + """Tier 4: End-to-End Real-World Application Scenarios (5 cases).""" + + @pytest.mark.asyncio + async def test_tier4_s1_custom_member_restriction_scenario(self, monkeypatch): + """67. Scenario 1: Custom Member Restriction Flow (disable query execute -> 403 -> re-enable -> 200).""" + # Step 1: Default member has query execution allowed + session = AsyncMock() + session.execute.return_value = _mock_result(None) + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("queries.execute") + member_ctx = { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + assert await dep(workspace=member_ctx) == member_ctx + + # Step 2: Security admin revokes member query execution in settings + restricted_settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": {"queries.execute": False}}, + ) + session.execute.return_value = _mock_result(restricted_settings) + with pytest.raises(HTTPException) as exc_info: + await dep(workspace=member_ctx) + assert exc_info.value.status_code == 403 + + # Step 3: Security admin re-enables member query execution + restored_settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": {"queries.execute": True}}, + ) + session.execute.return_value = _mock_result(restored_settings) + assert await dep(workspace=member_ctx) == member_ctx + + @pytest.mark.asyncio + async def test_tier4_s2_admin_permission_customization_scenario(self, monkeypatch): + """68. Scenario 2: Admin Permission Customization (toggle dashboard creation off for Admin role).""" + # Step 1: Owner disables dashboard creation for admin role + custom_settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"admin": {"dashboards.create": False}}, + ) + session = AsyncMock() + session.execute.return_value = _mock_result(custom_settings) + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("dashboards.create") + admin_ctx = {"workspace_id": WORKSPACE_ID, "role": "admin", "user_id": ADMIN_ID} + with pytest.raises(HTTPException) as exc_info: + await dep(workspace=admin_ctx) + assert exc_info.value.status_code == 403 + + # Step 2: Owner re-enables dashboard creation for admin role + reenabled_settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={}, + ) + session.execute.return_value = _mock_result(reenabled_settings) + assert await dep(workspace=admin_ctx) == admin_ctx + + @pytest.mark.asyncio + async def test_tier4_s3_workspace_invite_expiry_override_scenario( + self, monkeypatch + ): + """69. Scenario 3: Workspace Invite Expiry & Default Role Integration Scenario.""" + # Step 1: Owner configures custom invite settings (role=admin, expiry=14 days) + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="admin", + invite_expiry_days=14, + ) + session = AsyncMock() + session.execute.side_effect = [ + _mock_result(settings), + _mock_result(None), + _mock_result(None), + _mock_result(None), + ] + session.get.return_value = SimpleNamespace(name="BoloDB Org") + monkeypatch.setattr( + ws_ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + _patch_session(monkeypatch, session, ws_ctrl) + + # Step 2: Send invite without specifying role or expiry + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "contractor@example.com", None, OWNER_ID + ) + + # Step 3: Assert invite object properties match workspace defaults + assert invite["role"] == "admin" + assert invite["email"] == "contractor@example.com" + expected_expiry = datetime.now(timezone.utc) + timedelta(days=14) + assert abs((invite["expires_at"] - expected_expiry).total_seconds()) < 60 + + def test_tier4_s4_non_owner_settings_update_attempt_scenario(self, app_instance): + """70. Scenario 4: Non-Owner Settings Update Attempt (Member/Admin -> 403, Owner -> 200).""" + client = TestClient(app_instance) + + # 1. Member attempts to patch workspace settings + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + res_member = client.patch( + f"/api/workspaces/{WORKSPACE_ID}/settings", json={"invite_expiry_days": 30} + ) + assert res_member.status_code == 403 + + # 2. Admin attempts to patch workspace settings + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "admin", + "user_id": ADMIN_ID, + } + res_admin = client.patch( + f"/api/workspaces/{WORKSPACE_ID}/settings", json={"invite_expiry_days": 30} + ) + assert res_admin.status_code == 403 + + # 3. Owner updates workspace settings successfully + app_instance.dependency_overrides[get_current_workspace] = lambda: { + "workspace_id": WORKSPACE_ID, + "role": "owner", + "user_id": OWNER_ID, + } + with patch( + "backend.app.routes.workspaces.ctrl.update_workspace_settings", + AsyncMock(return_value={"invite_expiry_days": 30}), + ): + res_owner = client.patch( + f"/api/workspaces/{WORKSPACE_ID}/settings", + json={"invite_expiry_days": 30}, + ) + assert res_owner.status_code == 200 + assert res_owner.json()["invite_expiry_days"] == 30 + + app_instance.dependency_overrides.clear() + + @pytest.mark.asyncio + async def test_tier4_s5_full_end_to_end_permission_matrix_modification( + self, monkeypatch + ): + """71. Scenario 5: Full End-to-End Permission Matrix Modification & Endpoint Validation.""" + # 1. Initial State: Owner views default settings + session = AsyncMock() + session.execute.return_value = _mock_result(None) + session.add = Mock() + _patch_session(monkeypatch, session, ws_ctrl) + + initial = await ws_ctrl.get_workspace_settings(WORKSPACE_ID) + assert initial["resolved_matrix"]["admin"]["members.remove"] is True + assert initial["resolved_matrix"]["member"]["dashboards.create"] is False + + # 2. Owner applies multi-role matrix modification + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), role_permissions={} + ) + session.execute.return_value = _mock_result(existing) + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + + payload = { + "role_permissions": { + "admin": {"members.remove": False}, + "member": {"dashboards.create": True}, + } + } + updated = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, payload, actor_id=OWNER_ID + ) + assert updated["resolved_matrix"]["admin"]["members.remove"] is False + assert updated["resolved_matrix"]["member"]["dashboards.create"] is True + + # 3. Verify route access dependency enforcement + modified_settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions=payload["role_permissions"], + ) + session.execute.return_value = _mock_result(modified_settings) + _patch_session(monkeypatch, session, dependencies) + + # Admin attempting members.remove gets 403 + admin_ctx = {"workspace_id": WORKSPACE_ID, "role": "admin", "user_id": ADMIN_ID} + dep_remove = require_permission("members.remove") + with pytest.raises(HTTPException) as exc_info: + await dep_remove(workspace=admin_ctx) + assert exc_info.value.status_code == 403 + + # Member attempting dashboards.create succeeds + member_ctx = { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + dep_create = require_permission("dashboards.create") + assert await dep_create(workspace=member_ctx) == member_ctx + + # 4. Owner resets matrix back to defaults + reset_settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={}, + ) + session.execute.return_value = _mock_result(reset_settings) + + # Admin members.remove now succeeds + assert await dep_remove(workspace=admin_ctx) == admin_ctx + + # Member dashboards.create now raises 403 + with pytest.raises(HTTPException) as exc_info_member: + await dep_create(workspace=member_ctx) + assert exc_info_member.value.status_code == 403 diff --git a/tests/test_query_db_scoping.py b/tests/test_query_db_scoping.py new file mode 100644 index 00000000..44af1ea8 --- /dev/null +++ b/tests/test_query_db_scoping.py @@ -0,0 +1,395 @@ +"""Every entry point must stay on the database the request selected. + +A workspace can hold several databases, and the caller picks one per request +with the `X-Db-Id` header. Each controller below used to check connectivity +against that selection but then read `db.get_db_id(workspace_id)` again for +knowledge and logging — which returns the workspace *default*. The result was a +query that ran against database B while being prompted with database A's +glossary, verified examples and catalog, and then logged against A. + +These tests record every call the controllers make and assert the selected id +reaches each collaborator. They are deliberately strict about *which* id each +call carries: asserting only that "a db_id was passed" is what let the mismatch +survive in the first place. +""" + +import asyncio +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException + +from backend.app.controllers import query as query_ctrl +from backend.app.models.api import QueryReq + +WORKSPACE = "workspace-1" +SELECTED_DB = "secondary-db" +DEFAULT_DB = "default-db" + + +class RecordingDB: + """A manager whose `get_db_id` always names the workspace default. + + That is the whole point: if a controller reaches for `get_db_id` instead of + the id it was handed, the recorded calls show `default-db` and the test + fails. + """ + + def __init__(self, connected=True): + self.calls = [] + self._connected = connected + + def connected(self, workspace_id, db_id=None): + self.calls.append(("connected", workspace_id, db_id)) + return self._connected + + def get_db_id(self, workspace_id, db_id=None): + self.calls.append(("get_db_id", workspace_id, db_id)) + return DEFAULT_DB + + def get_schema(self, workspace_id, db_id=None, refresh=False): + self.calls.append(("get_schema", workspace_id, db_id)) + return { + "secondary_orders": { + "columns": [{"name": "id", "type": "integer"}], + "samples": [], + } + } + + def get_dialect(self, workspace_id, db_id=None): + self.calls.append(("get_dialect", workspace_id, db_id)) + return "postgres" + + def execute(self, workspace_id, sql, db_id=None): + self.calls.append(("execute", workspace_id, sql, db_id)) + return {"columns": ["id"], "rows": [[1]], "truncated": False} + + def db_ids_seen(self, name): + """Every db_id this manager was asked about under `name`.""" + return [c[-1] for c in self.calls if c[0] == name] + + +class RecordingKB: + def __init__(self): + self.calls = [] + + async def get_glossary(self, workspace_id, db_id): + self.calls.append(("get_glossary", workspace_id, db_id)) + return [] + + async def get_catalog(self, workspace_id, db_id): + self.calls.append(("get_catalog", workspace_id, db_id)) + return {} + + async def retrieve_similar(self, workspace_id, db_id, q, k=3): + self.calls.append(("retrieve_similar", workspace_id, db_id)) + return [] + + async def add_verified(self, workspace_id, db_id, question, sql, restatement): + self.calls.append(("add_verified", workspace_id, db_id)) + + async def trust_level(self, workspace_id, db_id): + self.calls.append(("trust_level", workspace_id, db_id)) + return {"level": "Supervised", "verified": 1, "db_id_seen": db_id} + + async def get_verified(self, workspace_id, db_id): + self.calls.append(("get_verified", workspace_id, db_id)) + return [{"question": f"starter-from-{db_id}"}] + + def db_ids_seen(self, name=None): + return [c[2] for c in self.calls if name is None or c[0] == name] + + +class RecordingSessionLog: + def __init__(self): + self.calls = [] + + def log_query(self, db_id, question, out): + self.calls.append(("log_query", db_id, question)) + return "query-123" + + def log_feedback(self, query_id, verdict, reason): + self.calls.append(("log_feedback", query_id, verdict, reason)) + + +class Providers: + def get(self, workspace_id): + return object() + + +@pytest.fixture(autouse=True) +def _no_stored_connection(monkeypatch): + """Keep the reconnect path away from a real database in these tests. + + `_resolve_db_id` falls back to stored credentials when nothing is connected; + that path has its own tests. Here the manager reports connected, so this + only guards the disconnected case. + """ + + async def _none(*args, **kwargs): + return None + + monkeypatch.setattr(query_ctrl.mdb, "get_recent_connection_by_db_id", _none) + monkeypatch.setattr(query_ctrl.mdb, "get_latest_recent_connection", _none) + + +# ── verify ────────────────────────────────────────────────────────── + + +def test_verify_scopes_knowledge_to_selected_database(): + db, kb = RecordingDB(), RecordingKB() + req = SimpleNamespace( + question="How many users signed up?", + sql="select count(*) from users", + restatement="Count all users", + ) + + result = asyncio.run(query_ctrl.verify(WORKSPACE, db, kb, req, db_id=SELECTED_DB)) + + assert db.calls[0] == ("connected", WORKSPACE, SELECTED_DB), ( + "connectivity must be checked against the selected database" + ) + assert kb.calls[0] == ("add_verified", WORKSPACE, SELECTED_DB), ( + "the verified example belongs to the database it was verified against" + ) + assert kb.calls[1] == ("trust_level", WORKSPACE, SELECTED_DB) + assert result["trust"]["db_id_seen"] == SELECTED_DB + assert DEFAULT_DB not in kb.db_ids_seen() + + +def test_verify_rejects_when_nothing_is_connected(): + db, kb = RecordingDB(connected=False), RecordingKB() + req = SimpleNamespace(question="q", sql="select 1", restatement="r") + + with pytest.raises(HTTPException) as excinfo: + asyncio.run(query_ctrl.verify(WORKSPACE, db, kb, req, db_id=SELECTED_DB)) + + assert excinfo.value.status_code == 409 + assert kb.calls == [], "nothing should be written when there is no database" + + +# ── feedback ──────────────────────────────────────────────────────── + + +def test_feedback_scopes_knowledge_and_starters_to_selected_database(): + db, kb, session_log = RecordingDB(), RecordingKB(), RecordingSessionLog() + req = SimpleNamespace( + query_id="query-123", + verdict="correct", + reason="answer matched", + question="How many secondary customers?", + sql="select count(*) from secondary_customers", + restatement="Count secondary customers", + ) + + out = asyncio.run( + query_ctrl.feedback(WORKSPACE, db, kb, session_log, req, db_id=SELECTED_DB) + ) + + assert kb.calls[0] == ("add_verified", WORKSPACE, SELECTED_DB) + assert ("trust_level", WORKSPACE, SELECTED_DB) in kb.calls + assert ("get_verified", WORKSPACE, SELECTED_DB) in kb.calls + assert DEFAULT_DB not in kb.db_ids_seen() + assert out["starters"] == [f"starter-from-{SELECTED_DB}"], ( + "starters must come from the database the user is actually on" + ) + + +def test_feedback_records_nothing_verified_when_answer_was_wrong(): + db, kb, session_log = RecordingDB(), RecordingKB(), RecordingSessionLog() + req = SimpleNamespace( + query_id="query-123", + verdict="wrong", + reason="bad join", + question="q", + sql="select 1", + restatement="r", + ) + + out = asyncio.run( + query_ctrl.feedback(WORKSPACE, db, kb, session_log, req, db_id=SELECTED_DB) + ) + + assert "add_verified" not in [c[0] for c in kb.calls] + assert "starters" not in out + assert session_log.calls[0][0] == "log_feedback" + + +# ── explain / execute ─────────────────────────────────────────────── + + +def test_explain_reads_the_dialect_of_the_selected_database(): + db = RecordingDB() + captured = {} + + async def fake_explain_sql(provider, sql, dialect): + captured["dialect"] = dialect + return {"explanation": "counts rows"} + + original = query_ctrl.explain_sql + query_ctrl.explain_sql = fake_explain_sql + try: + asyncio.run( + query_ctrl.explain( + WORKSPACE, + db, + Providers(), + SimpleNamespace(sql="select 1"), + db_id=SELECTED_DB, + ) + ) + finally: + query_ctrl.explain_sql = original + + assert captured["dialect"] == "postgres" + assert db.db_ids_seen("get_dialect") == [SELECTED_DB], ( + "the dialect must be read from the selected database, not the default" + ) + + +def test_execute_runs_against_the_selected_database(): + db = RecordingDB() + + asyncio.run( + query_ctrl.execute( + WORKSPACE, db, SimpleNamespace(sql="select 1"), db_id=SELECTED_DB + ) + ) + + executes = [c for c in db.calls if c[0] == "execute"] + assert executes == [("execute", WORKSPACE, "select 1", SELECTED_DB)] + + +# ── the full pipeline, streaming and not ──────────────────────────── + + +@pytest.fixture +def stubbed_pipeline(monkeypatch): + """Replace the model and SQL machinery so only db_id routing is under test.""" + + async def fake_generate_sql(*args, **kwargs): + return { + "sql": "SELECT id FROM secondary_orders", + "restatement": "secondary orders", + "chart": {"type": "table"}, + } + + async def fake_conversation_owned_by(workspace_id, user_id, conversation_id): + return False + + async def fake_save_query(**kwargs): + return None + + monkeypatch.setattr(query_ctrl, "generate_sql", fake_generate_sql) + monkeypatch.setattr( + query_ctrl, + "validate_sql", + lambda sql, schema, dialect: {"ok": True, "errors": []}, + ) + monkeypatch.setattr( + query_ctrl, "link_relevant_tables", lambda *a, **k: ["secondary_orders"] + ) + monkeypatch.setattr( + query_ctrl, + "compact_schema", + lambda schema, tables, samples: "secondary_orders(id)", + ) + monkeypatch.setattr( + query_ctrl, + "compute_confidence", + lambda retrieved, exec_result: ("high", "r", True), + ) + monkeypatch.setattr(query_ctrl, "filter_catalog", lambda catalog, tables: catalog) + monkeypatch.setattr( + query_ctrl.mdb, "conversation_owned_by", fake_conversation_owned_by + ) + monkeypatch.setattr(query_ctrl.mdb, "save_query", fake_save_query) + + +def test_streaming_query_keeps_knowledge_schema_and_log_on_one_database( + stubbed_pipeline, +): + db, kb, session_log = RecordingDB(), RecordingKB(), RecordingSessionLog() + req = QueryReq(question="Show secondary orders", context=[], conversation_id=None) + + async def collect(): + return [ + event + async for event in query_ctrl.run_query_stream( + WORKSPACE, + db, + kb, + None, + Providers(), + session_log, + req, + db_id=SELECTED_DB, + user_id="user-1", + ) + ] + + events = asyncio.run(collect()) + + assert kb.db_ids_seen() == [SELECTED_DB] * len(kb.calls), ( + f"knowledge was read from the wrong database: {kb.calls}" + ) + assert db.db_ids_seen("get_schema") == [SELECTED_DB] + assert [c[3] for c in db.calls if c[0] == "execute"] == [SELECTED_DB] + assert session_log.calls == [("log_query", SELECTED_DB, "Show secondary orders")], ( + "the query must be logged against the database it ran on" + ) + + result = events[-1] + assert result["kind"] == "result" + assert result["data"]["query_id"] == "query-123" + + +def test_non_streaming_query_keeps_knowledge_schema_and_log_on_one_database( + stubbed_pipeline, monkeypatch +): + db, kb, session_log = RecordingDB(), RecordingKB(), RecordingSessionLog() + + async def fake_repair_loop(*args, **kwargs): + return { + "sql": "SELECT id FROM secondary_orders", + "restatement": "secondary orders", + "chart": {"type": "table"}, + "result": {"columns": ["id"], "rows": [[1]], "truncated": False}, + "attempts": [{"errors": []}], + "ok": True, + } + + monkeypatch.setattr(query_ctrl, "run_repair_loop", fake_repair_loop) + req = QueryReq(question="Show secondary orders", context=[], conversation_id=None) + + out = asyncio.run( + query_ctrl.run_query( + WORKSPACE, + db, + kb, + None, + Providers(), + session_log, + req, + db_id=SELECTED_DB, + user_id="user-1", + ) + ) + + assert kb.db_ids_seen() == [SELECTED_DB] * len(kb.calls), ( + f"knowledge was read from the wrong database: {kb.calls}" + ) + assert db.db_ids_seen("get_schema") == [SELECTED_DB] + assert [c[1] for c in session_log.calls if c[0] == "log_query"] == [SELECTED_DB] + assert out["query_id"] == "query-123" + + +def test_selection_falls_back_to_the_workspace_default_when_unspecified(): + """No `X-Db-Id` still has to resolve to a concrete id, once.""" + db, kb = RecordingDB(), RecordingKB() + req = SimpleNamespace(question="q", sql="select 1", restatement="r") + + asyncio.run(query_ctrl.verify(WORKSPACE, db, kb, req, db_id=None)) + + assert kb.calls[0] == ("add_verified", WORKSPACE, DEFAULT_DB) + assert kb.calls[1] == ("trust_level", WORKSPACE, DEFAULT_DB) diff --git a/tests/test_query_feedback_verify.py b/tests/test_query_feedback_verify.py index 4fbc6298..d82058ae 100644 --- a/tests/test_query_feedback_verify.py +++ b/tests/test_query_feedback_verify.py @@ -1,5 +1,5 @@ """Tests for backend/app/controllers/query.py's ``feedback`` and ``verify`` -functions — both now scope every knowledge-base call to (user_id, db_id) via +functions — both now scope every knowledge-base call to (workspace_id, db_id) via the async KnowledgeService. """ @@ -17,10 +17,10 @@ def __init__(self, connected=True, db_id="db-1"): self._connected = connected self._db_id = db_id - def connected(self, user_id): + def connected(self, workspace_id): return self._connected - def get_db_id(self, user_id): + def get_db_id(self, workspace_id): return self._db_id @@ -32,15 +32,17 @@ def __init__(self, trust="trust-value", verified=None): self._trust = trust self._verified = verified if verified is not None else [] - async def add_verified(self, user_id, db_id, question, sql, restatement=""): - self.add_verified_calls.append((user_id, db_id, question, sql, restatement)) + async def add_verified(self, workspace_id, db_id, question, sql, restatement=""): + self.add_verified_calls.append( + (workspace_id, db_id, question, sql, restatement) + ) - async def trust_level(self, user_id, db_id): - self.trust_level_calls.append((user_id, db_id)) + async def trust_level(self, workspace_id, db_id): + self.trust_level_calls.append((workspace_id, db_id)) return self._trust - async def get_verified(self, user_id, db_id): - self.get_verified_calls.append((user_id, db_id)) + async def get_verified(self, workspace_id, db_id): + self.get_verified_calls.append((workspace_id, db_id)) return self._verified @@ -79,7 +81,7 @@ def test_feedback_raises_409_when_not_connected(): with pytest.raises(HTTPException) as exc_info: asyncio.run( query_ctrl.feedback( - "u1", db, kb, session_log, _feedback_req(verdict="correct") + "w1", db, kb, session_log, _feedback_req(verdict="correct") ) ) assert exc_info.value.status_code == 409 @@ -91,7 +93,7 @@ def test_feedback_logs_feedback_with_request_fields(): session_log = DummySessionLog() req = _feedback_req(verdict="incorrect", reason="wrong table") - asyncio.run(query_ctrl.feedback("u1", db, kb, session_log, req)) + asyncio.run(query_ctrl.feedback("w1", db, kb, session_log, req)) assert session_log.feedback_calls == [("q-1", "incorrect", "wrong table")] @@ -102,10 +104,10 @@ def test_feedback_correct_verdict_adds_verified_scoped_to_user_and_db(): session_log = DummySessionLog() req = _feedback_req(verdict="correct") - asyncio.run(query_ctrl.feedback("u1", db, kb, session_log, req)) + asyncio.run(query_ctrl.feedback("w1", db, kb, session_log, req)) assert kb.add_verified_calls == [ - ("u1", "db-42", "How many orders?", "SELECT 1", "One") + ("w1", "db-42", "How many orders?", "SELECT 1", "One") ] @@ -115,7 +117,7 @@ def test_feedback_incorrect_verdict_does_not_add_verified(): session_log = DummySessionLog() req = _feedback_req(verdict="incorrect") - asyncio.run(query_ctrl.feedback("u1", db, kb, session_log, req)) + asyncio.run(query_ctrl.feedback("w1", db, kb, session_log, req)) assert kb.add_verified_calls == [] @@ -126,11 +128,11 @@ def test_feedback_returns_trust_level_scoped_to_user_and_db(): session_log = DummySessionLog() req = _feedback_req(verdict="incorrect") - result = asyncio.run(query_ctrl.feedback("u1", db, kb, session_log, req)) + result = asyncio.run(query_ctrl.feedback("w1", db, kb, session_log, req)) assert result["ok"] is True assert result["trust"] == "trust:db-42" - assert kb.trust_level_calls == [("u1", "db-42")] + assert kb.trust_level_calls == [("w1", "db-42")] def test_feedback_correct_verdict_includes_starters_from_verified(): @@ -140,11 +142,11 @@ def test_feedback_correct_verdict_includes_starters_from_verified(): session_log = DummySessionLog() req = _feedback_req(verdict="correct") - result = asyncio.run(query_ctrl.feedback("u1", db, kb, session_log, req)) + result = asyncio.run(query_ctrl.feedback("w1", db, kb, session_log, req)) # Starters are capped at the first 6 verified questions. assert result["starters"] == [f"q{i}" for i in range(6)] - assert kb.get_verified_calls == [("u1", "db-42")] + assert kb.get_verified_calls == [("w1", "db-42")] def test_feedback_incorrect_verdict_has_no_starters_key(): @@ -153,7 +155,7 @@ def test_feedback_incorrect_verdict_has_no_starters_key(): session_log = DummySessionLog() req = _feedback_req(verdict="incorrect") - result = asyncio.run(query_ctrl.feedback("u1", db, kb, session_log, req)) + result = asyncio.run(query_ctrl.feedback("w1", db, kb, session_log, req)) assert "starters" not in result assert kb.get_verified_calls == [] @@ -164,7 +166,7 @@ def test_verify_raises_409_when_not_connected(): kb = DummyKB() with pytest.raises(HTTPException) as exc_info: - asyncio.run(query_ctrl.verify("u1", db, kb, _verify_req())) + asyncio.run(query_ctrl.verify("w1", db, kb, _verify_req())) assert exc_info.value.status_code == 409 @@ -173,9 +175,9 @@ def test_verify_adds_verified_scoped_to_user_and_db(): kb = DummyKB() req = _verify_req(question="Top products?", sql="SELECT 2", restatement="Top") - asyncio.run(query_ctrl.verify("u1", db, kb, req)) + asyncio.run(query_ctrl.verify("w1", db, kb, req)) - assert kb.add_verified_calls == [("u1", "db-7", "Top products?", "SELECT 2", "Top")] + assert kb.add_verified_calls == [("w1", "db-7", "Top products?", "SELECT 2", "Top")] def test_verify_returns_ok_and_trust_level_scoped_to_user_and_db(): @@ -183,7 +185,7 @@ def test_verify_returns_ok_and_trust_level_scoped_to_user_and_db(): kb = DummyKB(trust="trust:db-7") req = _verify_req() - result = asyncio.run(query_ctrl.verify("u1", db, kb, req)) + result = asyncio.run(query_ctrl.verify("w1", db, kb, req)) assert result == {"ok": True, "trust": "trust:db-7"} - assert kb.trust_level_calls == [("u1", "db-7")] + assert kb.trust_level_calls == [("w1", "db-7")] diff --git a/tests/test_query_pipeline.py b/tests/test_query_pipeline.py index 8432e33a..44486ed4 100644 --- a/tests/test_query_pipeline.py +++ b/tests/test_query_pipeline.py @@ -26,19 +26,19 @@ def __init__(self, exec_results=None): self.exec_results = list(exec_results or []) self.executed = [] - def connected(self, user_id): + def connected(self, workspace_id): return True - def get_db_id(self, user_id): + def get_db_id(self, workspace_id): return "db1" - def get_schema(self, user_id): + def get_schema(self, workspace_id): return SCHEMA - def get_dialect(self, user_id): + def get_dialect(self, workspace_id): return "sqlite" - def execute(self, user_id, sql): + def execute(self, workspace_id, sql): self.executed.append(sql) if self.exec_results: return self.exec_results.pop(0) @@ -51,13 +51,13 @@ class FakeKB: def __init__(self, catalog=None): self._catalog = catalog or {} - async def get_glossary(self, user_id, db_id): + async def get_glossary(self, workspace_id, db_id): return [] - async def get_catalog(self, user_id, db_id): + async def get_catalog(self, workspace_id, db_id): return self._catalog - async def retrieve_similar(self, user_id, db_id, q, k=3): + async def retrieve_similar(self, workspace_id, db_id, q, k=3): return [] @@ -79,7 +79,7 @@ class FakeProviders: def __init__(self, provider): self._p = provider - def get(self, user_id): + def get(self, workspace_id): return self._p @@ -105,7 +105,7 @@ def _sql_json(sql, restatement="r"): def _run(provider, db=None): return asyncio.run( run_query( - "u1", + "w1", db or FakeDB(), FakeKB(), CFG, @@ -212,7 +212,7 @@ def __init__(self, schema): super().__init__() self._schema = schema - def get_schema(self, user_id): + def get_schema(self, workspace_id): return self._schema @@ -229,7 +229,7 @@ def test_schema_retry_adds_table_the_model_asked_for(): ) out = asyncio.run( run_query( - "u1", + "w1", db, FakeKB(), CFG, @@ -272,7 +272,7 @@ def test_shortlist_pass_runs_on_big_schemas_and_boosts_picks(): ) out = asyncio.run( run_query( - "u1", + "w1", db, FakeKB(), CFG, @@ -299,7 +299,7 @@ def test_shortlist_failure_falls_back_to_local_linking(): ) out = asyncio.run( run_query( - "u1", + "w1", db, FakeKB(), CFG, diff --git a/tests/test_query_stream.py b/tests/test_query_stream.py index 611b52d2..758cf5ff 100644 --- a/tests/test_query_stream.py +++ b/tests/test_query_stream.py @@ -31,7 +31,7 @@ def _collect(provider, db=None, monkeypatch_saves=None, kb=None): async def _run(): events = [] async for ev in run_query_stream( - "u1", + "w1", db or FakeDB(), kb or FakeKB(), CFG, diff --git a/tests/test_sample_data.py b/tests/test_sample_data.py new file mode 100644 index 00000000..b5eb6856 --- /dev/null +++ b/tests/test_sample_data.py @@ -0,0 +1,203 @@ +"""The sample webshop must load, join up, and read as current data. + +The sample store is a trimmed copy of the PostgreSQL sample database at +https://github.com/JannikArndt/PostgreSQLSampleDatabase, replayed into SQLite by +`backend.sample_data`. It is the first thing a new user sees, so a broken load, +a mangled `money` column or dates stuck in 2018 all show up as the product being +wrong rather than the data being old. +""" + +import sqlite3 +from datetime import datetime, timedelta + +import pytest + +from backend import sample_data + +EXPECTED_ROWS = { + "colors": 143, + "sizes": 15, + "labels": 1170, + "products": 1000, + "articles": 17730, + "stock": 17730, + "customers": 1000, + "addresses": 1000, + "orders": 2000, + "order_positions": 5985, +} + + +@pytest.fixture(scope="module") +def sample_db(tmp_path_factory): + """Build the sample database once, into a temp file, and open it.""" + path = tmp_path_factory.mktemp("sample") / "webshop.db" + sample_data._build(path) + conn = sqlite3.connect(str(path)) + conn.row_factory = sqlite3.Row + yield conn + conn.close() + + +def test_every_table_loads_with_its_full_row_count(sample_db): + counts = { + table: sample_db.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0] + for table in EXPECTED_ROWS + } + assert counts == EXPECTED_ROWS + + +def test_reserved_and_singular_table_names_are_renamed(sample_db): + """`order`/`customer`/`address` are hostile names to ask questions against.""" + names = { + r[0] + for r in sample_db.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ) + } + assert {"orders", "customers", "addresses"} <= names + assert not ({"order", "customer", "address"} & names) + + +def test_money_columns_load_as_numbers_not_dollar_strings(sample_db): + """`$361.81` has to become 361.81, or every SUM() in the demo is a no-op.""" + row = sample_db.execute( + "SELECT total, shippingcost FROM orders WHERE total IS NOT NULL LIMIT 1" + ).fetchone() + assert isinstance(row["total"], float) + assert isinstance(row["shippingcost"], float) + + total = sample_db.execute("SELECT ROUND(SUM(total), 2) FROM orders").fetchone()[0] + assert total > 0 + + non_numeric = sample_db.execute( + "SELECT COUNT(*) FROM orders WHERE typeof(total) NOT IN ('real', 'integer', 'null')" + ).fetchone()[0] + assert non_numeric == 0 + + +def test_booleans_and_nulls_survive_the_copy_format(sample_db): + actives = { + r[0] for r in sample_db.execute("SELECT DISTINCT currentlyactive FROM articles") + } + assert actives <= {0, 1} + assert 1 in actives, "some articles should be active" + + # `\N` in the dump means NULL, not the literal two characters. + literal_nulls = sample_db.execute( + "SELECT COUNT(*) FROM articles WHERE updated = '\\N'" + ).fetchone()[0] + assert literal_nulls == 0 + assert ( + sample_db.execute( + "SELECT COUNT(*) FROM articles WHERE updated IS NULL" + ).fetchone()[0] + > 0 + ) + + +def test_foreign_keys_all_resolve(sample_db): + """Orphaned rows would make every JOIN in the demo silently drop data.""" + orphans = { + "articles→products": "SELECT COUNT(*) FROM articles a LEFT JOIN products p ON p.id = a.productid WHERE p.id IS NULL", + "order_positions→orders": "SELECT COUNT(*) FROM order_positions op LEFT JOIN orders o ON o.id = op.orderid WHERE o.id IS NULL", + "order_positions→articles": "SELECT COUNT(*) FROM order_positions op LEFT JOIN articles a ON a.id = op.articleid WHERE a.id IS NULL", + "orders→customers": "SELECT COUNT(*) FROM orders o LEFT JOIN customers c ON c.id = o.customerid WHERE c.id IS NULL", + "stock→articles": "SELECT COUNT(*) FROM stock s LEFT JOIN articles a ON a.id = s.articleid WHERE a.id IS NULL", + } + found = { + name: sample_db.execute(sql).fetchone()[0] for name, sql in orphans.items() + } + assert found == dict.fromkeys(orphans, 0) + + +def test_order_history_runs_up_to_the_present_day(sample_db): + """Upstream's data stops in 2018; "last month" must still return rows.""" + newest = sample_db.execute("SELECT MAX(ordertimestamp) FROM orders").fetchone()[0] + newest_dt = datetime.strptime(newest[:19], "%Y-%m-%d %H:%M:%S") + assert datetime.now() - newest_dt < timedelta(days=2), ( + f"newest order is {newest} — sample data should end at today" + ) + + last_month = sample_db.execute( + "SELECT COUNT(*) FROM orders WHERE ordertimestamp >= date('now', '-1 month')" + ).fetchone()[0] + assert last_month > 0, "a 'last month' question must have something to answer with" + + +def test_history_still_spans_years_after_being_shifted(sample_db): + """Shifting must move the window, not compress it into a single day.""" + oldest, newest = sample_db.execute( + "SELECT MIN(ordertimestamp), MAX(ordertimestamp) FROM orders" + ).fetchone() + span = datetime.strptime(newest[:19], "%Y-%m-%d %H:%M:%S") - datetime.strptime( + oldest[:19], "%Y-%m-%d %H:%M:%S" + ) + assert span > timedelta(days=365) + + +def test_dates_of_birth_are_left_alone(sample_db): + """Only timestamps slide forward — shifting birthdays would invent infants.""" + newest_birth = sample_db.execute( + "SELECT MAX(dateofbirth) FROM customers" + ).fetchone()[0] + assert newest_birth < "2010-01-01" + + +@pytest.mark.parametrize( + "question,sql", + [ + ( + "orders last month", + "SELECT COUNT(*) AS orders_placed FROM orders " + "WHERE ordertimestamp >= date('now','start of month','-1 month') " + "AND ordertimestamp < date('now','start of month')", + ), + ( + "revenue by category", + "SELECT p.category, ROUND(SUM(op.amount * op.price), 2) AS revenue " + "FROM order_positions op JOIN articles a ON a.id = op.articleid " + "JOIN products p ON p.id = a.productid GROUP BY p.category ORDER BY revenue DESC", + ), + ( + "top customers by spend", + "SELECT c.firstname || ' ' || c.lastname AS customer, ROUND(SUM(o.total), 2) AS total_spend " + "FROM orders o JOIN customers c ON c.id = o.customerid " + "GROUP BY c.id ORDER BY total_spend DESC LIMIT 10", + ), + ( + "out of stock articles", + "SELECT a.id, a.description, s.count AS in_stock FROM articles a " + "JOIN stock s ON s.articleid = a.id WHERE s.count = 0 AND a.currentlyactive = 1", + ), + ], +) +def test_seeded_starter_questions_return_rows(sample_db, question, sql): + """The SQL seeded by `KnowledgeService.seed_sample` must run on this schema. + + These are shown as starter questions and used as verified examples, so SQL + that no longer matches the schema would teach the model the wrong shape. + """ + rows = sample_db.execute(sql).fetchall() + assert rows, f"seeded query for {question!r} returned nothing" + + +def test_build_is_atomic(tmp_path, monkeypatch): + """A failed build must not leave a half-loaded database for the next request.""" + target = tmp_path / "webshop.db" + + real_read_dump = sample_data._read_dump + + def exploding_dump(): + for i, block in enumerate(real_read_dump()): + if i >= 2: + raise RuntimeError("dump truncated") + yield block + + monkeypatch.setattr(sample_data, "_read_dump", exploding_dump) + + with pytest.raises(RuntimeError): + sample_data._build(target) + + assert not target.exists(), "a partial build must not be left in place" + assert list(tmp_path.iterdir()) == [], "the temporary file must be cleaned up" diff --git a/tests/test_schema_link.py b/tests/test_schema_link.py index ca32a862..279666ec 100644 --- a/tests/test_schema_link.py +++ b/tests/test_schema_link.py @@ -19,8 +19,8 @@ def small_schema(): "distinct_values": {}, }, "orders": { - "columns": [{"name": "id"}, {"name": "user_id"}], - "foreign_keys": [{"column": "user_id", "references": "users.id"}], + "columns": [{"name": "id"}, {"name": "workspace_id"}], + "foreign_keys": [{"column": "workspace_id", "references": "users.id"}], "row_count": 50, "distinct_values": {}, }, diff --git a/tests/test_structured_output.py b/tests/test_structured_output.py index a597f0d5..4403fdb7 100644 --- a/tests/test_structured_output.py +++ b/tests/test_structured_output.py @@ -2,7 +2,13 @@ import asyncio -from backend.app.llm import SQL_SCHEMA, generate_sql, parse_sql_output +from backend.app.llm import ( + DEFAULT_CHART, + SQL_SCHEMA, + generate_sql, + parse_chart_spec, + parse_sql_output, +) # --- parse_sql_output ------------------------------------------------------- @@ -12,7 +18,12 @@ def test_parse_dict_input_normalized(): out = parse_sql_output( {"sql": "SELECT 1", "restatement": "one", "assumptions": ["a"]} ) - assert out == {"sql": "SELECT 1", "restatement": "one", "assumptions": ["a"]} + assert out == { + "sql": "SELECT 1", + "restatement": "one", + "assumptions": ["a"], + "chart": DEFAULT_CHART, + } def test_parse_plain_json_text(): @@ -48,12 +59,22 @@ def test_blank_assumptions_string_is_empty_list(): def test_non_object_json_yields_defaults(): # A JSON array isn't a valid response object -> safe defaults, no raise. out = parse_sql_output("[1, 2, 3]") - assert out == {"sql": "", "restatement": "", "assumptions": []} + assert out == { + "sql": "", + "restatement": "", + "assumptions": [], + "chart": DEFAULT_CHART, + } def test_garbage_text_yields_defaults(): out = parse_sql_output("the model said no") - assert out == {"sql": "", "restatement": "", "assumptions": []} + assert out == { + "sql": "", + "restatement": "", + "assumptions": [], + "chart": DEFAULT_CHART, + } def test_non_string_sql_is_stringified(): @@ -63,7 +84,68 @@ def test_non_string_sql_is_stringified(): def test_empty_dict_defaults(): - assert parse_sql_output({}) == {"sql": "", "restatement": "", "assumptions": []} + assert parse_sql_output({}) == { + "sql": "", + "restatement": "", + "assumptions": [], + "chart": DEFAULT_CHART, + } + + +# --- chart spec ------------------------------------------------------------- + + +def test_chart_spec_kept_when_valid(): + out = parse_sql_output( + { + "sql": "x", + "restatement": "y", + "chart": { + "type": "line", + "x_axis": "month", + "y_axis": "revenue", + "title": "Revenue by month", + "reason": "a measure over time", + }, + } + ) + assert out["chart"] == { + "type": "line", + "x_axis": "month", + "y_axis": "revenue", + "title": "Revenue by month", + "reason": "a measure over time", + } + + +def test_chart_type_is_lowercased(): + assert parse_chart_spec({"type": "BAR"})["type"] == "bar" + + +def test_unknown_chart_type_falls_back_to_table(): + # A hallucinated type must not break rendering — a table always works. + assert parse_chart_spec({"type": "sankey", "x_axis": "a"}) == DEFAULT_CHART + + +def test_missing_chart_falls_back_to_table(): + assert parse_sql_output({"sql": "x", "restatement": "y"})["chart"] == DEFAULT_CHART + + +def test_non_dict_chart_falls_back_to_table(): + assert parse_chart_spec("line") == DEFAULT_CHART + assert parse_chart_spec(None) == DEFAULT_CHART + + +def test_chart_axis_fields_coerced_to_strings(): + spec = parse_chart_spec({"type": "bar", "x_axis": 7, "y_axis": " total "}) + assert spec["x_axis"] == "" # non-string dropped rather than stringified + assert spec["y_axis"] == "total" + + +def test_chart_default_is_not_shared_between_calls(): + first = parse_chart_spec({}) + first["type"] = "mutated" + assert parse_chart_spec({})["type"] == "table" # --- SQL_SCHEMA ------------------------------------------------------------- @@ -73,9 +155,19 @@ def test_schema_shape(): assert SQL_SCHEMA["name"] == "sql_result" schema = SQL_SCHEMA["schema"] assert schema["type"] == "object" - assert set(schema["required"]) == {"sql", "restatement", "assumptions"} + assert set(schema["required"]) == {"sql", "restatement", "assumptions", "chart"} assert schema["additionalProperties"] is False assert schema["properties"]["assumptions"]["type"] == "array" + chart = schema["properties"]["chart"] + assert chart["type"] == "object" + assert chart["properties"]["type"]["enum"] == [ + "table", + "bar", + "line", + "area", + "pie", + "number", + ] # --- generate_sql wiring ---------------------------------------------------- diff --git a/tests/test_system_controller.py b/tests/test_system_controller.py index d487ee21..d4c12bdc 100644 --- a/tests/test_system_controller.py +++ b/tests/test_system_controller.py @@ -19,16 +19,16 @@ def __init__(self, connected=True, info=None, dialect="sqlite"): self._info = info or {"url": "sqlite:///t.db", "db_id": "db1", "tables": 3} self._dialect = dialect - def connected(self, user_id): + def connected(self, workspace_id, db_id=None): return self._connected - def get_db_id(self, user_id): + def get_db_id(self, workspace_id, db_id=None): return self._info["db_id"] - def get_info(self, user_id): + def get_info(self, workspace_id, db_id=None): return self._info - def get_dialect(self, user_id): + def get_dialect(self, workspace_id, db_id=None): return self._dialect @@ -37,20 +37,20 @@ def __init__(self, count=0): self._count = count self.calls = [] - async def count_verified(self, user_id, db_id): - self.calls.append(("count_verified", user_id, db_id)) + async def count_verified(self, workspace_id, db_id): + self.calls.append(("count_verified", workspace_id, db_id)) return self._count - async def trust_level(self, user_id, db_id): - self.calls.append(("trust_level", user_id, db_id)) + async def trust_level(self, workspace_id, db_id): + self.calls.append(("trust_level", workspace_id, db_id)) return {"level": "Supervised", "verified": self._count, "pct": 8, "note": ""} - async def get_glossary(self, user_id, db_id): - self.calls.append(("get_glossary", user_id, db_id)) + async def get_glossary(self, workspace_id, db_id): + self.calls.append(("get_glossary", workspace_id, db_id)) return [{"term": "Revenue"}] - async def get_verified(self, user_id, db_id): - self.calls.append(("get_verified", user_id, db_id)) + async def get_verified(self, workspace_id, db_id): + self.calls.append(("get_verified", workspace_id, db_id)) return [{"question": f"q-{i}"} for i in range(8)] @@ -64,7 +64,9 @@ async def fake_get_user_by_id(user_id): monkeypatch.setattr(system_ctrl.mdb, "get_user_by_id", fake_get_user_by_id) db = DummyDB(connected=False) kb = DummyKB() - state = asyncio.run(system_ctrl.get_state("u1", db, {"last_db_url": "x"}, kb)) + state = asyncio.run( + system_ctrl.get_state("u1", "w1", "db1", db, {"last_db_url": "x"}, kb) + ) assert state["connected"] is False assert state["tour_completed"] is True assert "database" not in state @@ -77,7 +79,7 @@ async def fake_get_user_by_id(user_id): monkeypatch.setattr(system_ctrl.mdb, "get_user_by_id", fake_get_user_by_id) db = DummyDB(connected=False) kb = DummyKB() - state = asyncio.run(system_ctrl.get_state("u1", db, {}, kb)) + state = asyncio.run(system_ctrl.get_state("u1", "w1", "db1", db, {}, kb)) assert state["tour_completed"] is False @@ -89,27 +91,47 @@ async def fake_get_user_by_id(user_id): db = DummyDB(connected=False) kb = DummyKB() state = asyncio.run( - system_ctrl.get_state("u1", db, {"last_db_url": "sqlite:///secret.db"}, kb) + system_ctrl.get_state( + "u1", "w1", "db1", db, {"last_db_url": "sqlite:///secret.db"}, kb + ) ) assert "last_db_url" not in state["config"] +def _stub_recent_connection(monkeypatch, conn=None): + """The connected path looks up the stored connection for its alias. + + It is a real DB call keyed by workspace UUID, so it has to be stubbed out + for these controller-level tests. + """ + + async def fake_get_recent_connection_by_db_id(workspace_id, db_id): + return conn + + monkeypatch.setattr( + system_ctrl.mdb, + "get_recent_connection_by_db_id", + fake_get_recent_connection_by_db_id, + ) + + def test_get_state_connected_scopes_kb_calls_to_user_and_db(monkeypatch): async def fake_get_user_by_id(user_id): return {"tour_completed": False} monkeypatch.setattr(system_ctrl.mdb, "get_user_by_id", fake_get_user_by_id) + _stub_recent_connection(monkeypatch, {"alias_name": "Prod"}) db = DummyDB(connected=True, info={"url": "u", "db_id": "db-1", "tables": 5}) kb = DummyKB(count=8) - state = asyncio.run(system_ctrl.get_state("u1", db, {}, kb)) + state = asyncio.run(system_ctrl.get_state("u1", "w1", "db-1", db, {}, kb)) assert state["database"]["has_knowledge"] is True assert state["trust"]["level"] == "Supervised" assert state["glossary"] == [{"term": "Revenue"}] assert len(state["starters"]) == 6 # capped at 6 for call in kb.calls: - assert call[1] == "u1" + assert call[1] == "w1" assert call[2] == "db-1" @@ -118,9 +140,10 @@ async def fake_get_user_by_id(user_id): return {"tour_completed": False} monkeypatch.setattr(system_ctrl.mdb, "get_user_by_id", fake_get_user_by_id) + _stub_recent_connection(monkeypatch) db = DummyDB(connected=True) kb = DummyKB(count=0) - state = asyncio.run(system_ctrl.get_state("u1", db, {}, kb)) + state = asyncio.run(system_ctrl.get_state("u1", "w1", "db1", db, {}, kb)) assert state["database"]["has_knowledge"] is False @@ -161,15 +184,28 @@ async def fake_set_tour_completed(user_id): def test_state_route_passes_all_dependencies_to_controller(monkeypatch): called = {} - async def fake_get_state(user_id, db, cfg, kb): - called["args"] = (user_id, db, cfg, kb) + async def fake_get_current_workspace(x_workspace_id, user_token): + called["auth_workspace"] = (x_workspace_id, user_token) + return {"workspace_id": "verified-w1", "role": "member", "user_id": "u1"} + + async def fake_get_state(user_id, workspace_id, db_id, db, cfg, kb): + called["args"] = (user_id, workspace_id, db_id, db, cfg, kb) return {"connected": False} + monkeypatch.setattr( + system_routes, "get_current_workspace", fake_get_current_workspace + ) monkeypatch.setattr(system_routes.ctrl, "get_state", fake_get_state) result = asyncio.run( system_routes.state( - user_token={"user_id": "u1"}, db="db-ref", cfg="cfg-ref", kb="kb-ref" + user_token={"user_id": "u1"}, + x_workspace_id="w1", + x_db_id="db1", + db="db-ref", + cfg="cfg-ref", + kb="kb-ref", ) ) assert result == {"connected": False} - assert called["args"] == ("u1", "db-ref", "cfg-ref", "kb-ref") + assert called["auth_workspace"] == ("w1", {"user_id": "u1"}) + assert called["args"] == ("u1", "verified-w1", "db1", "db-ref", "cfg-ref", "kb-ref") diff --git a/tests/test_workspace_api_models.py b/tests/test_workspace_api_models.py new file mode 100644 index 00000000..17351ab0 --- /dev/null +++ b/tests/test_workspace_api_models.py @@ -0,0 +1,74 @@ +"""Tests for the request models in backend/app/models/workspace_api.py. + +Workspace-name limits and the bulk-invite cap are enforced here rather than +only in the browser, so a direct API call can't bypass them. +""" + +import pytest +from pydantic import ValidationError + +from backend.app.models.workspace_api import ( + MAX_BULK_INVITES, + WORKSPACE_NAME_MAX, + WorkspaceBulkInviteCreate, + WorkspaceCreate, + WorkspaceUpdate, +) + + +@pytest.mark.parametrize("name", ["Acme", " Acme Analytics ", "AB"]) +def test_workspace_create_accepts_reasonable_names(name): + assert WorkspaceCreate(name=name).name == name.strip() + + +@pytest.mark.parametrize("name", ["", " ", "A", " A "]) +def test_workspace_create_rejects_short_names(name): + with pytest.raises(ValidationError): + WorkspaceCreate(name=name) + + +def test_workspace_create_rejects_overlong_names(): + with pytest.raises(ValidationError): + WorkspaceCreate(name="x" * (WORKSPACE_NAME_MAX + 1)) + + +def test_workspace_create_accepts_a_name_at_the_limit(): + name = "x" * WORKSPACE_NAME_MAX + assert WorkspaceCreate(name=name).name == name + + +def test_workspace_update_allows_omitting_the_name(): + # PATCH with no name is a no-op update, not a validation error. + assert WorkspaceUpdate().name is None + assert WorkspaceUpdate(name=None).name is None + + +def test_workspace_update_validates_a_supplied_name(): + assert WorkspaceUpdate(name=" Acme ").name == "Acme" + with pytest.raises(ValidationError): + WorkspaceUpdate(name="A") + + +def test_bulk_invite_requires_at_least_one_address(): + with pytest.raises(ValidationError): + WorkspaceBulkInviteCreate(emails=[]) + + +def test_bulk_invite_caps_the_batch_size(): + ok = [f"user{i}@example.com" for i in range(MAX_BULK_INVITES)] + assert len(WorkspaceBulkInviteCreate(emails=ok).emails) == MAX_BULK_INVITES + with pytest.raises(ValidationError): + WorkspaceBulkInviteCreate(emails=ok + ["one.too.many@example.com"]) + + +def test_bulk_invite_accepts_malformed_addresses_for_per_row_reporting(): + """Validation is deliberately deferred: one bad address must not 422 the + whole batch — the controller reports it as an ``invalid`` row instead.""" + req = WorkspaceBulkInviteCreate(emails=["good@example.com", "not-an-email"]) + assert req.emails == ["good@example.com", "not-an-email"] + assert req.role == "member" + + +def test_bulk_invite_rejects_the_owner_role(): + with pytest.raises(ValidationError): + WorkspaceBulkInviteCreate(emails=["a@b.com"], role="owner") diff --git a/tests/test_workspace_defaults_empirical.py b/tests/test_workspace_defaults_empirical.py new file mode 100644 index 00000000..1ab8215b --- /dev/null +++ b/tests/test_workspace_defaults_empirical.py @@ -0,0 +1,486 @@ +"""Empirical verification test suite for Workspace Defaults integration. + +Tests cover: +1. Invite creation and resending using `default_invite_role` and `invite_expiry_days`. +2. Activity log retention filtering using `activity_retention_days`. +3. Fallbacks when WorkspaceSettings is missing. +4. Edge cases, boundary timestamps, precedence rules, and dynamic updates. +""" + +import uuid +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from fastapi import HTTPException + +import backend.app.controllers.workspaces as ws_ctrl +import backend.app.controllers.activity as activity_ctrl +from backend.app.models.workspace_settings import WorkspaceSettings + + +WORKSPACE_ID = str(uuid.uuid4()) +INVITER_ID = str(uuid.uuid4()) +MEMBER_USER_ID = str(uuid.uuid4()) + + +def _result(value): + res = Mock(name="QueryResult") + res.scalar_one_or_none = Mock(return_value=value) + return res + + +def _patch_session(monkeypatch, session, target_module=ws_ctrl): + factory = Mock() + factory.return_value = session + session.__aenter__.return_value = session + monkeypatch.setattr(target_module, "async_session", factory) + return factory + + +@pytest.fixture(autouse=True) +def _silence_side_effects(monkeypatch): + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + monkeypatch.setattr( + ws_ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + + +# ===================================================================== +# 1. INVITE CREATION & RESENDING WORKSPACE DEFAULTS TESTS +# ===================================================================== + + +@pytest.mark.asyncio +async def test_invite_user_uses_custom_default_invite_role(monkeypatch): + """When role is None, invite_user must default to settings.default_invite_role.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="admin", + invite_expiry_days=7, + ) + session = AsyncMock() + + res_settings = _result(settings) + res_user = _result(None) + res_member = _result(None) + res_invite = _result(None) + + session.execute.side_effect = [res_settings, res_user, res_member, res_invite] + session.get.return_value = SimpleNamespace(name="Acme Corp") + _patch_session(monkeypatch, session, ws_ctrl) + + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "new_admin@example.com", role=None, inviter_id=INVITER_ID + ) + + assert invite["role"] == "admin" + assert invite["email"] == "new_admin@example.com" + + +@pytest.mark.asyncio +async def test_invite_user_explicit_role_overrides_workspace_default(monkeypatch): + """Explicit role parameter must override default_invite_role in WorkspaceSettings.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="admin", + invite_expiry_days=7, + ) + session = AsyncMock() + + res_settings = _result(settings) + res_user = _result(None) + res_member = _result(None) + res_invite = _result(None) + + session.execute.side_effect = [res_settings, res_user, res_member, res_invite] + session.get.return_value = SimpleNamespace(name="Acme Corp") + _patch_session(monkeypatch, session, ws_ctrl) + + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, + "explicit_member@example.com", + role="member", + inviter_id=INVITER_ID, + ) + + assert invite["role"] == "member" + + +@pytest.mark.asyncio +async def test_invite_user_custom_expiry_days_calculation(monkeypatch): + """invite_expiry_days from settings must calculate exact expires_at timestamp.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="member", + invite_expiry_days=30, + ) + session = AsyncMock() + + res_settings = _result(settings) + res_user = _result(None) + res_member = _result(None) + res_invite = _result(None) + + session.execute.side_effect = [res_settings, res_user, res_member, res_invite] + session.get.return_value = SimpleNamespace(name="Acme Corp") + _patch_session(monkeypatch, session, ws_ctrl) + + start_time = datetime.now(timezone.utc) + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "thirty_day@example.com", role=None, inviter_id=INVITER_ID + ) + + expected_expiry = start_time + timedelta(days=30) + diff = abs((invite["expires_at"] - expected_expiry).total_seconds()) + assert diff < 5 + + +@pytest.mark.asyncio +async def test_invite_user_boundary_expiry_days(monkeypatch): + """Test boundary expiry values (1 day and 365 days).""" + for expiry_val in (1, 365): + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="member", + invite_expiry_days=expiry_val, + ) + session = AsyncMock() + + session.execute.side_effect = [ + _result(settings), + _result(None), + _result(None), + _result(None), + ] + session.get.return_value = SimpleNamespace(name="Acme Corp") + _patch_session(monkeypatch, session, ws_ctrl) + + start_time = datetime.now(timezone.utc) + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, + f"boundary_{expiry_val}@example.com", + role=None, + inviter_id=INVITER_ID, + ) + + expected_expiry = start_time + timedelta(days=expiry_val) + diff = abs((invite["expires_at"] - expected_expiry).total_seconds()) + assert diff < 5 + + +@pytest.mark.asyncio +async def test_invite_user_fallback_when_settings_missing(monkeypatch): + """When no WorkspaceSettings record exists, fallback to default role='member' and expiry=7 days.""" + session = AsyncMock() + + # Settings is None + res_settings = _result(None) + res_user = _result(None) + res_member = _result(None) + res_invite = _result(None) + + session.execute.side_effect = [res_settings, res_user, res_member, res_invite] + session.get.return_value = SimpleNamespace(name="Acme Corp") + _patch_session(monkeypatch, session, ws_ctrl) + + start_time = datetime.now(timezone.utc) + invite = await ws_ctrl.invite_user( + WORKSPACE_ID, "fallback@example.com", role=None, inviter_id=INVITER_ID + ) + + assert invite["role"] == "member" + expected_expiry = start_time + timedelta(days=7) + diff = abs((invite["expires_at"] - expected_expiry).total_seconds()) + assert diff < 5 + + +@pytest.mark.asyncio +async def test_invite_user_invalid_role_raises_400(monkeypatch): + """Invalid role string (e.g. 'owner' or 'guest') must raise HTTP 400.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="member", + invite_expiry_days=7, + ) + session = AsyncMock() + session.execute.return_value = _result(settings) + _patch_session(monkeypatch, session, ws_ctrl) + + for invalid_role in ("owner", "superadmin", "guest"): + with pytest.raises(HTTPException) as exc_info: + await ws_ctrl.invite_user( + WORKSPACE_ID, + "invalid_role@example.com", + role=invalid_role, + inviter_id=INVITER_ID, + ) + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == "Invalid role for invite" + + +@pytest.mark.asyncio +async def test_resend_invite_uses_custom_invite_expiry_days(monkeypatch): + """Resending invite must extend expires_at using WorkspaceSettings.invite_expiry_days.""" + invite_id = str(uuid.uuid4()) + existing_invite = SimpleNamespace( + id=uuid.UUID(invite_id), + workspace_id=uuid.UUID(WORKSPACE_ID), + email="resend@example.com", + token="mock_invite_token_val", + accepted_at=None, + expires_at=datetime.now(timezone.utc) - timedelta(days=1), # Expired + ) + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + invite_expiry_days=14, + ) + + session = AsyncMock() + session.execute.side_effect = [ + _result(existing_invite), # _get_invite query + _result(settings), # WorkspaceSettings query + ] + session.get.return_value = SimpleNamespace(name="Acme Corp") + _patch_session(monkeypatch, session, ws_ctrl) + + start_time = datetime.now(timezone.utc) + resend_res = await ws_ctrl.resend_invite( + WORKSPACE_ID, invite_id, actor_id=INVITER_ID + ) + + assert resend_res["ok"] is True + expected_expiry = start_time + timedelta(days=14) + diff = abs((resend_res["expires_at"] - expected_expiry).total_seconds()) + assert diff < 5 + + +@pytest.mark.asyncio +async def test_resend_invite_fallback_when_settings_missing(monkeypatch): + """Resending invite with missing WorkspaceSettings falls back to 7 days expiry.""" + invite_id = str(uuid.uuid4()) + existing_invite = SimpleNamespace( + id=uuid.UUID(invite_id), + workspace_id=uuid.UUID(WORKSPACE_ID), + email="resend_fallback@example.com", + token="mock_invite_token_val", + accepted_at=None, + expires_at=datetime.now(timezone.utc) + timedelta(days=1), + ) + + session = AsyncMock() + session.execute.side_effect = [ + _result(existing_invite), + _result(None), # Settings missing + ] + session.get.return_value = SimpleNamespace(name="Acme Corp") + _patch_session(monkeypatch, session, ws_ctrl) + + start_time = datetime.now(timezone.utc) + resend_res = await ws_ctrl.resend_invite( + WORKSPACE_ID, invite_id, actor_id=INVITER_ID + ) + + assert resend_res["ok"] is True + expected_expiry = start_time + timedelta(days=7) + diff = abs((resend_res["expires_at"] - expected_expiry).total_seconds()) + assert diff < 5 + + +# ===================================================================== +# 2. ACTIVITY LOG RETENTION FILTERING TESTS +# ===================================================================== + + +@pytest.mark.asyncio +async def test_get_workspace_activity_filters_by_custom_retention_days(monkeypatch): + """get_workspace_activity must filter logs based on WorkspaceSettings.activity_retention_days.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + activity_retention_days=10, + ) + session = AsyncMock() + + res_settings = _result(settings) + + # 4 log entries at different ages + now = datetime.now(timezone.utc) + log_2d = ( + SimpleNamespace( + id=uuid.uuid4(), + workspace_id=uuid.UUID(WORKSPACE_ID), + actor_id=uuid.UUID(INVITER_ID), + event_type="event.2d", + resource_type="test", + resource_id=None, + metadata_={}, + created_at=now - timedelta(days=2), + ), + "user2d@example.com", + ) + log_5d = ( + SimpleNamespace( + id=uuid.uuid4(), + workspace_id=uuid.UUID(WORKSPACE_ID), + actor_id=uuid.UUID(INVITER_ID), + event_type="event.5d", + resource_type="test", + resource_id=None, + metadata_={}, + created_at=now - timedelta(days=5), + ), + "user5d@example.com", + ) + + res_activity = Mock() + # Mock return values as filtered by SQL query based on cutoff_date + res_activity.all.return_value = [log_2d, log_5d] + + session.execute.side_effect = [res_settings, res_activity] + _patch_session(monkeypatch, session, activity_ctrl) + + logs = await activity_ctrl.get_workspace_activity(WORKSPACE_ID, limit=50) + + # Verify cutoff date calculation passed to SQL execution statement + call_args = session.execute.call_args_list + assert len(call_args) == 2 + stmt = call_args[1].args[0] + + # Extract cutoff date from query parameters + params = stmt.compile().params + cutoff_param = next(v for v in params.values() if isinstance(v, datetime)) + + expected_cutoff = now - timedelta(days=10) + diff = abs((cutoff_param - expected_cutoff).total_seconds()) + assert diff < 5 + + assert len(logs) == 2 + assert [log["event_type"] for log in logs] == ["event.2d", "event.5d"] + + +@pytest.mark.asyncio +async def test_get_workspace_activity_fallback_retention_when_settings_missing( + monkeypatch, +): + """When WorkspaceSettings is missing, activity_retention_days falls back to 30 days.""" + session = AsyncMock() + + res_settings = _result(None) + res_activity = Mock() + res_activity.all.return_value = [] + + session.execute.side_effect = [res_settings, res_activity] + _patch_session(monkeypatch, session, activity_ctrl) + + now = datetime.now(timezone.utc) + await activity_ctrl.get_workspace_activity(WORKSPACE_ID, limit=50) + + call_args = session.execute.call_args_list + stmt = call_args[1].args[0] + params = stmt.compile().params + cutoff_param = next(v for v in params.values() if isinstance(v, datetime)) + + expected_cutoff = now - timedelta(days=30) + diff = abs((cutoff_param - expected_cutoff).total_seconds()) + assert diff < 5 + + +@pytest.mark.asyncio +async def test_iter_workspace_activity_respects_custom_retention_days(monkeypatch): + """iter_workspace_activity generator must respect workspace activity retention cutoff.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + activity_retention_days=15, + ) + session = AsyncMock() + + res_settings = _result(settings) + now = datetime.now(timezone.utc) + log_item = ( + SimpleNamespace( + id=uuid.uuid4(), + workspace_id=uuid.UUID(WORKSPACE_ID), + actor_id=uuid.UUID(INVITER_ID), + event_type="exported.event", + resource_type="export", + resource_id=None, + metadata_={}, + created_at=now - timedelta(days=3), + ), + "exporter@example.com", + ) + + res_activity = Mock() + res_activity.all.return_value = [log_item] + + session.execute.side_effect = [res_settings, res_activity] + _patch_session(monkeypatch, session, activity_ctrl) + + retained_logs = [] + async for row in activity_ctrl.iter_workspace_activity(WORKSPACE_ID): + retained_logs.append(row) + + assert len(retained_logs) == 1 + assert retained_logs[0]["event_type"] == "exported.event" + + +@pytest.mark.asyncio +async def test_get_workspace_activity_combines_event_type_filter_and_retention( + monkeypatch, +): + """get_workspace_activity applies both retention cutoff date and event_type filter.""" + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + activity_retention_days=60, + ) + session = AsyncMock() + + res_settings = _result(settings) + res_activity = Mock() + res_activity.all.return_value = [] + + session.execute.side_effect = [res_settings, res_activity] + _patch_session(monkeypatch, session, activity_ctrl) + + await activity_ctrl.get_workspace_activity( + WORKSPACE_ID, limit=20, event_type="member.invite_sent" + ) + + stmt = session.execute.call_args_list[1].args[0] + compiled_str = str(stmt) + + assert "workspace_activity_log.event_type =" in compiled_str + assert "workspace_activity_log.created_at >=" in compiled_str + + +@pytest.mark.asyncio +async def test_update_workspace_settings_modifies_defaults(monkeypatch): + """Updating settings changes default_invite_role, invite_expiry_days, and activity_retention_days.""" + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="member", + invite_expiry_days=7, + activity_retention_days=30, + role_permissions={}, + ) + session = AsyncMock() + session.execute.return_value = _result(existing) + _patch_session(monkeypatch, session, ws_ctrl) + + update_payload = { + "default_invite_role": "admin", + "invite_expiry_days": 90, + "activity_retention_days": 180, + } + + updated = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, update_payload, actor_id=INVITER_ID + ) + + assert updated["default_invite_role"] == "admin" + assert updated["invite_expiry_days"] == 90 + assert updated["activity_retention_days"] == 180 + assert existing.default_invite_role == "admin" + assert existing.invite_expiry_days == 90 + assert existing.activity_retention_days == 180 diff --git a/tests/test_workspace_lifecycle.py b/tests/test_workspace_lifecycle.py new file mode 100644 index 00000000..3d0b0610 --- /dev/null +++ b/tests/test_workspace_lifecycle.py @@ -0,0 +1,411 @@ +"""Tests for the workspace lifecycle controllers added alongside delete/leave +and invite management: ``delete_workspace``, ``leave_workspace``, +``list_pending_invites``, ``rescind_invite`` and ``resend_invite``. + +The DB is mocked with the same session-factory pattern used by +``test_pgdatabase_users.py`` — ``async_session`` is a plain sync callable that +returns an AsyncSession, so the factory must be a ``Mock``, not an ``AsyncMock``. +""" + +import uuid +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from fastapi import HTTPException + +import backend.app.controllers.workspaces as ctrl + +WORKSPACE_ID = str(uuid.uuid4()) +USER_ID = str(uuid.uuid4()) +INVITE_ID = str(uuid.uuid4()) + + +def _patch_session(monkeypatch, session): + factory = Mock() + factory.return_value = session + session.__aenter__.return_value = session + monkeypatch.setattr(ctrl, "async_session", factory) + return factory + + +@pytest.fixture(autouse=True) +def _silence_side_effects(monkeypatch): + """Activity logging and email both open their own connections.""" + monkeypatch.setattr(ctrl, "log_activity", AsyncMock()) + monkeypatch.setattr( + ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + + +def _result(value): + result = Mock() + result.scalar_one_or_none.return_value = value + return result + + +def _member(role="member"): + return SimpleNamespace( + workspace_id=uuid.UUID(WORKSPACE_ID), user_id=uuid.UUID(USER_ID), role=role + ) + + +def _invite(expires_in_days=3, accepted_at=None): + return SimpleNamespace( + id=uuid.UUID(INVITE_ID), + workspace_id=uuid.UUID(WORKSPACE_ID), + email="new@example.com", + role="member", + token="BOLO-TEST-TOKEN", + accepted_at=accepted_at, + expires_at=datetime.now(timezone.utc) + timedelta(days=expires_in_days), + ) + + +# ── delete_workspace ── + + +@pytest.mark.asyncio +async def test_delete_workspace_removes_the_row(monkeypatch): + session = AsyncMock() + session.get.return_value = SimpleNamespace(id=WORKSPACE_ID, name="Acme") + _patch_session(monkeypatch, session) + + assert await ctrl.delete_workspace(WORKSPACE_ID, USER_ID) == {"ok": True} + # Deleting the single workspaces row is the whole operation — every + # workspace-scoped table cascades from it. + session.delete.assert_awaited_once() + session.commit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_delete_workspace_404s_when_missing(monkeypatch): + session = AsyncMock() + session.get.return_value = None + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.delete_workspace(WORKSPACE_ID, USER_ID) + assert exc.value.status_code == 404 + session.delete.assert_not_awaited() + + +# ── leave_workspace ── + + +@pytest.mark.asyncio +async def test_leave_workspace_removes_a_plain_member(monkeypatch): + session = AsyncMock() + session.execute.return_value = _result(_member("member")) + _patch_session(monkeypatch, session) + + assert await ctrl.leave_workspace(WORKSPACE_ID, USER_ID) == {"ok": True} + session.delete.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_sole_owner_cannot_leave(monkeypatch): + session = AsyncMock() + session.execute.return_value = _result(_member("owner")) + session.scalar.return_value = 1 + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.leave_workspace(WORKSPACE_ID, USER_ID) + assert exc.value.status_code == 400 + session.delete.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_owner_can_leave_when_another_owner_remains(monkeypatch): + session = AsyncMock() + session.execute.return_value = _result(_member("owner")) + session.scalar.return_value = 2 + _patch_session(monkeypatch, session) + + assert await ctrl.leave_workspace(WORKSPACE_ID, USER_ID) == {"ok": True} + session.delete.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_leave_workspace_404s_for_non_member(monkeypatch): + session = AsyncMock() + session.execute.return_value = _result(None) + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.leave_workspace(WORKSPACE_ID, USER_ID) + assert exc.value.status_code == 404 + + +# ── list_pending_invites ── + + +@pytest.mark.asyncio +async def test_list_pending_invites_shape(monkeypatch): + invite = _invite() + session = AsyncMock() + result = Mock() + result.all.return_value = [(invite, "admin@example.com")] + session.execute.return_value = result + _patch_session(monkeypatch, session) + + rows = await ctrl.list_pending_invites(WORKSPACE_ID) + assert rows == [ + { + "id": INVITE_ID, + "email": "new@example.com", + "role": "member", + "invited_by": "admin@example.com", + "expires_at": invite.expires_at, + } + ] + # The token is deliberately absent — listing invites must not leak codes + # that would let an admin join as someone else. + assert "token" not in rows[0] + + +# ── rescind_invite ── + + +@pytest.mark.asyncio +async def test_rescind_invite_deletes_the_row(monkeypatch): + session = AsyncMock() + session.execute.return_value = _result(_invite()) + _patch_session(monkeypatch, session) + + assert await ctrl.rescind_invite(WORKSPACE_ID, INVITE_ID, USER_ID) == {"ok": True} + session.delete.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_rescind_invite_rejects_an_accepted_invite(monkeypatch): + session = AsyncMock() + session.execute.return_value = _result( + _invite(accepted_at=datetime.now(timezone.utc)) + ) + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.rescind_invite(WORKSPACE_ID, INVITE_ID, USER_ID) + assert exc.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_rescind_invite_404s_on_unknown_id(monkeypatch): + session = AsyncMock() + session.execute.return_value = _result(None) + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.rescind_invite(WORKSPACE_ID, INVITE_ID, USER_ID) + assert exc.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_rescind_invite_404s_on_malformed_id(monkeypatch): + """A non-UUID path segment is a missing invite, not a 500.""" + session = AsyncMock() + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.rescind_invite(WORKSPACE_ID, "not-a-uuid", USER_ID) + assert exc.value.status_code == 404 + + +# ── resend_invite ── + + +@pytest.mark.asyncio +async def test_resend_invite_keeps_a_live_token_and_extends_expiry(monkeypatch): + invite = _invite(expires_in_days=3) + original_token = invite.token + original_expiry = invite.expires_at + session = AsyncMock() + session.execute.return_value = _result(invite) + session.get.return_value = SimpleNamespace(name="Acme") + _patch_session(monkeypatch, session) + + res = await ctrl.resend_invite(WORKSPACE_ID, INVITE_ID, USER_ID) + + assert res["ok"] is True + assert res["email_sent"] is True + # Codes already in someone's inbox must keep working. + assert invite.token == original_token + assert invite.expires_at > original_expiry + ctrl.send_workspace_invite_email.assert_awaited_once_with( + "new@example.com", "Acme", original_token + ) + + +@pytest.mark.asyncio +async def test_resend_invite_regenerates_an_expired_token(monkeypatch): + invite = _invite(expires_in_days=-1) + original_token = invite.token + session = AsyncMock() + session.execute.return_value = _result(invite) + session.get.return_value = SimpleNamespace(name="Acme") + _patch_session(monkeypatch, session) + + await ctrl.resend_invite(WORKSPACE_ID, INVITE_ID, USER_ID) + + assert invite.token != original_token + assert invite.expires_at > datetime.now(timezone.utc) + + +@pytest.mark.asyncio +async def test_resend_invite_reports_when_email_is_not_configured(monkeypatch): + monkeypatch.setattr( + ctrl, "send_workspace_invite_email", AsyncMock(return_value=False) + ) + session = AsyncMock() + session.execute.return_value = _result(_invite()) + session.get.return_value = SimpleNamespace(name="Acme") + _patch_session(monkeypatch, session) + + res = await ctrl.resend_invite(WORKSPACE_ID, INVITE_ID, USER_ID) + assert res["ok"] is True + assert res["email_sent"] is False + + +@pytest.mark.asyncio +async def test_resend_invite_rejects_an_accepted_invite(monkeypatch): + session = AsyncMock() + session.execute.return_value = _result( + _invite(accepted_at=datetime.now(timezone.utc)) + ) + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.resend_invite(WORKSPACE_ID, INVITE_ID, USER_ID) + assert exc.value.status_code == 400 + ctrl.send_workspace_invite_email.assert_not_awaited() + + +# ── transfer_ownership ── + + +def _members_result(*members): + result = Mock() + scalars = Mock() + scalars.all.return_value = list(members) + result.scalars.return_value = scalars + return result + + +@pytest.mark.asyncio +async def test_transfer_ownership_promotes_target_and_demotes_actor(monkeypatch): + target_id = str(uuid.uuid4()) + actor = _member("owner") + target = SimpleNamespace( + workspace_id=uuid.UUID(WORKSPACE_ID), + user_id=uuid.UUID(target_id), + role="member", + ) + session = AsyncMock() + session.execute.return_value = _members_result(actor, target) + _patch_session(monkeypatch, session) + + res = await ctrl.transfer_ownership(WORKSPACE_ID, USER_ID, target_id) + + assert res == {"ok": True, "new_owner_id": target_id} + # Both sides move in the same transaction, so the workspace is never left + # with two owners or none. + assert target.role == "owner" + assert actor.role == "admin" + session.commit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_transfer_ownership_rejects_self(monkeypatch): + session = AsyncMock() + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.transfer_ownership(WORKSPACE_ID, USER_ID, USER_ID) + assert exc.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_transfer_ownership_404s_when_target_is_not_a_member(monkeypatch): + session = AsyncMock() + session.execute.return_value = _members_result(_member("owner")) + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.transfer_ownership(WORKSPACE_ID, USER_ID, str(uuid.uuid4())) + assert exc.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_transfer_ownership_403s_when_actor_is_not_owner(monkeypatch): + target_id = str(uuid.uuid4()) + actor = _member("admin") + target = SimpleNamespace( + workspace_id=uuid.UUID(WORKSPACE_ID), + user_id=uuid.UUID(target_id), + role="member", + ) + session = AsyncMock() + session.execute.return_value = _members_result(actor, target) + _patch_session(monkeypatch, session) + + with pytest.raises(HTTPException) as exc: + await ctrl.transfer_ownership(WORKSPACE_ID, USER_ID, target_id) + assert exc.value.status_code == 403 + assert target.role == "member" + + +# ── bulk_invite ── + + +@pytest.mark.asyncio +async def test_bulk_invite_reports_a_status_per_address(monkeypatch): + async def fake_invite_user(workspace_id, email, role, inviter_id): + if email == "taken@example.com": + raise HTTPException(400, "User is already a member") + return {"email": email} + + monkeypatch.setattr(ctrl, "invite_user", fake_invite_user) + + res = await ctrl.bulk_invite( + WORKSPACE_ID, + [" New@Example.com ", "taken@example.com", "not-an-email", "new@example.com"], + "member", + USER_ID, + ) + + assert res["total"] == 4 + assert res["invited"] == 1 + assert [r["status"] for r in res["results"]] == [ + "invited", + "already_member", + "invalid", + "duplicate", + ] + # Addresses are normalised, so casing and stray spaces don't create dupes. + assert res["results"][0]["email"] == "new@example.com" + + +@pytest.mark.asyncio +async def test_bulk_invite_continues_past_an_unexpected_failure(monkeypatch): + async def fake_invite_user(workspace_id, email, role, inviter_id): + if email == "boom@example.com": + raise HTTPException(400, "Could not process invite") + return {"email": email} + + monkeypatch.setattr(ctrl, "invite_user", fake_invite_user) + + res = await ctrl.bulk_invite( + WORKSPACE_ID, ["boom@example.com", "ok@example.com"], "member", USER_ID + ) + assert [r["status"] for r in res["results"]] == ["failed", "invited"] + assert res["invited"] == 1 + + +@pytest.mark.asyncio +async def test_bulk_invite_rejects_an_invalid_role(monkeypatch): + with pytest.raises(HTTPException) as exc: + await ctrl.bulk_invite(WORKSPACE_ID, ["a@b.com"], "owner", USER_ID) + assert exc.value.status_code == 400 diff --git a/tests/test_workspace_permission_defaults.py b/tests/test_workspace_permission_defaults.py new file mode 100644 index 00000000..aea7f7a5 --- /dev/null +++ b/tests/test_workspace_permission_defaults.py @@ -0,0 +1,103 @@ +"""Settings responses must say what the defaults are, not just the outcome. + +The permissions UI shows a *minimum role* per capability and marks the ones +still following the workspace default. It can only do that if the backend says +what the default is — otherwise the UI has to keep its own copy of the defaults, +which goes stale the moment the registry changes and then quietly shows a role +the server does not actually grant. +""" + +from backend.app.controllers.workspaces import _permission_matrix +from backend.app.permissions import ( + DEFAULT_ROLE_PERMISSIONS, + MEMBER_DEFAULT_PERMISSIONS, + PERMISSIONS, + resolve_role_permissions, +) + +ROLE_RANK = ["member", "admin", "owner"] + + +def min_role(matrix, key): + """The UI's reading of a matrix: lowest-ranked role granted this capability.""" + for role in ROLE_RANK: + if matrix[role][key]: + return role + return "owner" + + +def test_default_matrix_covers_every_registered_permission(): + defaults = _permission_matrix(None) + assert set(defaults) == {"owner", "admin", "member"} + for role in defaults: + assert set(defaults[role]) == set(PERMISSIONS), ( + f"{role} is missing capabilities the registry defines" + ) + + +def test_default_matrix_matches_the_registry_not_a_second_copy(): + defaults = _permission_matrix(None) + for role in ("owner", "admin", "member"): + assert defaults[role] == DEFAULT_ROLE_PERMISSIONS[role] + + +def test_defaults_are_already_rank_ordered(): + """No capability may reach a lower role without reaching the higher ones. + + The dropdown can only express "this role and above". A default that broke + that ordering could not be displayed faithfully. + """ + defaults = _permission_matrix(None) + for key in PERMISSIONS: + if defaults["member"][key]: + assert defaults["admin"][key], f"{key}: members have it but admins don't" + if defaults["admin"][key]: + assert defaults["owner"][key], f"{key}: admins have it but owners don't" + + +def test_owners_keep_every_capability_even_when_overridden(): + revoked = {"owner": dict.fromkeys(PERMISSIONS, False), "admin": {}, "member": {}} + matrix = _permission_matrix(revoked) + assert all(matrix["owner"].values()), "owner access is not revocable" + + +def test_a_custom_matrix_differs_from_the_defaults_only_where_overridden(): + overrides = {"member": {"connections.manage": True}} + resolved = _permission_matrix(overrides) + defaults = _permission_matrix(None) + + changed = [k for k in PERMISSIONS if resolved["member"][k] != defaults["member"][k]] + assert changed == ["connections.manage"] + assert resolved["admin"] == defaults["admin"], "an override must not leak roles" + + +def test_minimum_role_round_trips_through_the_stored_format(): + """What the dropdown writes must read back as the same selection. + + The UI stores a minimum role by expanding it into per-role booleans; if that + expansion and the reverse reading disagree, a saved choice comes back as a + different one. + """ + for chosen in ROLE_RANK: + key = "connections.manage" + overrides = { + "admin": {key: chosen in ("admin", "member")}, + "member": {key: chosen == "member"}, + } + assert min_role(_permission_matrix(overrides), key) == chosen + + +def test_member_defaults_are_read_only_capabilities_only(): + """Defaults should never hand members a capability that changes the workspace.""" + mutating = { + k + for k in PERMISSIONS + if any( + k.endswith(suffix) + for suffix in (".manage", ".remove", ".invite", ".update_role", ".settings") + ) + } + assert not (MEMBER_DEFAULT_PERMISSIONS & mutating), ( + "members must not get workspace-changing capabilities by default" + ) + assert resolve_role_permissions("member")["connections.manage"] is False diff --git a/tests/test_workspace_settings_api.py b/tests/test_workspace_settings_api.py new file mode 100644 index 00000000..a66525f4 --- /dev/null +++ b/tests/test_workspace_settings_api.py @@ -0,0 +1,284 @@ +"""Integration test suite for Workspace Settings API, require_permission dependency, +permission matrix overrides, invite defaults, and activity log retention. +""" + +import uuid +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from fastapi import HTTPException + +import backend.app.controllers.workspaces as ws_ctrl +import backend.app.controllers.activity as activity_ctrl +from backend.app.dependencies import require_permission +from backend.app.models.workspace_settings import WorkspaceSettings + +WORKSPACE_ID = str(uuid.uuid4()) +USER_ID = str(uuid.uuid4()) +ADMIN_ID = str(uuid.uuid4()) +MEMBER_ID = str(uuid.uuid4()) + + +def _result(value): + res = Mock(name="QueryResult") + res.scalar_one_or_none = Mock(return_value=value) + return res + + +def _patch_session(monkeypatch, session, target_module=ws_ctrl): + factory = Mock() + factory.return_value = session + session.__aenter__.return_value = session + monkeypatch.setattr(target_module, "async_session", factory) + return factory + + +@pytest.fixture(autouse=True) +def _silence_side_effects(monkeypatch): + monkeypatch.setattr(ws_ctrl, "log_activity", AsyncMock()) + monkeypatch.setattr( + ws_ctrl, "send_workspace_invite_email", AsyncMock(return_value=True) + ) + + +# ── 1. Settings Controller Tests ── + + +@pytest.mark.asyncio +async def test_get_workspace_settings_creates_defaults_if_missing(monkeypatch): + session = AsyncMock() + session.execute.return_value = _result(None) + + saved_settings = [] + + def fake_add(obj): + saved_settings.append(obj) + + session.add = fake_add + _patch_session(monkeypatch, session, ws_ctrl) + + settings = await ws_ctrl.get_workspace_settings(WORKSPACE_ID) + + assert settings["workspace_id"] == WORKSPACE_ID + assert settings["default_invite_role"] == "member" + assert settings["invite_expiry_days"] == 7 + assert settings["activity_retention_days"] == 30 + assert settings["role_permissions"] == {} + assert "resolved_matrix" in settings + assert settings["resolved_matrix"]["owner"]["queries.execute"] is True + assert settings["resolved_matrix"]["admin"]["dashboards.create"] is True + assert settings["resolved_matrix"]["member"]["dashboards.create"] is False + + +@pytest.mark.asyncio +async def test_update_workspace_settings_excludes_owner_overrides(monkeypatch): + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="member", + invite_expiry_days=7, + activity_retention_days=30, + role_permissions={}, + ) + session = AsyncMock() + session.execute.return_value = _result(existing) + _patch_session(monkeypatch, session, ws_ctrl) + + update_payload = { + "default_invite_role": "admin", + "invite_expiry_days": 14, + "activity_retention_days": 60, + "role_permissions": { + "owner": {"queries.execute": False}, # Must be excluded! + "member": {"dashboards.create": True}, + }, + } + + updated = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, update_payload, actor_id=USER_ID + ) + + assert updated["default_invite_role"] == "admin" + assert updated["invite_expiry_days"] == 14 + assert updated["activity_retention_days"] == 60 + assert "owner" not in updated["role_permissions"] + assert updated["role_permissions"] == {"member": {"dashboards.create": True}} + # Owner matrix remains all True + assert updated["resolved_matrix"]["owner"]["queries.execute"] is True + # Member matrix gets overridden + assert updated["resolved_matrix"]["member"]["dashboards.create"] is True + + +# ── 2. Invite Controller Integration Tests ── + + +@pytest.mark.asyncio +async def test_invite_user_uses_workspace_settings_defaults(monkeypatch): + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + default_invite_role="admin", + invite_expiry_days=14, + ) + session = AsyncMock() + + res_settings = _result(settings) + res_user = _result(None) + res_member = _result(None) + res_invite = _result(None) + + session.execute.side_effect = [res_settings, res_user, res_member, res_invite] + session.get.return_value = SimpleNamespace(name="Acme Corp") + _patch_session(monkeypatch, session, ws_ctrl) + + invite = await ws_ctrl.invite_user(WORKSPACE_ID, "test@example.com", None, USER_ID) + + assert invite["role"] == "admin" + expected_expiry = datetime.now(timezone.utc) + timedelta(days=14) + diff = abs((invite["expires_at"] - expected_expiry).total_seconds()) + assert diff < 60 + + +# ── 3. Activity Controller Integration Tests ── + + +@pytest.mark.asyncio +async def test_get_workspace_activity_uses_settings_retention_days(monkeypatch): + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + activity_retention_days=5, + ) + session = AsyncMock() + + res_settings = _result(settings) + + old_log = SimpleNamespace( + id=uuid.uuid4(), + workspace_id=uuid.UUID(WORKSPACE_ID), + actor_id=uuid.UUID(USER_ID), + event_type="test.event", + resource_type="test", + resource_id=None, + metadata_={}, + created_at=datetime.now(timezone.utc) - timedelta(days=2), + ) + res_activity = Mock() + res_activity.all.return_value = [(old_log, "user@example.com")] + + session.execute.side_effect = [res_settings, res_activity] + _patch_session(monkeypatch, session, activity_ctrl) + + logs = await activity_ctrl.get_workspace_activity(WORKSPACE_ID, limit=50) + + assert len(logs) == 1 + assert logs[0]["event_type"] == "test.event" + + +# ── 4. Require Permission & Route Integration Tests ── + + +@pytest.mark.asyncio +async def test_require_permission_owner_bypasses(monkeypatch): + dep = require_permission("dashboards.create") + workspace_context = { + "workspace_id": WORKSPACE_ID, + "role": "owner", + "user_id": USER_ID, + } + + res = await dep(workspace=workspace_context) + assert res == workspace_context + + +@pytest.mark.asyncio +async def test_require_permission_custom_override_denies(monkeypatch): + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"admin": {"dashboards.create": False}}, + ) + session = AsyncMock() + session.execute.return_value = _result(settings) + + from backend.app import dependencies + + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("dashboards.create") + workspace_context = { + "workspace_id": WORKSPACE_ID, + "role": "admin", + "user_id": ADMIN_ID, + } + + with pytest.raises(HTTPException) as exc_info: + await dep(workspace=workspace_context) + + assert exc_info.value.status_code == 403 + assert "Permission 'dashboards.create' required" in exc_info.value.detail + + +@pytest.mark.asyncio +async def test_require_permission_custom_override_allows(monkeypatch): + settings = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"member": {"dashboards.create": True}}, + ) + session = AsyncMock() + session.execute.return_value = _result(settings) + + from backend.app import dependencies + + _patch_session(monkeypatch, session, dependencies) + + dep = require_permission("dashboards.create") + workspace_context = { + "workspace_id": WORKSPACE_ID, + "role": "member", + "user_id": MEMBER_ID, + } + + res = await dep(workspace=workspace_context) + assert res == workspace_context + + +@pytest.mark.asyncio +async def test_update_workspace_settings_deep_merges_roles_and_strips_owner( + monkeypatch, +): + existing = WorkspaceSettings( + workspace_id=uuid.UUID(WORKSPACE_ID), + role_permissions={"admin": {"members.invite": False}}, + ) + session = AsyncMock() + session.execute.return_value = _result(existing) + _patch_session(monkeypatch, session, ws_ctrl) + + # First update member role permissions; admin permissions must be preserved. + res1 = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, + { + "role_permissions": { + "member": {"dashboards.create": True}, + "owner": {"foo": True}, + } + }, + actor_id=USER_ID, + ) + + assert "owner" not in res1["role_permissions"] + assert res1["role_permissions"]["admin"] == {"members.invite": False} + assert res1["role_permissions"]["member"] == {"dashboards.create": True} + + # Second update admin role permissions; existing member permissions and admin keys are merged. + res2 = await ws_ctrl.update_workspace_settings( + WORKSPACE_ID, + {"role_permissions": {"admin": {"queries.delete_saved": False}}}, + actor_id=USER_ID, + ) + + assert "owner" not in res2["role_permissions"] + assert res2["role_permissions"]["admin"] == { + "members.invite": False, + "queries.delete_saved": False, + } + assert res2["role_permissions"]["member"] == {"dashboards.create": True} diff --git a/tests/test_workspace_settings_empirically.py b/tests/test_workspace_settings_empirically.py new file mode 100644 index 00000000..148ba7ec --- /dev/null +++ b/tests/test_workspace_settings_empirically.py @@ -0,0 +1,396 @@ +"""Empirical verification test suite for WorkspaceSettings model defaults and Alembic migration e8f901a23b4c. + +Written by Challenger 2 to stress test model constraints, defaults, JSONB serialization, +and migration DDL definitions. +""" + +import uuid +from datetime import datetime, timezone + +import sqlalchemy as sa +from sqlalchemy import create_engine, select, String, Integer, DateTime +from sqlalchemy.dialects import postgresql +from sqlalchemy.dialects.postgresql import UUID as PgUUID, JSONB +from sqlalchemy.orm import sessionmaker, Mapped, mapped_column, declarative_base + +from backend.app.models.workspace_settings import WorkspaceSettings +from backend.alembic.versions import ( + e8f901a23b4c_add_workspace_settings_table as migration, +) + +SQLiteTestBase = declarative_base() + + +# ===================================================================== +# 1. MODEL DEFAULTS AND CONSTRAINT VERIFICATION +# ===================================================================== + + +def test_workspace_settings_model_defaults(): + """Verify exact model column defaults, types, and constraints on WorkspaceSettings.""" + table = WorkspaceSettings.__table__ + + # Table name check + assert table.name == "workspace_settings" + + # Column presence check + cols = {c.name: c for c in table.columns} + assert set(cols.keys()) == { + "workspace_id", + "default_invite_role", + "invite_expiry_days", + "activity_retention_days", + "role_permissions", + "created_at", + "updated_at", + } + + # Column 1: workspace_id + c_wid = cols["workspace_id"] + assert c_wid.primary_key is True + assert c_wid.nullable is False + assert isinstance(c_wid.type, PgUUID) + assert c_wid.type.as_uuid is True + + # Foreign key check on workspace_id + fk_list = list(c_wid.foreign_keys) + assert len(fk_list) == 1 + fk = fk_list[0] + assert fk._get_colspec() == "workspaces.id" + assert fk.ondelete == "CASCADE" + + # Column 2: default_invite_role + c_role = cols["default_invite_role"] + assert c_role.nullable is False + assert isinstance(c_role.type, String) + assert c_role.type.length == 20 + assert c_role.default.arg == "member" + + # Column 3: invite_expiry_days + c_expiry = cols["invite_expiry_days"] + assert c_expiry.nullable is False + assert isinstance(c_expiry.type, Integer) + assert c_expiry.default.arg == 7 + + # Column 4: activity_retention_days + c_retention = cols["activity_retention_days"] + assert c_retention.nullable is False + assert isinstance(c_retention.type, Integer) + assert c_retention.default.arg == 30 + + # Column 5: role_permissions + c_perms = cols["role_permissions"] + assert c_perms.nullable is False + assert isinstance(c_perms.type, JSONB) + assert c_perms.default.arg is dict or c_perms.default.arg.__name__ == "dict" + + # Columns 6 & 7: timestamps + c_created = cols["created_at"] + c_updated = cols["updated_at"] + assert c_created.nullable is False + assert isinstance(c_created.type, DateTime) + assert c_created.type.timezone is True + assert c_updated.nullable is False + assert isinstance(c_updated.type, DateTime) + assert c_updated.type.timezone is True + + +# ===================================================================== +# 2. MODEL INSTANTIATION & MUTABILITY ISOLATION +# ===================================================================== + + +def test_instantiation_with_defaults(): + """Verify instantiating WorkspaceSettings with minimal arguments populates default values.""" + test_uuid = uuid.uuid4() + ws = WorkspaceSettings(workspace_id=test_uuid) + + assert ws.workspace_id == test_uuid + assert ws.default_invite_role == "member" + assert ws.invite_expiry_days == 7 + assert ws.activity_retention_days == 30 + assert ws.role_permissions == {} + + +def test_instantiation_isolation(): + """Verify that multiple instances get distinct dictionary objects for role_permissions.""" + id1 = uuid.uuid4() + id2 = uuid.uuid4() + ws1 = WorkspaceSettings(workspace_id=id1) + ws2 = WorkspaceSettings(workspace_id=id2) + + assert ws1.role_permissions is not ws2.role_permissions + + # Mutating ws1 should not affect ws2 + ws1.role_permissions["admin"] = {"members.invite": False} + assert ws1.role_permissions == {"admin": {"members.invite": False}} + assert ws2.role_permissions == {} + + +def test_instantiation_with_custom_values(): + """Verify instantiating WorkspaceSettings with non-default values preserves kwargs.""" + test_uuid = uuid.uuid4() + custom_perms = { + "member": {"catalog.view": True, "members.invite": True}, + "admin": {"workspace.delete": False}, + } + ws = WorkspaceSettings( + workspace_id=test_uuid, + default_invite_role="admin", + invite_expiry_days=14, + activity_retention_days=90, + role_permissions=custom_perms, + ) + + assert ws.workspace_id == test_uuid + assert ws.default_invite_role == "admin" + assert ws.invite_expiry_days == 14 + assert ws.activity_retention_days == 90 + assert ws.role_permissions == custom_perms + + +# ===================================================================== +# 3. JSON SERIALIZATION / DESERIALIZATION & MUTATION BEHAVIOR +# ===================================================================== + + +class SQLiteWorkspaceSettings(SQLiteTestBase): + __tablename__ = "test_ws_settings" + + workspace_id: Mapped[str] = mapped_column(String, primary_key=True) + default_invite_role: Mapped[str] = mapped_column( + String(20), nullable=False, default="member" + ) + invite_expiry_days: Mapped[int] = mapped_column(Integer, nullable=False, default=7) + activity_retention_days: Mapped[int] = mapped_column( + Integer, nullable=False, default=30 + ) + role_permissions: Mapped[dict] = mapped_column( + sa.JSON, nullable=False, default=dict + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + nullable=False, + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + nullable=False, + ) + + def __init__(self, **kw): + kw.setdefault("default_invite_role", "member") + kw.setdefault("invite_expiry_days", 7) + kw.setdefault("activity_retention_days", 30) + kw.setdefault("role_permissions", {}) + super().__init__(**kw) + + +def test_sqlite_orm_json_serialization_and_mutation(): + """Verify ORM insert, retrieve, JSON serialization, and mutation tracking behavior.""" + engine = create_engine("sqlite:///:memory:", echo=False) + SQLiteTestBase.metadata.create_all(engine) + Session = sessionmaker(bind=engine) + session = Session() + + wid = str(uuid.uuid4()) + complex_permissions = { + "admin": { + "members.invite": True, + "members.remove": False, + "sub_config": {"nested_key": 123, "tags": ["alpha", "beta"]}, + }, + "member": { + "queries.execute": True, + "unicode_test": "🚀 BoloDB SQL", + }, + } + + ws = SQLiteWorkspaceSettings( + workspace_id=wid, + default_invite_role="member", + invite_expiry_days=7, + activity_retention_days=30, + role_permissions=complex_permissions, + ) + + session.add(ws) + session.commit() + + # Query back from DB + retrieved = session.execute( + select(SQLiteWorkspaceSettings).where( + SQLiteWorkspaceSettings.workspace_id == wid + ) + ).scalar_one() + + assert retrieved.workspace_id == wid + assert retrieved.default_invite_role == "member" + assert retrieved.invite_expiry_days == 7 + assert retrieved.activity_retention_days == 30 + assert isinstance(retrieved.role_permissions, dict) + assert retrieved.role_permissions == complex_permissions + assert retrieved.role_permissions["admin"]["sub_config"]["tags"] == [ + "alpha", + "beta", + ] + assert retrieved.role_permissions["member"]["unicode_test"] == "🚀 BoloDB SQL" + + # Empirical test: In-place mutation without re-assignment or flag_modified does NOT mark field dirty + retrieved.role_permissions["member"]["queries.execute"] = False + session.commit() + # Re-fetch from DB in new session to verify DB content + session.expire_all() + unmodified_in_db = session.execute( + select(SQLiteWorkspaceSettings).where( + SQLiteWorkspaceSettings.workspace_id == wid + ) + ).scalar_one() + # In-place dict change without re-assignment leaves DB unchanged (True) + assert unmodified_in_db.role_permissions["member"]["queries.execute"] is True + + # Empirical test: Explicit dictionary reassignment DOES mark field dirty and update DB + updated_perms = dict(unmodified_in_db.role_permissions) + updated_perms["member"] = dict(updated_perms["member"]) + updated_perms["member"]["queries.execute"] = False + unmodified_in_db.role_permissions = updated_perms + session.commit() + + session.expire_all() + modified_in_db = session.execute( + select(SQLiteWorkspaceSettings).where( + SQLiteWorkspaceSettings.workspace_id == wid + ) + ).scalar_one() + assert modified_in_db.role_permissions["member"]["queries.execute"] is False + + session.close() + + +def test_postgresql_ddl_compilation(): + """Verify PostgreSQL DDL generation for WorkspaceSettings table.""" + engine = create_engine("postgresql://user:pass@localhost/db") + table = WorkspaceSettings.__table__ + ddl = str(sa.schema.CreateTable(table).compile(engine)) + + assert "CREATE TABLE workspace_settings" in ddl + assert "workspace_id UUID NOT NULL" in ddl + assert "default_invite_role VARCHAR(20) NOT NULL" in ddl + assert "invite_expiry_days INTEGER NOT NULL" in ddl + assert "activity_retention_days INTEGER NOT NULL" in ddl + assert "role_permissions JSONB NOT NULL" in ddl + assert "PRIMARY KEY (workspace_id)" in ddl + assert ( + "FOREIGN KEY(workspace_id) REFERENCES workspaces (id) ON DELETE CASCADE" in ddl + ) + + +# ===================================================================== +# 4. ALEMBIC MIGRATION (e8f901a23b4c) VERIFICATION +# ===================================================================== + + +def test_alembic_migration_metadata(): + """Verify revision metadata in Alembic migration file.""" + assert migration.revision == "e8f901a23b4c" + assert migration.down_revision == "c1a4d7e29b58" + assert migration.branch_labels is None + assert migration.depends_on is None + + +def test_alembic_migration_upgrade_and_downgrade(monkeypatch): + """Empirically test upgrade() and downgrade() functions by intercepting op calls.""" + create_table_calls = [] + drop_table_calls = [] + + def mock_create_table(name, *columns, **kwargs): + create_table_calls.append((name, columns, kwargs)) + + def mock_drop_table(name, **kwargs): + drop_table_calls.append((name, kwargs)) + + monkeypatch.setattr(migration.op, "create_table", mock_create_table) + monkeypatch.setattr(migration.op, "drop_table", mock_drop_table) + + # Execute upgrade + migration.upgrade() + assert len(create_table_calls) == 1 + table_name, columns, kwargs = create_table_calls[0] + assert table_name == "workspace_settings" + + # Extract column names and constraints + col_dict = {} + constraints = [] + for item in columns: + if isinstance(item, sa.Column): + col_dict[item.name] = item + elif isinstance(item, (sa.ForeignKeyConstraint, sa.PrimaryKeyConstraint)): + constraints.append(item) + + assert set(col_dict.keys()) == { + "workspace_id", + "default_invite_role", + "invite_expiry_days", + "activity_retention_days", + "role_permissions", + "created_at", + "updated_at", + } + + # Verify column definitions and server defaults + wid_col = col_dict["workspace_id"] + assert wid_col.nullable is False + assert isinstance(wid_col.type, sa.UUID) + + role_col = col_dict["default_invite_role"] + assert role_col.nullable is False + assert isinstance(role_col.type, sa.String) + assert role_col.type.length == 20 + assert role_col.server_default.arg == "member" + + expiry_col = col_dict["invite_expiry_days"] + assert expiry_col.nullable is False + assert isinstance(expiry_col.type, sa.Integer) + assert expiry_col.server_default.arg == "7" + + retention_col = col_dict["activity_retention_days"] + assert retention_col.nullable is False + assert isinstance(retention_col.type, sa.Integer) + assert retention_col.server_default.arg == "30" + + perms_col = col_dict["role_permissions"] + assert perms_col.nullable is False + assert isinstance(perms_col.type, postgresql.JSONB) + assert perms_col.server_default.arg == "{}" + + created_col = col_dict["created_at"] + assert created_col.nullable is False + assert isinstance(created_col.type, sa.DateTime) + assert created_col.type.timezone is True + assert str(created_col.server_default.arg) == "now()" + + updated_col = col_dict["updated_at"] + assert updated_col.nullable is False + assert isinstance(updated_col.type, sa.DateTime) + assert updated_col.type.timezone is True + assert str(updated_col.server_default.arg) == "now()" + + # Verify constraints + fk_constraints = [c for c in constraints if isinstance(c, sa.ForeignKeyConstraint)] + pk_constraints = [c for c in constraints if isinstance(c, sa.PrimaryKeyConstraint)] + + assert len(fk_constraints) == 1 + fk = fk_constraints[0] + assert fk.column_keys == ["workspace_id"] + assert fk.ondelete == "CASCADE" + + assert len(pk_constraints) == 1 + pk = pk_constraints[0] + pk_cols = list(pk._pending_colargs) + assert pk_cols == ["workspace_id"] + + # Execute downgrade + migration.downgrade() + assert len(drop_table_calls) == 1 + assert drop_table_calls[0][0] == "workspace_settings"