diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de43685..1cdca59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -512,6 +512,11 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + - name: Apply database migrations + env: + DATABASE_URL: "postgresql://ci:ci@localhost:5432/ci_db" + run: alembic upgrade head + - name: Run test suite with coverage env: DATABASE_URL: "postgresql://ci:ci@localhost:5432/ci_db" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7424506..aad7d71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -213,22 +213,29 @@ Most list methods return an empty list on failure. Methods that fetch one resour ```bash # Python 3.10+ pip install -r requirements.txt -# Installs Flask, Azure SDK clients, requests, psycopg2, PyJWT, and PyYAML for CI workflow validation. +# Installs Flask, Alembic, Azure SDK clients, requests, psycopg2, PyJWT, and PyYAML for CI workflow validation. # Frontend # The frontend directory is currently a scaffold. The React dashboard MVP is on the roadmap. -# API -FLASK_APP=api/app.py flask run --debug - # Database (Docker) docker run --name openshield-db \ -e POSTGRES_USER=openshield \ -e POSTGRES_PASSWORD=openshield \ -e POSTGRES_DB=openshield \ -p 5432:5432 -d postgres + +export DATABASE_URL=postgresql://openshield:openshield@localhost:5432/openshield +alembic upgrade head + +# API +FLASK_APP=api/app.py flask run --debug ``` +See [Database Migrations](docs/database-migrations.md) before creating or applying +a schema change. Migration revisions are written explicitly because OpenShield +does not use ORM metadata. + --- ## Code Standards diff --git a/README.md b/README.md index de0b752..0999c5b 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,10 @@ export AZURE_CLIENT_ID=your-client-id export AZURE_CLIENT_SECRET=your-client-secret export AZURE_TENANT_ID=your-tenant-id export JWT_SECRET=your-strong-secret # used to protect write endpoints (scan trigger, AI) +export DATABASE_URL=postgresql://openshield:openshield@localhost:5432/openshield + +# Create or update the database schema +alembic upgrade head # Run a scan python -c " @@ -175,6 +179,9 @@ print(json.dumps(result, indent=2)) FLASK_APP=api/app.py flask run ``` +See [Database Migrations](docs/database-migrations.md) for schema changes and the +one-time onboarding step required for existing production databases. + **Frontend (React dashboard)** ```bash diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..1ac0d3f --- /dev/null +++ b/alembic.ini @@ -0,0 +1,42 @@ +[alembic] +script_location = %(here)s/alembic +prepend_sys_path = . +path_separator = os + +# DATABASE_URL is read from the environment by alembic/env.py. + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..f1ce2f8 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,54 @@ +"""Alembic migration environment for OpenShield.""" + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import create_engine, pool + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# OpenShield intentionally uses raw SQL rather than ORM metadata. Migration +# revisions therefore define every schema operation explicitly. +target_metadata = None + + +def get_database_url() -> str: + """Return the configured PostgreSQL URL or fail before migration work.""" + database_url = os.environ.get("DATABASE_URL") + if not database_url: + raise RuntimeError("DATABASE_URL environment variable is required") + return database_url + + +def run_migrations_offline() -> None: + """Generate SQL without opening a database connection.""" + context.configure( + url=get_database_url(), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations against the configured database.""" + connectable = create_engine(get_database_url(), poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..e3ac2e8 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# Revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Apply this migration.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Revert this migration.""" + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/b3f1a2c4d5e6_baseline_schema.py b/alembic/versions/b3f1a2c4d5e6_baseline_schema.py new file mode 100644 index 0000000..0f2002b --- /dev/null +++ b/alembic/versions/b3f1a2c4d5e6_baseline_schema.py @@ -0,0 +1,93 @@ +"""Create the baseline OpenShield schema. + +Revision ID: b3f1a2c4d5e6 +Revises: +Create Date: 2026-07-05 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# Revision identifiers, used by Alembic. +revision: str = "b3f1a2c4d5e6" +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create the current scans and findings tables.""" + op.create_table( + "scans", + sa.Column("scan_id", postgresql.UUID(), nullable=False), + sa.Column("subscription_id", sa.Text(), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("claimed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("total_findings", sa.Integer(), server_default=sa.text("0"), nullable=True), + sa.Column("score", sa.Integer(), server_default=sa.text("NULL"), nullable=True), + sa.Column( + "cve_enrichment_status", + sa.Text(), + server_default=sa.text("'PENDING'"), + nullable=True, + ), + sa.Column("status", sa.Text(), server_default=sa.text("'pending'"), nullable=True), + sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.PrimaryKeyConstraint("scan_id", name="scans_pkey"), + ) + + op.create_table( + "findings", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("scan_id", postgresql.UUID(), nullable=True), + sa.Column("rule_id", sa.Text(), nullable=False), + sa.Column("rule_name", sa.Text(), nullable=False), + sa.Column("severity", sa.Text(), nullable=False), + sa.Column("category", sa.Text(), nullable=True), + sa.Column("resource_id", sa.Text(), nullable=True), + sa.Column("resource_name", sa.Text(), nullable=True), + sa.Column("resource_type", sa.Text(), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("remediation", sa.Text(), nullable=True), + sa.Column("playbook", sa.Text(), nullable=True), + sa.Column("frameworks", postgresql.JSONB(), nullable=True), + sa.Column("metadata", postgresql.JSONB(), nullable=True), + sa.Column( + "cve_references", + postgresql.JSONB(), + server_default=sa.text("'[]'::jsonb"), + nullable=True, + ), + sa.Column("cvss_score", sa.Float(), server_default=sa.text("NULL"), nullable=True), + sa.Column( + "exploit_available", + sa.Boolean(), + server_default=sa.text("false"), + nullable=True, + ), + sa.Column("detected_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["scan_id"], + ["scans.scan_id"], + name="findings_scan_id_fkey", + ), + sa.PrimaryKeyConstraint("id", name="findings_pkey"), + ) + + op.create_index("idx_findings_scan_id", "findings", ["scan_id"], unique=False) + op.create_index("idx_findings_severity", "findings", ["severity"], unique=False) + op.create_index("idx_findings_rule_id", "findings", ["rule_id"], unique=False) + + +def downgrade() -> None: + """Drop the current findings and scans tables.""" + op.drop_index("idx_findings_rule_id", table_name="findings") + op.drop_index("idx_findings_severity", table_name="findings") + op.drop_index("idx_findings_scan_id", table_name="findings") + op.drop_table("findings") + op.drop_table("scans") diff --git a/api/app.py b/api/app.py index 942a6f3..64209c5 100644 --- a/api/app.py +++ b/api/app.py @@ -128,20 +128,6 @@ def create_app() -> Flask: "Do not use this setting with real Azure scan data in production." ) - # ------------------------------------------------------------------ # - # Database Management # - # ------------------------------------------------------------------ # - # Skip migrations when DATABASE_URL is not set (e.g. during unit tests). - # Deployment startup always sets DATABASE_URL so migrations still run in - # production and staging. Tests that need a real database should set - # DATABASE_URL explicitly in their environment. - if os.environ.get("DATABASE_URL"): - with app.app_context(): - db = DatabaseManager() - db.run_migrations() - else: - logger.info("DATABASE_URL not set — skipping database migrations. Set DATABASE_URL to connect to PostgreSQL.") - @app.teardown_appcontext def close_db(error=None): """Ensure the database connection is closed after the request.""" diff --git a/api/models/finding.py b/api/models/finding.py index f901f19..070071f 100644 --- a/api/models/finding.py +++ b/api/models/finding.py @@ -73,8 +73,8 @@ def to_dict(self) -> Dict[str, Any]: class DatabaseManager: """Manages PostgreSQL persistence for scans, findings, and scoring. - All public methods open a new connection on first use. Call connect() - explicitly if you want to pre-warm the connection. + Database operations open a new connection on first use. Call connect() + explicitly to pre-warm the connection. """ def __init__(self, dsn: Optional[str] = None) -> None: @@ -115,128 +115,12 @@ def ping(self) -> bool: cur.fetchone() return True - # ------------------------------------------------------------------ # - # Schema # - # ------------------------------------------------------------------ # - def init_db(self) -> None: - """Alias for run_migrations. Called by startup.sh on every boot. - - Calling this is always safe; run_migrations() handles both fresh - databases and existing ones via IF NOT EXISTS guards throughout. - """ - self.run_migrations() - - def create_tables(self) -> None: - """Create the findings, scans, and rules tables if they do not exist. - - Includes all columns, including CVE columns, so fresh databases - never need the ALTER TABLE path in run_migrations(). - """ - conn = self._get_conn() - with conn.cursor() as cur: - cur.execute(""" - CREATE TABLE IF NOT EXISTS scans ( - scan_id UUID PRIMARY KEY, - subscription_id TEXT NOT NULL, - started_at TIMESTAMPTZ NOT NULL, - claimed_at TIMESTAMPTZ, - completed_at TIMESTAMPTZ, - total_findings INTEGER DEFAULT 0, - score INTEGER DEFAULT NULL, - cve_enrichment_status TEXT DEFAULT 'PENDING', - status TEXT DEFAULT 'pending', - attempt_count INTEGER DEFAULT 0, - error_message TEXT - ); - """) - cur.execute(""" - CREATE TABLE IF NOT EXISTS findings ( - id SERIAL PRIMARY KEY, - scan_id UUID REFERENCES scans(scan_id), - rule_id TEXT NOT NULL, - rule_name TEXT NOT NULL, - severity TEXT NOT NULL, - category TEXT, - resource_id TEXT, - resource_name TEXT, - resource_type TEXT, - description TEXT, - remediation TEXT, - playbook TEXT, - frameworks JSONB, - metadata JSONB, - cve_references JSONB DEFAULT '[]', - cvss_score FLOAT DEFAULT NULL, - exploit_available BOOLEAN DEFAULT FALSE, - detected_at TIMESTAMPTZ NOT NULL - ); - """) - cur.execute(""" - CREATE INDEX IF NOT EXISTS idx_findings_scan_id - ON findings(scan_id); - CREATE INDEX IF NOT EXISTS idx_findings_severity - ON findings(severity); - CREATE INDEX IF NOT EXISTS idx_findings_rule_id - ON findings(rule_id); - """) - conn.commit() - logger.info("Database tables created / verified") - - def run_migrations(self) -> None: - """Ensure the schema is fully current. Safe to call on every startup. - - Calls create_tables() first so the call order never matters; this - method is safe whether the database is brand new or has existing data. - - On a fresh database: - create_tables() creates all tables including CVE columns. - The ALTER TABLE below is a no-op (IF NOT EXISTS). - - On a pre-CVE database (existed before this feature was merged): - create_tables() verifies tables exist and skips creation. - The ALTER TABLE adds the three CVE columns. - - Concurrent startup safety: - Both CREATE TABLE IF NOT EXISTS and ALTER TABLE ADD COLUMN IF NOT - EXISTS are atomic at the PostgreSQL catalog level. Two Render - instances racing at boot will not error; the second call silently - no-ops on whichever statement the first already completed. - """ - self.create_tables() - - conn = self._get_conn() - try: - with conn.cursor() as cur: - cur.execute(""" - ALTER TABLE findings - ADD COLUMN IF NOT EXISTS cve_references JSONB DEFAULT '[]', - ADD COLUMN IF NOT EXISTS cvss_score FLOAT DEFAULT NULL, - ADD COLUMN IF NOT EXISTS exploit_available BOOLEAN DEFAULT FALSE - """) - cur.execute(""" - ALTER TABLE scans - ADD COLUMN IF NOT EXISTS cve_enrichment_status TEXT DEFAULT 'COMPLETED', - ADD COLUMN IF NOT EXISTS status TEXT DEFAULT 'completed', - ADD COLUMN IF NOT EXISTS attempt_count INTEGER DEFAULT 0, - ADD COLUMN IF NOT EXISTS error_message TEXT, - ADD COLUMN IF NOT EXISTS claimed_at TIMESTAMPTZ - """) - # Fix: If status already existed but was backfilled as 'pending' (e.g. from - # a previous buggy deploy), force it to 'completed' for all historical - # scans that have already finished. - cur.execute( - "UPDATE scans SET status = 'completed' WHERE status = 'pending' AND completed_at IS NOT NULL" - ) - - # Backfill claimed_at for any currently running scans so they don't get - # immediately marked as stale by the new recovery logic. - cur.execute("UPDATE scans SET claimed_at = started_at WHERE status = 'running' AND claimed_at IS NULL") - conn.commit() - logger.info("CVE migrations applied successfully") - except Exception as e: - logger.error("Failed to run CVE migrations: %s", e) - conn.rollback() + """Deprecated compatibility hook; schema changes are managed by Alembic.""" + logger.warning( + "DatabaseManager.init_db() is deprecated and no longer manages the schema; " + "run 'alembic upgrade head' instead" + ) # ------------------------------------------------------------------ # # Write # diff --git a/docs/api-render-deploy.md b/docs/api-render-deploy.md index d5dc2bd..dc233d6 100644 --- a/docs/api-render-deploy.md +++ b/docs/api-render-deploy.md @@ -8,7 +8,7 @@ This test plan covers the verification of the OpenShield API deployment to Render (Starter instance or higher). The goal is to confirm: - The Render Web Service builds and deploys the Flask app successfully. -- The database is automatically initialized on startup via `init_db`. +- Alembic upgrades the database once in `startup.sh` before application processes start. - The pre-commit hook and GitHub Actions CI pipeline gate the code properly. - The CI pipeline is **community-friendly**, allowing forks to pass even without custom secrets. - Real Azure scan tests are gated behind `RUN_REAL_SCAN=true` so contributor CI never depends on live Azure credentials. @@ -22,7 +22,7 @@ To ensure the highest reliability of the deployment while remaining community-fr ### 2.1 Infrastructure and Pipeline Strategy * **Targeting Render over Azure F1:** Azure App Service's F1 tier imposes a strict 60 CPU-minute daily cap. Render provides unmetered CPU and always-on availability on paid instances, making it significantly more reliable for production environments. -* **Database Initialization:** The `api/models/finding.py` was updated with an `init_db` method. This method ensures that all required tables (`scans`, `findings`) are created automatically during the first deployment, preventing HTTP 500 errors. +* **Database Migrations:** `startup.sh` runs `alembic upgrade head` before starting the worker and Gunicorn. Existing production databases must complete the documented one-time `alembic stamp head` step before this release is deployed. * **Pre-commit Hook:** Fails fast. By running syntax checks and local API smoke tests *before* the commit is allowed, we prevent broken code from polluting the remote branch. * **Community-Friendly CI Gate:** The GitHub Action is designed to be zero-friction for contributors. * **Optional Smoke Tests:** If `JWT_SECRET` is not set (typical for forks), the smoke test step is gracefully skipped rather than failing the build. @@ -81,8 +81,8 @@ API_URL=https://openshield-api.onrender.com JWT_SECRET= \ | File | Purpose | |---|---| -| `startup.sh` | Container startup script, DB initialization, and Gunicorn execution | -| `api/models/finding.py` | Added `init_db` to ensure schema existence on startup | +| `startup.sh` | Runs Alembic once, then starts the worker and Gunicorn | +| `alembic/` | Versioned PostgreSQL schema migrations | | `.github/workflows/deploy.yml` | Flexible GitHub Actions workflow (optional smoke tests) | | `tests/smoke_test.py` | 23-case functional test suite with default secret support | | `.git/hooks/pre-commit` | Local Git hook enforcing syntax checks and local smoke tests | @@ -139,7 +139,7 @@ To enable the automated smoke tests in the CI/CD pipeline, you **must** add the **DP-02 — Render executes startup script successfully** * **Steps:** Push code to GitHub and monitor Render deployment logs. -* **Expected:** Logs show DB initialization (`Database initialized.`) and Gunicorn starting. +* **Expected:** Logs show Alembic upgrading to `head` before the worker and Gunicorn start. **DP-03 — GitHub Actions CI pipeline passes** * **Steps:** Push a commit and monitor the GitHub Actions tab. diff --git a/docs/azure-setup.md b/docs/azure-setup.md index 7aeef2a..63e345e 100644 --- a/docs/azure-setup.md +++ b/docs/azure-setup.md @@ -137,7 +137,14 @@ brew services start postgresql@15 createdb openshield ``` -The `DatabaseManager.create_tables()` call in the scan trigger will create the schema automatically on first run. +Create the database schema with Alembic before running a scan or starting the API: + +```bash +set -a; source .env; set +a +alembic upgrade head +``` + +See [Database Migrations](database-migrations.md) before onboarding an existing production database. --- @@ -197,7 +204,7 @@ Render is recommended for hosting the OpenShield API. Use the Starter instance o - Name: `openshield-api` - Branch: `main` - Build Command: `pip install -r requirements.txt` - - Start Command: `gunicorn api.app:create_app()` + - Start Command: `./startup.sh` - Instance Type: `Free` 5. Add environment variables under Environment: diff --git a/docs/cve_correlation_feature.md b/docs/cve_correlation_feature.md index 40052dd..a082081 100644 --- a/docs/cve_correlation_feature.md +++ b/docs/cve_correlation_feature.md @@ -21,7 +21,7 @@ The CVE Correlation feature integrates the MITRE National Vulnerability Database | scanner/engine.py | Decoupled Scan. Removed synchronous enrichment from the scan lifecycle. | Performance: Azure scans now return immediately without waiting for NVD rate limits (7s per resource type). | | api/routes/scans.py | New Endpoint. Added `POST /api/scans//enrich`. | Flexibility: CVE enrichment can now be triggered on-demand or by a background job after the scan completes. | | api/models/finding.py | Updated Scan model and added enrichment status tracking. | Persistence: Adds `cve_enrichment_status` to track `PENDING`, `COMPLETED`, or `FAILED` states. | -| api/app.py | Added db.run_migrations call at startup. | Auto-Deployment: Ensures the database schema is updated automatically on any environment where the app is launched. | +| alembic/versions/ | Defines CVE columns in the versioned database schema. | Deployment: Alembic owns schema changes independently of Flask application startup. | | api/routes/score.py | Added GET /api/score/cve-summary endpoint. | Dashboard UI: Provides the frontend with high-level data like Total Known Exploits and enrichment status. | | api/routes/findings.py | Returns findings from the database without JIT enrichment. | Performance: Ensures predictable and fast API responses for findings. | diff --git a/docs/database-migrations.md b/docs/database-migrations.md new file mode 100644 index 0000000..66791cb --- /dev/null +++ b/docs/database-migrations.md @@ -0,0 +1,107 @@ +# Database Migrations + +OpenShield manages its PostgreSQL schema with Alembic. Application queries remain +raw SQL; migration files define schema operations explicitly and do not introduce +ORM models or SQLAlchemy metadata. + +All commands below run from the repository root and require `DATABASE_URL` in the +environment: + +```bash +export DATABASE_URL=postgresql://openshield:openshield@localhost:5432/openshield +``` + +## Fresh databases + +Create the complete current schema by upgrading to the latest revision: + +```bash +alembic upgrade head +alembic current +``` + +Production startup runs `alembic upgrade head` once, before starting the worker +and Gunicorn. The Flask application does not create or migrate tables. + +## Deployment checklist + +For a brand-new database: + +1. Set `DATABASE_URL`. +2. Deploy normally; `startup.sh` runs `alembic upgrade head`. +3. Confirm `alembic current` reports the head revision. + +For an existing production database that already has OpenShield tables: + +1. Back up the database. +2. Verify the live schema matches the baseline revision in `alembic/versions/`. +3. Run `alembic stamp head` once before the first Alembic-enabled deployment. +4. Deploy normally; `startup.sh` must continue to run only `alembic upgrade head`. + +Do not automate `alembic stamp head` in application code, startup scripts, +containers, or deployment configuration. + +## Existing production databases + +The baseline migration represents the schema that OpenShield already created in +production before Alembic was adopted. It must not be run against an unversioned +database that already contains the `scans` and `findings` tables. + +Before deploying this Alembic-enabled release to an existing production database: + +1. Back up the database. +2. Verify that `scans`, `findings`, their columns, constraints, and indexes match + the baseline revision in `alembic/versions/`. +3. Record the existing schema at the baseline revision without executing DDL: + + ```bash + alembic stamp head + alembic current + ``` + +This is a one-time operational step for existing databases. Never add automatic +stamping to application code, startup scripts, containers, or deployment +configuration. After stamping, normal deployments use only: + +```bash +alembic upgrade head +``` + +## Creating a migration + +Create an empty revision with a short, descriptive, imperative message: + +```bash +alembic revision -m "add finding source" +``` + +Because OpenShield has no ORM metadata, do not use `--autogenerate`. Implement +both `upgrade()` and `downgrade()` explicitly with Alembic operations, then review +the generated file before applying it. + +Migration files use Alembic's standard `_.py` naming. Keep the +message lowercase and specific to one schema change, for example +`add_scan_region` or `index_findings_detected_at`. Each revision must point to the +current head through `down_revision`; do not create parallel heads. + +## Contributor workflow + +1. Start PostgreSQL and export `DATABASE_URL`. +2. Update to the current schema with `alembic upgrade head`. +3. Create and implement the new revision. +4. Test the upgrade from the previous revision. +5. Test `alembic downgrade -1` and then `alembic upgrade head` on disposable data. +6. Test the full migration chain against a fresh empty database. +7. Run the application test suite. + +Useful inspection commands: + +```bash +alembic current +alembic history --verbose +alembic heads +alembic upgrade head --sql +``` + +Treat downgrades as destructive operations and run them only against a database +whose data can be restored. diff --git a/requirements.txt b/requirements.txt index 921e8f3..3571558 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ flask==3.1.3 flask-cors==6.0.5 +alembic==1.18.5 azure-identity==1.25.3 azure-mgmt-storage==21.0.0 azure-mgmt-network==25.0.0 diff --git a/startup.sh b/startup.sh index 6b3e0df..a53e86d 100755 --- a/startup.sh +++ b/startup.sh @@ -7,22 +7,8 @@ set -euo pipefail export OPENSHIELD_ENV="${OPENSHIELD_ENV:-production}" echo "=== OpenShield startup ===" -echo "Running database initialisation..." - -python -c " -import os, sys -try: - from api.models.finding import DatabaseManager - db = DatabaseManager(os.environ['DATABASE_URL']) - if hasattr(db, 'init_db'): - db.init_db() - print('Database initialised.') - else: - print('WARNING: DatabaseManager has no init_db() method — skipping.') -except Exception as e: - print(f'ERROR during DB init: {e}', file=sys.stderr) - sys.exit(1) -" +echo "Applying database migrations..." +alembic upgrade head echo "Startup complete. Starting background worker and Gunicorn..." # Start the background worker process with a simple restart loop @@ -33,4 +19,4 @@ echo "Startup complete. Starting background worker and Gunicorn..." done ) & -exec gunicorn --bind=0.0.0.0:$PORT --timeout 120 --workers 2 api.app:application \ No newline at end of file +exec gunicorn --bind=0.0.0.0:$PORT --timeout 120 --workers 2 api.app:application diff --git a/tests/conftest.py b/tests/conftest.py index 39f7260..02b505b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,6 @@ import secrets import time -from unittest.mock import MagicMock import pytest @@ -14,12 +13,7 @@ @pytest.fixture -def app(monkeypatch): - # Mock DatabaseManager before importing create_app - - mock_db = MagicMock() - monkeypatch.setattr("api.app.DatabaseManager", MagicMock(return_value=mock_db)) - +def app(): from api.app import create_app application = create_app() diff --git a/tests/test_auth.py b/tests/test_auth.py index 5aa07d5..1fbbfdb 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,11 +1,10 @@ """Tests for JWT authentication middleware — production and demo modes.""" -import os import secrets import time + import jwt import pytest -from unittest.mock import MagicMock _SECRET = secrets.token_urlsafe(32) @@ -23,8 +22,7 @@ def _make_token() -> str: @pytest.fixture def prod_client(monkeypatch): """Flask test client with default (JWT-required) auth mode.""" - monkeypatch.setattr("api.app.DatabaseManager", MagicMock()) - os.environ.pop("OPENSHIELD_PUBLIC_DEMO", None) + monkeypatch.delenv("OPENSHIELD_PUBLIC_DEMO", raising=False) from api.app import create_app app = create_app() @@ -36,17 +34,13 @@ def prod_client(monkeypatch): @pytest.fixture def demo_client(monkeypatch): """Flask test client with OPENSHIELD_PUBLIC_DEMO=true.""" - monkeypatch.setattr("api.app.DatabaseManager", MagicMock()) - os.environ["OPENSHIELD_PUBLIC_DEMO"] = "true" - try: - from api.app import create_app - - app = create_app() - app.config["TESTING"] = True - app.config["JWT_SECRET"] = _SECRET - yield app.test_client() - finally: - os.environ.pop("OPENSHIELD_PUBLIC_DEMO", None) + monkeypatch.setenv("OPENSHIELD_PUBLIC_DEMO", "true") + from api.app import create_app + + app = create_app() + app.config["TESTING"] = True + app.config["JWT_SECRET"] = _SECRET + return app.test_client() # ── /health is always public ───────────────────────────────────────────────── diff --git a/tests/test_jwt_config.py b/tests/test_jwt_config.py index b630c1b..cf1e6a4 100644 --- a/tests/test_jwt_config.py +++ b/tests/test_jwt_config.py @@ -1,7 +1,6 @@ """Tests for environment-aware JWT secret resolution in create_app().""" import secrets -from unittest.mock import MagicMock import pytest @@ -123,9 +122,6 @@ def test_render_strong_secret_accepted(monkeypatch): def test_create_app_production_uses_strong_secret(monkeypatch): """create_app() in production mode wires the config correctly.""" - # Mock DatabaseManager to avoid DB connection in CI - monkeypatch.setattr("api.app.DatabaseManager", MagicMock()) - secret = secrets.token_urlsafe(32) monkeypatch.setenv("OPENSHIELD_ENV", "production") monkeypatch.setenv("JWT_SECRET", secret) @@ -135,9 +131,6 @@ def test_create_app_production_uses_strong_secret(monkeypatch): def test_create_app_development_starts_without_secret(monkeypatch): """create_app() in development mode starts without a JWT_SECRET set.""" - # Mock DatabaseManager to avoid DB connection in CI - monkeypatch.setattr("api.app.DatabaseManager", MagicMock()) - monkeypatch.setenv("OPENSHIELD_ENV", "development") monkeypatch.delenv("JWT_SECRET", raising=False) app = create_app()