Makefile

.PHONY: check fmt test lint type db-up db-down db-logs obs-up obs-down migrate seed eval eval-gate eval-baseline flaky results dashboard ablations loadtest openapi sbom audit-deps

# Full gate: lint + format check + strict types + tests + the eval regression gate.
# Must be green before any commit. eval-gate needs a seeded database; without one
# it prints a SKIPPED banner and does not fail the build (see below).
check: lint type test eval-gate

lint:
	uv run ruff check src tests loadtest
	uv run ruff format --check src tests loadtest

type:
	uv run mypy src loadtest

test:
	uv run pytest

fmt:
	uv run ruff format src tests loadtest
	uv run ruff check --fix src tests loadtest

# Apply the migrations to the database GROUNDWORK_DATABASE_URL points at. A fresh
# clone needs this once (`make db-up && make migrate && make seed`): compose only
# starts an empty Postgres, and every db-marked test migrates its own throwaway
# database, so nothing else ever creates the schema in the dev/CI database.
migrate:
	uv run alembic upgrade head

# Seed the db with the committed fixture corpus (stub provider, no redis).
# The fixtures stand in for the real corpus until task 031 lands.
seed: migrate
	uv run python -m groundwork ingest . --root tests/fixtures/corpus

# Score the fixture golden set (hybrid, k=10) and write results/*.json.
eval:
	uv run python -m groundwork eval

# Regression gate (ADR 0021): re-run the eval and compare it to evals/baseline.json.
# Exit 0 = PASSED, 1 = FAILED (recall@k or MRR dropped by more than 2 absolute
# points), 2 = ERROR, 3 = SKIPPED. Exit 3 means the gate DID NOT RUN — there is
# no database or the index is empty — and is mapped to 0 here so `make check`
# stays green on a machine without Docker, exactly like the db-marked tests. The
# banner it prints says so in as many words: a skip is never a pass. This target
# never writes the baseline.
eval-gate:
	@uv run python -m groundwork eval-gate; status=$$?; \
	if [ $$status -eq 3 ]; then exit 0; fi; exit $$status

# Explicit, opt-in baseline promotion — the ONLY writer of evals/baseline.json.
# It is deliberately not a prerequisite of anything: a gate that can rewrite its
# own baseline defends nothing. Paste the printed delta into the commit message.
eval-baseline:
	uv run python -m groundwork eval-baseline

# Flakiness detector (task 102, ADR 0067): reruns the fixture eval 3 times and
# asserts every metric is byte-identical, guarding the determinism invariant
# (CLAUDE.md #17) every other eval tool (the gate, `compare`, history trend)
# implicitly trusts. Needs a seeded database, like `eval`/`eval-gate`; NOT a
# prerequisite of `make check` for the same reason `eval-gate` is mapped to 0
# on skip there instead of being duplicated here — this is a diagnostic you
# run deliberately, not a gate that must survive a machine without Docker.
flaky:
	uv run python -m groundwork flaky

# Export the live OpenAPI schema to docs/openapi.json (task 086). Building the
# app and reading .openapi() touches neither Postgres nor Redis — the db
# engine/arq pool/cache client are all created lazily on the first *request*,
# never at app-build time — so, like every other offline test/tool here, this
# needs no live stack. Deterministic (sorted keys, fixed indent): the same
# code always produces the same bytes (tests/test_openapi_contract.py proves
# it), so a schema diff in review is always a real contract change, not noise.
# The contract test is the actual CI guard; this only refreshes the committed
# file, which you should re-run and re-commit whenever a route's schema
# changes.
openapi:
	uv run python scripts/export_openapi.py

# SBOM + supply-chain guard (task 111, CLAUDE.md #5): both read ONLY uv.lock
# and CLAUDE.md from disk (tomllib + a regex over the committed rule-5 bullet)
# — no PyPI, no pip-audit, no subprocess, no socket, fully offline like every
# other tool here (tests/test_sbom.py proves the no-network property).
#
# `sbom` writes a deterministic, CycloneDX-style manifest of every package
# uv.lock resolved (direct and transitive) to docs/sbom.json. Not part of
# `check`: like `openapi`, it is a generator you re-run and re-commit when
# the lockfile changes, not a gate.
sbom:
	uv run python scripts/sbom.py sbom

# `audit-deps` is the actual guard: it cross-checks this project's DIRECT
# dependencies (uv.lock's root package, i.e. exactly what pyproject.toml
# declares) against the CLAUDE.md rule-5 allowlist, parsed from the live file
# so it can never drift from a second hardcoded copy. Exit 0 when every direct
# dependency is allowlisted, 1 otherwise — `uv add`ing anything off-list makes
# this fail the moment it lands in uv.lock. Deliberately checks direct
# dependencies only, not the whole transitive closure pulled in by resolving
# them (rule 5 governs what this project chooses to depend on, not what its
# dependencies themselves require) — see groundwork/sbom.py's module docstring.
audit-deps:
	uv run python scripts/sbom.py audit

# Regenerate RESULTS.md from results/*.json (deterministic; --check verifies).
results:
	uv run python scripts/update_results.py

# Render results/dashboard.html: one self-contained HTML file (inline CSS/SVG,
# no JS, no external asset) with the metric trend + bootstrap CIs, the
# per-category breakdown (task 070), and the quality/cost table (task 074).
# Deterministic like `make results`; not committed (see .gitignore) — it is
# derived purely from results/*.json, which is itself gitignored.
dashboard:
	uv run python -m groundwork dashboard

# The ablation matrix (ADR 0022): 12 configs = {dense,sparse,hybrid} x {rerank
# on/off} x chunk {256,512}, all scored on the same golden set, then RESULTS.md.
#
# Explicitly opt-in and deliberately NOT a prerequisite of `make check`: it
# REBUILDS THE INDEX at each chunk size, and a check target must never mutate the
# dev database's corpus.
#
# DESTRUCTIVE: each rebuild TRUNCATEs documents/chunks/chunk_embeddings and
# re-ingests --corpus (the fixture corpus by default). Anything else you ingested
# into the dev database is deleted, not merged. Point GROUNDWORK_DATABASE_URL at a
# throwaway database if that matters to you.
#
# It restores the index to the default chunk size when it finishes — including
# when a cell raises. A HARD KILL (SIGKILL, OOM, a second Ctrl-C) during the
# minutes it spends re-indexing runs no restore and leaves the index at 256 or
# 512. That is not silently survivable: every document records the chunk size it
# was built at, so `make eval` and the gate READ IT BACK and refuse to score an
# index they did not expect, naming `make seed` as the repair (ADR 0022). They
# will not publish 256-index numbers under an 800 label.
#
# Run it from a COMMITTED tree. Results carry the git sha they were produced at,
# and RESULTS.md refuses a `-dirty` one — a number nobody can reproduce does not
# belong in a published table. On a dirty tree the runs are still written (and the
# command says so), but the table regeneration below will refuse to touch them.
#
# The `eval` block re-scores the same six retrieval configs at the DEFAULT chunk
# size — the index `make seed` builds and the one the service actually ships. They
# are reference rows, not ablation cells (the matrix is the 12 above), and the
# table needs them: bolding a winner chosen only from 256/512 would crown a config
# that is beaten by the shipping default, which is an artifact of the experiment
# rather than a finding. `ablate` restores the index to the default chunk size as
# it exits, so these run against the right index with no reseed of their own.
ablations:
	uv run python -m groundwork ablate
	uv run python -m groundwork eval --mode dense
	uv run python -m groundwork eval --mode dense --rerank
	uv run python -m groundwork eval --mode sparse
	uv run python -m groundwork eval --mode sparse --rerank
	uv run python -m groundwork eval --mode hybrid
	uv run python -m groundwork eval --mode hybrid --rerank
	uv run python scripts/update_results.py

# Local dev services (compose.yaml): pgvector Postgres + Redis, loopback-only.
db-up:
	docker compose up -d --wait

db-down:
	docker compose down

db-logs:
	docker compose logs -f

# The observability profile (ADR 0024): Prometheus + Grafana, loopback-only,
# with the committed dashboard provisioned. Opt-in — a plain `make db-up` does
# not start them. Prometheus scrapes the API's /metrics on the HOST (via
# host.docker.internal), so run the API for the panels to have anything to show:
#   uv run uvicorn groundwork.api.app:create_app --factory
# Grafana: http://127.0.0.1:3000 (stock admin/admin, forced change at first
# login — no credentials are committed). Prometheus: http://127.0.0.1:9090.
obs-up:
	docker compose --profile observability up -d --wait

obs-down:
	docker compose --profile observability down

# The load test (ADR 0025): 20 users for 2 minutes, headless, 9:1 streaming
# /query/stream to /healthz noise. Writes loadtest/report_<sha>.md with the
# measured TTFT/total percentiles, throughput, the server's own /metrics
# figures, and the host it all ran on.
#
# Explicitly opt-in and deliberately NOT part of `make check`: two minutes of
# saturating load in the gate would make every session slow and would go flaky
# the moment the box is busy. The suite tests the measurement code instead
# (tests/test_loadtest.py), never the two-minute run.
#
# It needs the stack up and the API running — the load test does not start them.
# The /healthz preflight aborts the run before a single user is spawned rather
# than hammer a closed port for two minutes and publish a table of zeros:
#   make db-up && make seed
#   uv run uvicorn groundwork.api.app:create_app --factory
#
# --stop-timeout is not optional here, it is part of the measurement. With locust's
# default of 0, test stop KILLS every user mid-task (runners.py stop_users takes the
# force=True branch on a falsy stop_timeout) — and a stream killed after the server
# has recorded its first token but before the client has recorded its sample is a
# stream the two sides count differently, i.e. a CROSS-CHECK FAILED stamped on a run
# with zero real failures. 10s lets every in-flight task finish (the longest observed
# stream is ~0.1s) and costs nothing on a clean run.
#
# Exit codes: 0 = clean run, 1 = the run had failed requests, a task exception,
# or a client/server cross-check that did not add up (the acceptance criterion
# is zero failures at 20 users), 2 = nothing measurable (API down, no tokens).
# The code is only ever escalated, never lowered, and a 0 is never written back
# into locust's own exit-code chain: a run locust already failed cannot be
# published green.
#
# Run it from a COMMITTED tree, like the ablations: the report names the sha it
# measured, and a report of a `-dirty` sha is stamped NOT REPRODUCIBLE and
# gitignored — nobody can check out the code that produced it.
LOADTEST_HOST ?= http://127.0.0.1:8000
loadtest:
	uv run locust -f loadtest/locustfile.py --headless \
		--users 20 --spawn-rate 5 --run-time 2m --stop-timeout 10s \
		--host $(LOADTEST_HOST) --only-summary