Skip to content

Measure the harvest step LF13 named, and strike it (#553) #1860

Measure the harvest step LF13 named, and strike it (#553)

Measure the harvest step LF13 named, and strike it (#553) #1860

Workflow file for this run

name: CI
# Lint + test gate for backend (Python/uv) and frontend (React/Vite), plus a
# build of the shipped API image.
# Modelled after the anyplot pipeline: Ruff for lint+format, pytest for the
# compute/API layer, tsc+vite build for the SPA. Keep this in sync with the
# local dev commands documented in CLAUDE.md.
#
# Two conventions this file keeps, both adopted from anyplot:
#
# - Every `uses:` is pinned to a commit SHA with the version as a comment. A
# movable tag is a write handle into these runners, and both the backend and
# the frontend job hand CODECOV_TOKEN to the environment a preceding action
# already runs in. Dependabot bumps SHA pins exactly as it bumps tags (the
# `github-actions` group in .github/dependabot.yml), so this costs nothing to
# maintain — a new workflow starts pinned, never with a tag.
# - Every job carries `timeout-minutes`. GitHub's default is 360, so a hung
# pytest (a Cloud SQL connection attempt, a Postgres service that never turns
# ready) would burn six hours per job and report nothing useful. Measured over
# the last 20 runs the green ones sit at min 157 s / median 190 s / max 491 s;
# the caps below are generous over that maximum so a cold uv cache still fits.
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
# Cancel superseded runs on the same ref to save CI minutes.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
backend:
name: Backend (ruff + pytest)
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
# tests/test_reserved_history.py walks every blob ever committed, to
# catch a reserved-dataset payload entering the PUBLIC history. On a
# shallow checkout there is no history to walk and the guard would
# skip. The repo is small (~12k objects), so the full fetch is cheap.
fetch-depth: 0
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- name: Set up Python
run: uv python install
- name: Install dependencies
run: uv sync --extra dev --extra test --frozen
- name: Ruff lint
run: uv run ruff check --output-format=github .
- name: Ruff format
run: uv run ruff format --check .
- name: Pytest
run: uv run pytest --cov=core --cov=api --cov-report=xml
- name: Upload coverage to Codecov
# Skip on fork PRs (CODECOV_TOKEN is not exposed to them). fail_ci_if_error
# stays false so a missing token / codecov hiccup never reds CI.
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
flags: backend
fail_ci_if_error: false
migrations:
name: Migrations (alembic upgrade head)
runs-on: ubuntu-latest
timeout-minutes: 15
# Full migration chain incl. seeds against a real Postgres 16 — catches
# broken revisions, JSONB/DDL issues and seed-file drift that the SQLite
# test harness cannot see. The service DB is throwaway; no shared Cloud SQL
# instance is ever touched here.
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: kurrentschrift
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- name: Set up Python
run: uv python install
# Runtime dependencies suffice: alembic + asyncpg are project deps, and
# the migration run needs neither the dev nor the test extras.
- name: Install dependencies
run: uv sync --frozen
# alembic/env.py reads DATABASE_URL (async mode needs the asyncpg driver
# in the URL) and falls back to INSTANCE_CONNECTION_NAME otherwise.
- name: Alembic upgrade head
env:
DATABASE_URL: postgresql+asyncpg://postgres:postgres@localhost:5432/kurrentschrift
run: uv run alembic upgrade head
# Model ↔ migration drift: a model column added without a revision passes
# `upgrade head` silently; autogenerate-diff against the migrated DB
# catches it.
- name: Alembic check (model/migration drift)
env:
DATABASE_URL: postgresql+asyncpg://postgres:postgres@localhost:5432/kurrentschrift
run: uv run alembic check
# Reversibility of the newest revision: downgrade one step and re-upgrade,
# so a broken/missing downgrade never lands unnoticed.
- name: Alembic downgrade/upgrade roundtrip
env:
DATABASE_URL: postgresql+asyncpg://postgres:postgres@localhost:5432/kurrentschrift
run: uv run alembic downgrade -1 && uv run alembic upgrade head
frontend:
name: Frontend (build)
runs-on: ubuntu-latest
timeout-minutes: 20
defaults:
run:
working-directory: app
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
# 22 = active LTS (Node 20 reached end-of-life 2026-04-30).
node-version: 22
cache: npm
cache-dependency-path: app/package-lock.json
- name: Install dependencies
run: npm ci
- name: ESLint
run: npm run lint
# Fast fixture-driven Vitest run — pins the shaping.ts ↔ core/shaping.py
# twin against tests/fixtures/shaping_cases.json (the Python side asserts
# the same fixture in tests/test_tri_script.py). --coverage feeds the
# Codecov frontend flag so SPA patch coverage stops being invisible.
- name: Test
run: npm run test -- --coverage
- name: Upload frontend coverage to Codecov
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./app/coverage/coverage-final.json
flags: frontend
fail_ci_if_error: false
# `npm run build` is `tsc && vite build`, so this gates the type-check too.
- name: Build
run: npm run build
changelog:
name: Changelog (fragment)
runs-on: ubuntu-latest
timeout-minutes: 10
# Every PR carries a changelog fragment (changelog.d/<slug>.md) instead of
# a bullet in the shared CHANGELOG.md — the seam where sibling PRs used to
# conflict (2026-08-30). Data-only PRs pass on their own; a PR with truly
# nothing to tell gets the `skip-changelog` label. Push runs on main have
# no base to diff against and are skipped.
#
# Dependabot is skipped by author: its PRs are the routine bumps the release
# notes leave out anyway, and a bot can neither write a fragment nor reach
# for the label — so the gate would just sit red on every Monday's batch
# (#468, 2026-08-31). A bump that DOES deserve a line (the peer-dep override
# of #235) reaches the changelog through the human PR that carries the fix.
if: >-
github.event_name == 'pull_request'
&& !contains(github.event.pull_request.labels.*.name, 'skip-changelog')
&& github.event.pull_request.user.login != 'dependabot[bot]'
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
# The gate diffs against the base branch, which a shallow checkout lacks.
fetch-depth: 0
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- name: Set up Python
run: uv python install
# The tool is standard library only; no project dependencies are synced.
- name: Fragment present and well-formed
run: uv run --no-project python -m tools.changelog check --base "origin/${{ github.base_ref }}"
docs-register:
name: Docs-Register (§14 index)
runs-on: ubuntu-latest
timeout-minutes: 10
# The campaign journal (messjournal.md §14) carries three "same PR"
# duties in prose: index the new entry, ledger the moved headline, bring the
# route's process page along. The audit of 2026-09-02 found all three lagging
# — process pages two adoptions behind, the headline history only in running
# text, one headline pair whose fixture root nobody could reconstruct. This
# job is the same shape as the changelog gate: it reads the committed files,
# so an entry added together with its register row passes, and an entry
# without one fails wherever it came from.
#
# It runs on pushes too: the rules hold for the tree, not just for a diff.
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
# `--base` names the entries a PR adds, which a shallow checkout lacks.
fetch-depth: 0
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- name: Set up Python
run: uv python install
# Standard library only, like the changelog gate; no extras are synced.
# `--base` only enriches the message (it names the entries the branch adds),
# so a push run without a base branch checks exactly the same rules.
- name: Every §14 entry indexed, every headline ledgered
env:
BASE_REF: ${{ github.base_ref }}
run: |
if [ -n "$BASE_REF" ]; then
uv run --no-project python -m tools.docs_register check --base "origin/$BASE_REF"
else
uv run --no-project python -m tools.docs_register check
fi
docs-budget:
name: Docs-Budget (reading cost)
runs-on: ubuntu-latest
timeout-minutes: 10
# What a session must load before it starts working is a number, and on
# 2026-09-04 that number was 110 997 tokens — grown one paragraph at a time,
# noticed only when somebody measured it. #521 and #524 brought it to 53 865;
# this job is what keeps it there. It also checks the three things that make
# the cheap path actually cheap: a large `lebend` doc carries a Stand block
# with a date no older than a month, the map in docs/index.md has exactly one
# row per file, and every relative link and `#anchor` in the repo resolves.
#
# Standard library only (it ships its own token proxy rather than pulling
# tiktoken, which would download its BPE table on every run), so nothing is
# synced. Runs on pushes too: the rules hold for the tree, not just a diff.
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- name: Set up Python
run: uv python install
- name: Reading paths inside their budget, Stand blocks fresh, every jump landing
run: uv run --no-project python -m tools.docs_budget check
# Printed unconditionally: the table is what someone reads when the gate
# goes red, and it is three lines of output.
- name: What each reading path costs today
if: always()
run: uv run --no-project python -m tools.docs_budget report
image:
name: Image (build + container smoke)
runs-on: ubuntu-latest
timeout-minutes: 25
# Until this job existed, the first build attempt of a changed Dockerfile
# happened in Cloud Build — after the merge. #473 is the proof: pyproject.toml
# fell out of the runtime stage, api.main::_project_version() would have
# reported 0.0.0 in production, and the Cloud Build smoke could not have seen
# it because /health carries no version field. A human reviewer caught it.
#
# The job runs in parallel with the other four, so the pipeline's wall clock
# is unchanged; it only adds runner minutes.
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
# push: false — nothing here reaches a registry; Cloud Build still owns the
# published image. load: true puts the result into the local daemon so the
# smoke below can actually run it. The GHA cache keeps the repeat builds of
# an unchanged Dockerfile at seconds instead of minutes.
- name: Build the API image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
file: api/Dockerfile
push: false
load: true
tags: kurrentschrift-api:ci
cache-from: type=gha
cache-to: type=gha,mode=max
# No database, no secrets: the container is started bare and only asked the
# questions that need no Cloud SQL. That is deliberate — this is a check of
# the IMAGE, not of the deployment.
- name: Container smoke (no database)
run: |
set -euo pipefail
docker run -d --name api -p 8000:8000 kurrentschrift-api:ci
ready=0
for _ in $(seq 30); do
if curl -fsS localhost:8000/health >/dev/null 2>&1; then ready=1; break; fi
sleep 2
done
if [ "$ready" -ne 1 ]; then
echo "::error::the container never answered /health within 60 s"
exit 1
fi
curl -fsS localhost:8000/health | grep -q '"healthy"'
echo "health OK"
# THE assert that would have caught #473. api.main reads pyproject.toml
# from next to api/ at startup (the project is a virtual uv workspace,
# so importlib.metadata knows nothing about it) and falls back to 0.0.0
# when the file is missing — silently, in a field nothing else probes.
want=$(python3 -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
got=$(curl -fsS localhost:8000/openapi.json | python3 -c "import json,sys;print(json.load(sys.stdin)['info']['version'])")
if [ "$got" != "$want" ]; then
echo "::error::the image reports version $got, pyproject.toml says $want"
exit 1
fi
echo "version OK: $got"
# Pins the runtime stage's COPY list against a future rebuild. Each of
# these is a file the image is expected to SERVE or RUN, not a build
# input: the crawler pages go out over /seo-proxy, the quiz seed is read
# by migrations 0010/0011 at apply time, and alembic/ is what the
# pre-deploy migrate job executes out of this very image.
docker exec api test -f /app/app/prerender/index.html
docker exec api test -f /app/tools/quizgen/quiz_words.json
docker exec api test -f /app/alembic.ini
docker exec api test -d /app/alembic
docker exec api test -d /app/data/sources
echo "COPY list OK"
# The measurement the two local rounds of 2026-09-04/05 named as their one
# remaining gap: `import api.main` timed in the REAL image — same layers,
# same interpreter patch level, same precompiled bytecode — instead of in a
# venv shaped like it (docs/notes/serve-image-importtime-2026-09-05.md
# §„Grenzen dieser Runde"). The image is already built and loaded here, so
# it costs one `docker run` and no infrastructure.
#
# Deliberately OUTPUT ONLY: no threshold, no gate, `continue-on-error` on
# top. Import time on a shared runner swings by more than any threshold
# worth setting, so a gate here would be noise holding a veto. The number
# is for a later round to quote, not for this PR to pass.
#
# The script travels over stdin rather than living in the image, because
# the image deliberately does not ship `tools/` and a measurement has no
# business inside the artefact it measures.
#
# The smoke container is STOPPED first: it is still serving from the step
# above, and its uvicorn would sit in the same CPU budget as the thing
# being timed. Stopping is enough — `docker logs api` still works on a
# stopped container, so the failure step below keeps its output.
- name: Import weight of api.main, measured in the image
continue-on-error: true
timeout-minutes: 3
run: |
set -euo pipefail
docker stop api >/dev/null
docker run --rm -i \
-e OPENBLAS_NUM_THREADS=1 -e OMP_NUM_THREADS=1 \
kurrentschrift-api:ci /app/.venv/bin/python - \
< .github/scripts/importtime_report.py
- name: Container logs on failure
if: failure()
run: docker logs api || true
# Threshold `warning` with four named exceptions, rather than a
# non-blocking run: that way a NEW warning blocks, which is the point of
# having the linter at all. All four exceptions are deliberate choices
# documented at their line in api/Dockerfile. app/Dockerfile declines
# exactly one, and inline: DL3064 on the origin gate's `ORIGIN_SECRET=""`
# default. The rule reads the variable NAME, the value is the empty
# string, and the reason it has to be declared at all stands beside the
# `# hadolint ignore=` there — on an ENV instruction of its own, so it
# excuses that line and nothing else.
- name: Hadolint (api/Dockerfile)
uses: hadolint/hadolint-action@06be81baf89a55ffd0e24b8f04a4185738dd3387 # v3.5.0
with:
dockerfile: api/Dockerfile
failure-threshold: warning
# DL3013 `pip install uv` unpinned — uv is the installer; the versions
# that matter are pinned in uv.lock, which the next line honours.
# DL3008 unpinned apt `curl` — pinning a Debian point release breaks the
# build on every security update of the base image.
# DL3066 non-numeric USER — `useradd -u 1000` gives it a fixed uid; the
# name is what the COPY --chown lines read.
# DL3025 shell-form HEALTHCHECK CMD — the `|| exit 1` fallback needs a
# shell; JSON form cannot express it.
ignore: DL3013,DL3008,DL3066,DL3025
- name: Hadolint (app/Dockerfile)
uses: hadolint/hadolint-action@06be81baf89a55ffd0e24b8f04a4185738dd3387 # v3.5.0
with:
dockerfile: app/Dockerfile
failure-threshold: warning
app-image:
name: App image (build + origin-gate smoke)
runs-on: ubuntu-latest
timeout-minutes: 25
# The same idea as the job above, for a sharper reason: what app/Dockerfile
# produces is not a program that fails to import, it is an nginx that either
# boots or does not — and since the origin gate
# (app/origin-gate.conf.template) the config it boots with is RENDERED at
# container start from two environment variables. Nothing before this job
# ever ran that entrypoint: the deploy's pre-traffic smoke was the first
# place the rendered config existed, and that lives in Cloud Build, after
# the merge.
#
# A separate job rather than more steps in `image`, so the SPA build never
# delays the API smoke and neither failure hides the other. It runs on every
# PR like `image` does; the GHA cache keeps an unchanged Dockerfile at
# seconds.
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
# Its own cache scope: this image shares no layer with the API's, and one
# shared scope would have them evicting each other.
- name: Build the app image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: app
file: app/Dockerfile
push: false
load: true
tags: kurrentschrift-app:ci
cache-from: type=gha,scope=app-image
cache-to: type=gha,mode=max,scope=app-image
# THE assert this job exists for: the gate's three states, against the
# real image, through the real entrypoint. tests/test_app_origin_gate.py
# can only say what the config TEXT says; only a running container can say
# that envsubst rendered it, that nginx accepted the result, and that the
# maps decide what they were meant to decide.
- name: Origin gate matrix (off / armed / armed with no secret)
run: |
set -euo pipefail
# Not a production value and never one: this container is thrown away
# at the end of the job, and the real secret has no business on a
# runner that prints its own logs on failure. It is the LENGTH of a
# real one on purpose — 64 characters, what `openssl rand -hex 32`
# produces. In the sibling repo a short placeholder passed this job
# while the production length did not: the tagged map key is
# `presented:` plus the secret, and nginx cannot hash a key longer
# than one bucket, so with the gate armed the container refused to
# start. That is the bug this line keeps caught (see
# map_hash_bucket_size in app/origin-gate.conf.template).
SECRET="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
start() { # start <name> <host-port> [docker run args...]
local name="$1" port="$2"; shift 2
docker run -d --name "$name" -p "$port:8080" "$@" kurrentschrift-app:ci >/dev/null
for _ in $(seq 30); do
if curl -fsS "localhost:$port/_health" >/dev/null 2>&1; then return 0; fi
sleep 1
done
echo "::error::container $name never answered /_health within 30 s"
docker logs "$name" || true
exit 1
}
code() { curl -s -o /dev/null -w '%{http_code}' "$@"; }
verdict() { curl -sI "$@" | tr -d '\r' | sed -n 's/^[Xx]-[Oo]rigin-[Gg]ate: //p'; }
want() { # want <what> <got> <expected>
if [ "$2" != "$3" ]; then echo "::error::$1: got '$2', expected '$3'"; exit 1; fi
echo "OK: $1 = $3"
}
# 1. Unset — the state this change ships in and rolls back to. Nothing
# is refused, and /_health already reports whether a header arrived,
# which is what makes arming a measurement instead of a leap.
start gate-off 8081
want "gate off: the shell still serves" "$(code localhost:8081/)" 200
want "gate off: no header" "$(verdict localhost:8081/_health)" off
want "gate off: a header arrived" "$(verdict -H 'X-Origin-Secret: anything' localhost:8081/_health)" off-seen
# 2. Armed. The door is shut, the exempt path is not, and the refusal
# says which of the two armed failures it is.
start gate-on 8082 -e ORIGIN_GATE=on -e ORIGIN_SECRET="$SECRET"
want "armed: no header is refused" "$(code localhost:8082/)" 403
want "armed: wrong secret is refused" "$(code -H "X-Origin-Secret: wrong" localhost:8082/)" 403
want "armed: the right secret passes" "$(code -H "X-Origin-Secret: $SECRET" localhost:8082/)" 200
want "armed: /_health stays exempt" "$(code localhost:8082/_health)" 200
want "armed: /_health names the failure" "$(verdict localhost:8082/_health)" missing
want "armed: a wrong secret is a mismatch" "$(verdict -H "X-Origin-Secret: wrong" localhost:8082/_health)" mismatch
want "armed: the right secret is ok" "$(verdict -H "X-Origin-Secret: $SECRET" localhost:8082/_health)" ok
# The refusal is a page, not nginx's stock one — which would print the
# exact nginx version to anyone knocking on the raw origin.
curl -s localhost:8082/ -o denied.html
grep -qF "nur Anfragen, die durch den Edge gekommen sind" denied.html \
|| { echo "::error::the 403 body is not the gate's own page"; cat denied.html; exit 1; }
# The one thing that must never leak, in the two places it could.
# `if !` rather than `grep … && exit`: under `set -e` a grep that
# finds nothing — the passing case — would end the script itself.
if grep -qF "$SECRET" denied.html; then
echo "::error::the 403 body carries the secret"; exit 1
fi
if docker logs gate-on 2>&1 | grep -qF "$SECRET"; then
echo "::error::the container logged the secret"; exit 1
fi
echo "OK: the secret is in neither the refusal nor the logs"
# The rendered config has to be valid nginx, and `nginx -t` says so
# without dumping it — `nginx -T` would print the secret into this log.
docker exec gate-on nginx -t
echo "OK: the rendered config validates"
# 3. Armed with NO secret must fail CLOSED. This is the whole reason
# the map keys are tagged: an untagged `"${ORIGIN_SECRET}"` key
# would render as `""`, which is exactly what an absent header
# looks like — and a forgotten variable would open the door to the
# entire internet while looking armed.
start gate-shut 8083 -e ORIGIN_GATE=on
want "armed, no secret: refused" "$(code localhost:8083/)" 403
want "armed, no secret: empty header too" "$(code -H 'X-Origin-Secret;' localhost:8083/)" 403
want "armed, no secret: /_health exempt" "$(code localhost:8083/_health)" 200
- name: Container logs on failure
if: failure()
run: |
for c in gate-off gate-on gate-shut; do
echo "=== $c ==="
docker logs "$c" 2>&1 || true
done