M14b: adaptive seeding works, and the ladder needs replicating #32
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # Never let a test reach a paid API. Live runs are opt-in and marked. | |
| ANTHROPIC_API_KEY: "" | |
| jobs: | |
| quality: | |
| name: lint + types | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - run: uv sync --all-extras --dev | |
| - name: ruff format | |
| run: uv run ruff format --check . | |
| - name: ruff lint | |
| run: uv run ruff check . | |
| - name: mypy (strict) | |
| run: uv run mypy | |
| test: | |
| name: test · ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - run: uv sync --all-extras --dev | |
| - name: pytest | |
| run: uv run pytest --cov --cov-report=term-missing -m "not live" | |
| invariants: | |
| name: scientific invariants | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - run: uv sync --all-extras --dev | |
| - name: invariant + isolation suites | |
| # These prove that no sequence of public API calls can violate a rule | |
| # in docs/03-data-model.md, and that role-visibility boundaries hold. | |
| # They are a separate job so a failure here is never mistaken for an | |
| # ordinary test failure. | |
| run: uv run pytest -m "invariant or isolation" -v | |
| postgres: | |
| name: invariants on postgres | |
| runs-on: ubuntu-latest | |
| # ADR-0001 claims the same schema and the same invariants run on both | |
| # backends. Untested, that is an assertion. This job makes it a check. | |
| services: | |
| postgres: | |
| image: postgres:16 | |
| env: | |
| POSTGRES_PASSWORD: nullius | |
| POSTGRES_DB: nullius_test | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| env: | |
| NULLIUS_TEST_DATABASE_URL: postgresql+psycopg://postgres:nullius@localhost:5432/nullius_test | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - run: uv sync --all-extras --dev | |
| - name: invariant + isolation suites against postgres | |
| run: uv run pytest -m "invariant or isolation" -v | |
| ground-truth: | |
| name: bank ground truth | |
| runs-on: ubuntu-latest | |
| # The bank's answers are recomputed from the data generating process on | |
| # every push. A change to the DGP that would silently move the ground | |
| # truth this project is scored against fails here. | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - run: uv sync --all-extras --dev | |
| - name: recompute every locked truth | |
| run: uv run nullius bank verify | |
| - name: recompute bank v2's truths, all sixty | |
| run: | | |
| uv run python -c " | |
| from nullius.bank.items import BANK_V2 | |
| from nullius.bank.lock import V2_LOCK_PATH, verify | |
| from nullius.bank.truth import ambiguous, boundary_margin | |
| from nullius.bank.lock import read_lock | |
| result = verify(V2_LOCK_PATH, items=BANK_V2) | |
| print(result) | |
| assert result.ok, result | |
| truths = list(read_lock(V2_LOCK_PATH).values()) | |
| assert ambiguous(truths) == [], ambiguous(truths) | |
| print(f'min oracle margin {min(boundary_margin(t) for t in truths):.1f} SE') | |
| " | |
| - name: report composition | |
| run: uv run nullius bank stats | |
| protocol: | |
| name: benchmark protocol and results | |
| runs-on: ubuntu-latest | |
| # The registered protocol must still describe the bank it was registered | |
| # against, and the committed results must re-score from their own per-item | |
| # rows. A results file whose headline numbers cannot be recomputed from the | |
| # outcomes it ships with is a screenshot, and this is what stops one being | |
| # committed. | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - run: uv sync --all-extras --dev | |
| - name: every registered protocol still describes its own bank | |
| run: | | |
| uv run python -c " | |
| import sys | |
| from pathlib import Path | |
| from nullius.benchmark.protocol import PROTOCOL_VERSIONS, verify_protocol | |
| # Enumerated from the registry rather than listed here, so a protocol | |
| # registered later cannot be left unverified by forgetting a line. | |
| failed = False | |
| for version, settings in sorted(PROTOCOL_VERSIONS.items()): | |
| path = Path(settings['path']) | |
| if not path.exists(): | |
| print(f'v{version}: {path} not committed'); failed = True; continue | |
| result = verify_protocol(path) | |
| print(f'v{version}: {result}') | |
| failed = failed or not result.ok | |
| sys.exit(1 if failed else 0) | |
| " | |
| - name: every committed results file re-scores to its own summary | |
| run: | | |
| uv run python -c " | |
| from pathlib import Path | |
| from nullius.benchmark.metrics import read_results | |
| for path in (Path('benchmark/results.lock.json'), | |
| Path('benchmark/results.v2.lock.json'), | |
| Path('benchmark/results.v3.lock.json'), | |
| Path('benchmark/results.v4.lock.json')): | |
| if not path.exists(): | |
| print(f'{path.name}: not committed yet'); continue | |
| report, runs = read_results(path) | |
| print(f'{path.name}: protocol {report.protocol_hash[:16]}, {len(runs)} arms') | |
| print(' ' + report.prediction_reason) | |
| for contrast in report.prediction_contrasts: | |
| print(' ' + str(contrast)) | |
| " | |
| report: | |
| name: the report builds and refuses a bad ledger | |
| runs-on: ubuntu-latest | |
| # M11's report exits non-zero on a ledger that does not verify or a claim | |
| # carrying a confidence the ledger no longer supports. Building a real | |
| # ledger here and rendering it is what keeps that promise honest: the | |
| # generator has to survive its own check on output nobody curated. | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - run: uv sync --all-extras --dev | |
| - name: carry three bank items through the full institution | |
| run: | | |
| uv run python -c " | |
| from pathlib import Path | |
| from nullius.bank.items import BANK_V1 | |
| from nullius.benchmark import arm_named, run_arm | |
| root = Path('.nullius/ci') | |
| run_arm(arm_named('B6'), database=root / 'b6.sqlite', | |
| workroot=root / 'b6', items=BANK_V1[:3]) | |
| " | |
| - name: render it, and fail if the ledger does not hold up | |
| run: > | |
| uv run nullius report build | |
| -d .nullius/ci/b6.sqlite | |
| -s .nullius/ci/b6/objects | |
| --out .nullius/ci/site | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: report | |
| path: .nullius/ci/site | |
| clean-clone: | |
| name: reproducible from a clean clone | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: astral-sh/setup-uv@v7 | |
| - name: no services, no keys, no container runtime | |
| run: | | |
| uv sync | |
| uv run nullius version | |
| uv run nullius doctor |