M30: things that stand on the floor #50

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main]
	pull_request:
	workflow_dispatch:

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	env:
	# Never let a test reach a paid API. Live runs are opt-in and marked.
	ANTHROPIC_API_KEY: ""

	jobs:
	quality:
	name: lint + types
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: ruff format
	run: uv run ruff format --check .
	- name: ruff lint
	run: uv run ruff check .
	- name: mypy (strict)
	run: uv run mypy

	test:
	name: test · ${{ matrix.os }}
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, windows-latest]
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: pytest
	run: uv run pytest --cov --cov-report=term-missing -m "not live"

	invariants:
	name: scientific invariants
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: invariant + isolation suites
	# These prove that no sequence of public API calls can violate a rule
	# in docs/03-data-model.md, and that role-visibility boundaries hold.
	# They are a separate job so a failure here is never mistaken for an
	# ordinary test failure.
	run: uv run pytest -m "invariant or isolation" -v

	postgres:
	name: invariants on postgres
	runs-on: ubuntu-latest
	# ADR-0001 claims the same schema and the same invariants run on both
	# backends. Untested, that is an assertion. This job makes it a check.
	services:
	postgres:
	image: postgres:16
	env:
	POSTGRES_PASSWORD: nullius
	POSTGRES_DB: nullius_test
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 5432:5432
	env:
	NULLIUS_TEST_DATABASE_URL: postgresql+psycopg://postgres:nullius@localhost:5432/nullius_test
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: invariant + isolation suites against postgres
	run: uv run pytest -m "invariant or isolation" -v

	ground-truth:
	name: bank ground truth
	runs-on: ubuntu-latest
	# The bank's answers are recomputed from the data generating process on
	# every push. A change to the DGP that would silently move the ground
	# truth this project is scored against fails here.
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: recompute every locked truth
	run: uv run nullius bank verify
	- name: recompute bank v2's truths, all sixty
	run: \|
	uv run python -c "
	from nullius.bank.items import BANK_V2
	from nullius.bank.lock import V2_LOCK_PATH, verify
	from nullius.bank.truth import ambiguous, boundary_margin
	from nullius.bank.lock import read_lock
	result = verify(V2_LOCK_PATH, items=BANK_V2)
	print(result)
	assert result.ok, result
	truths = list(read_lock(V2_LOCK_PATH).values())
	assert ambiguous(truths) == [], ambiguous(truths)
	print(f'min oracle margin {min(boundary_margin(t) for t in truths):.1f} SE')
	"
	- name: report composition
	run: uv run nullius bank stats

	protocol:
	name: benchmark protocol and results
	runs-on: ubuntu-latest
	# The registered protocol must still describe the bank it was registered
	# against, and the committed results must re-score from their own per-item
	# rows. A results file whose headline numbers cannot be recomputed from the
	# outcomes it ships with is a screenshot, and this is what stops one being
	# committed.
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: every registered protocol still describes its own bank
	run: \|
	uv run python -c "
	import sys
	from pathlib import Path
	from nullius.benchmark.protocol import PROTOCOL_VERSIONS, verify_protocol
	# Enumerated from the registry rather than listed here, so a protocol
	# registered later cannot be left unverified by forgetting a line.
	failed = False
	for version, settings in sorted(PROTOCOL_VERSIONS.items()):
	path = Path(settings['path'])
	if not path.exists():
	print(f'v{version}: {path} not committed'); failed = True; continue
	result = verify_protocol(path)
	print(f'v{version}: {result}')
	failed = failed or not result.ok
	sys.exit(1 if failed else 0)
	"
	- name: every committed results file re-scores to its own summary
	run: \|
	uv run python -c "
	import sys
	from pathlib import Path
	from nullius.benchmark.metrics import read_results
	# Globbed, not listed. A hand-maintained tuple beside the registry is
	# how the v6 results file would have gone unchecked here: this job
	# named five paths and there were six on disk. Fourth instance of the
	# same shape, after the job that listed protocols, the ladder that ran
	# eight arms under a nine-arm plan, and the paper's results-path table.
	paths = sorted(Path('benchmark').glob('results*.lock.json'))
	if not paths:
	sys.exit('no committed results files found; the glob matched nothing')
	print(f'{len(paths)} committed results file(s)')
	for path in paths:
	report, runs = read_results(path)
	print(f'{path.name}: protocol {report.protocol_hash[:16]}, {len(runs)} arms')
	print(' ' + report.prediction_reason)
	for contrast in report.prediction_contrasts:
	print(' ' + str(contrast))
	"

	report:
	name: the report builds and refuses a bad ledger
	runs-on: ubuntu-latest
	# M11's report exits non-zero on a ledger that does not verify or a claim
	# carrying a confidence the ledger no longer supports. Building a real
	# ledger here and rendering it is what keeps that promise honest: the
	# generator has to survive its own check on output nobody curated.
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: carry three bank items through the full institution
	run: \|
	uv run python -c "
	from pathlib import Path
	from nullius.bank.items import BANK_V1
	from nullius.benchmark import arm_named, run_arm
	root = Path('.nullius/ci')
	run_arm(arm_named('B6'), database=root / 'b6.sqlite',
	workroot=root / 'b6', items=BANK_V1[:3])
	"
	- name: render it, and fail if the ledger does not hold up
	run: >
	uv run nullius report build
	-d .nullius/ci/b6.sqlite
	-s .nullius/ci/b6/objects
	--out .nullius/ci/site
	- uses: actions/upload-artifact@v4
	with:
	name: report
	path: .nullius/ci/site

	reproducible:
	name: a custodied arm reproduces exactly
	runs-on: ubuntu-latest
	# The claim the README makes, checked on a clean machine. Until M15 it was
	# false for every arm that queries the Custodian: identifiers were random,
	# so the evaluation seed differed on every run and no custodied result could
	# be reproduced at all.
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: run the same arms twice and compare
	run: uv run pytest tests/test_reproducibility.py -q

	paper:
	name: the paper builds from committed sources
	runs-on: ubuntu-latest
	# Refuses to build when a protocol fails to verify or a results file fails
	# to re-score. That makes the paper a check on the record rather than a
	# description of it: if the two ever diverge, this job is what says so.
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: every protocol and every result, enumerated
	run: uv run nullius paper build --out paper/index.html --markdown FINDINGS.md
	- name: the committed findings are the generated ones
	# The repository's front door states results. If it can drift from them
	# it will, and a project whose thesis is "take nobody's word for it"
	# cannot ask a reader to take its README's word for it.
	run: git diff --exit-code -- FINDINGS.md
	- uses: actions/upload-artifact@v4
	with:
	name: paper
	path: paper/index.html

	station:
	name: the station builds from committed sources alone
	runs-on: ubuntu-latest
	# Aggregate mode reads only committed artifacts, so this job is also the
	# check that the page has no dependency on a ladder's ledger — those are
	# outputs and are not in the clone. It refuses on the same terms the paper
	# does: a drawing whose inputs no longer check out is worse than no drawing,
	# because it looks like evidence and it is prettier than the paper.
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	- run: uv sync --all-extras --dev
	- name: no ledger is present
	run: test ! -d .nullius
	- name: every room, every role, every state
	run: uv run nullius station build --out site/station.html
	- name: the map cannot drift from the enums that define it
	run: uv run pytest tests/test_station.py -q
	- uses: actions/upload-artifact@v4
	with:
	name: station
	path: site/station.html

	clean-clone:
	name: reproducible from a clean clone
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v5
	- uses: astral-sh/setup-uv@v7
	- name: no services, no keys, no container runtime
	run: \|
	uv sync
	uv run nullius version
	uv run nullius doctor

Navigation Menu

M30: things that stand on the floor #50

Workflow file

M30: things that stand on the floor #50

Uh oh!

Workflow file for this run