diff --git a/src/format_bench/model.py b/src/format_bench/model.py index 9caa903..30d6d9d 100644 --- a/src/format_bench/model.py +++ b/src/format_bench/model.py @@ -1,8 +1,10 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass from enum import StrEnum from pathlib import Path +from types import MappingProxyType class Lane(StrEnum): @@ -40,10 +42,18 @@ class ExecutionState(StrEnum): ExecutionState.FAILED: frozenset(), } _FAILURES = frozenset({ExecutionState.UNSUPPORTED, ExecutionState.FAILED}) +_ACTIVE = frozenset( + { + ExecutionState.DISCOVERED, + ExecutionState.ENCODED, + ExecutionState.ROUNDTRIP_VERIFIED, + ExecutionState.BENCHMARKED, + } +) def transition(current: ExecutionState, target: ExecutionState) -> ExecutionState: - allowed = _NEXT[current] | (_FAILURES if current not in _FAILURES else frozenset()) + allowed = _NEXT[current] | (_FAILURES if current in _ACTIVE else frozenset()) if target not in allowed: raise ValueError(f"illegal evidence transition: {current} -> {target}") return target @@ -65,7 +75,12 @@ class DatasetSpec: canonical_hash: str rows: int columns: tuple[ColumnSpec, ...] - expected_counts: dict[str, int] + expected_counts: Mapping[str, int] + + def __post_init__(self) -> None: + object.__setattr__( + self, "expected_counts", MappingProxyType(dict(self.expected_counts)) + ) def asset_path(self, root: Path) -> Path: path = Path(self.asset_name) diff --git a/src/format_bench/release.py b/src/format_bench/release.py index 57f1c31..f158bb9 100644 --- a/src/format_bench/release.py +++ b/src/format_bench/release.py @@ -29,10 +29,10 @@ def _safe_slug(value: str) -> str: def package_run(run_dir: Path, output: Path, platform: str) -> Path: manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8")) results = json.loads((run_dir / "results.json").read_text(encoding="utf-8")) - if manifest["state"] != ExecutionState.BENCHMARKED: - raise ValueError("release packaging requires benchmarked evidence") - if results["state"] != ExecutionState.BENCHMARKED: - raise ValueError("release packaging requires benchmarked results") + if manifest["state"] != ExecutionState.REPORTED: + raise ValueError("release packaging requires reported evidence") + if results["state"] != ExecutionState.REPORTED: + raise ValueError("release packaging requires reported results") if manifest["dataset_id"] != results["dataset_id"]: raise ValueError("release manifest and results dataset mismatch") diff --git a/src/format_bench/report.py b/src/format_bench/report.py index 56cbc62..1c17e35 100644 --- a/src/format_bench/report.py +++ b/src/format_bench/report.py @@ -3,7 +3,7 @@ import json from pathlib import Path -from .model import Comparability, ExecutionState +from .model import Comparability, ExecutionState, transition def _cell(value: object) -> str: @@ -164,8 +164,9 @@ def _prompt(results: dict) -> list[str]: def render_report(run_dir: Path) -> Path: manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8")) results = json.loads((run_dir / "results.json").read_text(encoding="utf-8")) - if manifest["state"] != ExecutionState.BENCHMARKED or results["state"] != ExecutionState.BENCHMARKED: - raise ValueError("report requires benchmarked manifest and results") + reportable = {ExecutionState.BENCHMARKED, ExecutionState.REPORTED} + if manifest["state"] not in reportable or results["state"] not in reportable: + raise ValueError("report requires benchmarked or reported manifest and results") if manifest["dataset_id"] != results["dataset_id"]: raise ValueError("manifest and results dataset mismatch") profile = results["profile"] @@ -188,4 +189,14 @@ def render_report(run_dir: Path) -> Path: ] path = run_dir / "report.md" path.write_text("\n".join(lines), encoding="utf-8") + # LLM contract: BENCHMARKED -> REPORTED after the human-readable evidence exists. + for payload, json_path in ( + (manifest, run_dir / "manifest.json"), + (results, run_dir / "results.json"), + ): + if payload["state"] == ExecutionState.BENCHMARKED: + payload["state"] = transition(ExecutionState.BENCHMARKED, ExecutionState.REPORTED) + json_path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) return path diff --git a/src/format_bench/workflow.py b/src/format_bench/workflow.py index 5111704..2997514 100644 --- a/src/format_bench/workflow.py +++ b/src/format_bench/workflow.py @@ -64,6 +64,8 @@ def prepare_run( _write_json(input_dir / "manifest.json", effective) entries = [] + # LLM contract: DISCOVERED -> ENCODED -> ROUNDTRIP_VERIFIED -> BENCHMARKED -> REPORTED. + # Active evidence may terminate as UNSUPPORTED or FAILED; terminal evidence never ranks. for adapter in selected or adapters(): description = adapter.describe() artifact_path = destination / "artifacts" / ( @@ -116,6 +118,7 @@ def verify_run(run_dir: Path, selected: dict[str, FormatAdapter] | None = None) (run_dir / run_manifest["input"]["manifest"]).read_text(encoding="utf-8") ) registered = selected or adapter_map() + # LLM contract: only ENCODED evidence can advance to ROUNDTRIP_VERIFIED here. for entry in run_manifest["formats"]: if entry["state"] != ExecutionState.ENCODED: continue diff --git a/tests/test_model.py b/tests/test_model.py index 92fc6ad..2f86db7 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -27,6 +27,10 @@ def test_lifecycle_rejects_skipped_or_terminal_transitions() -> None: transition(ExecutionState.DISCOVERED, ExecutionState.BENCHMARKED) with pytest.raises(ValueError, match="illegal evidence transition"): transition(ExecutionState.FAILED, ExecutionState.DISCOVERED) + with pytest.raises(ValueError, match="illegal evidence transition"): + transition(ExecutionState.REPORTED, ExecutionState.FAILED) + with pytest.raises(ValueError, match="illegal evidence transition"): + transition(ExecutionState.REPORTED, ExecutionState.UNSUPPORTED) def test_failure_is_available_from_an_active_state() -> None: @@ -45,6 +49,8 @@ def test_dataset_asset_path_stays_under_the_run_root() -> None: expected_counts={"rows": 1}, ) assert spec.asset_path(Path("datasets")) == Path("datasets/source.csv") + with pytest.raises(TypeError): + spec.expected_counts["rows"] = 2 unsafe = DatasetSpec(**{**spec.__dict__, "asset_name": "../source.csv"}) with pytest.raises(ValueError, match="safe relative path"): diff --git a/tests/test_release.py b/tests/test_release.py index bf6ad58..cc5743f 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -11,9 +11,9 @@ def test_release_package_is_deterministic_and_relative(tmp_path: Path) -> None: run = tmp_path / "run" (run / "input").mkdir(parents=True) - manifest = {"state": "BENCHMARKED", "dataset_id": "fixture"} + manifest = {"state": "REPORTED", "dataset_id": "fixture"} results = { - "state": "BENCHMARKED", + "state": "REPORTED", "dataset_id": "fixture", "profile": "fair", "run_id": "run-1", diff --git a/tests/test_report.py b/tests/test_report.py index e57c053..3493594 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -48,6 +48,8 @@ def test_prompt_report_is_deterministic_and_includes_exact_tokens(tmp_path: Path first = path.read_text() assert "| compact_tsv | 10 | 4 | 14 | 3 | 4 |" in first assert "Direct token counts for binary formats are N/A." in first + assert json.loads((tmp_path / "manifest.json").read_text())["state"] == "REPORTED" + assert json.loads((tmp_path / "results.json").read_text())["state"] == "REPORTED" assert render_report(tmp_path).read_text() == first