fix: close lifecycle terminal states (#74)

Author: Anionix

src/format_bench/model.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
+from collections.abc import Mapping
 from dataclasses import dataclass
 from enum import StrEnum
 from pathlib import Path
+from types import MappingProxyType
 
 
 class Lane(StrEnum):
@@ -40,10 +42,18 @@ class ExecutionState(StrEnum):
     ExecutionState.FAILED: frozenset(),
 }
 _FAILURES = frozenset({ExecutionState.UNSUPPORTED, ExecutionState.FAILED})
+_ACTIVE = frozenset(
+    {
+        ExecutionState.DISCOVERED,
+        ExecutionState.ENCODED,
+        ExecutionState.ROUNDTRIP_VERIFIED,
+        ExecutionState.BENCHMARKED,
+    }
+)
 
 
 def transition(current: ExecutionState, target: ExecutionState) -> ExecutionState:
-    allowed = _NEXT[current] | (_FAILURES if current not in _FAILURES else frozenset())
+    allowed = _NEXT[current] | (_FAILURES if current in _ACTIVE else frozenset())
     if target not in allowed:
         raise ValueError(f"illegal evidence transition: {current} -> {target}")
     return target
@@ -65,7 +75,12 @@ class DatasetSpec:
     canonical_hash: str
     rows: int
     columns: tuple[ColumnSpec, ...]
-    expected_counts: dict[str, int]
+    expected_counts: Mapping[str, int]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(
+            self, "expected_counts", MappingProxyType(dict(self.expected_counts))
+        )
 
     def asset_path(self, root: Path) -> Path:
         path = Path(self.asset_name)

src/format_bench/release.py

@@ -29,10 +29,10 @@ def _safe_slug(value: str) -> str:
 def package_run(run_dir: Path, output: Path, platform: str) -> Path:
     manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8"))
     results = json.loads((run_dir / "results.json").read_text(encoding="utf-8"))
-    if manifest["state"] != ExecutionState.BENCHMARKED:
-        raise ValueError("release packaging requires benchmarked evidence")
-    if results["state"] != ExecutionState.BENCHMARKED:
-        raise ValueError("release packaging requires benchmarked results")
+    if manifest["state"] != ExecutionState.REPORTED:
+        raise ValueError("release packaging requires reported evidence")
+    if results["state"] != ExecutionState.REPORTED:
+        raise ValueError("release packaging requires reported results")
     if manifest["dataset_id"] != results["dataset_id"]:
         raise ValueError("release manifest and results dataset mismatch")
 

src/format_bench/report.py

@@ -3,7 +3,7 @@
 import json
 from pathlib import Path
 
-from .model import Comparability, ExecutionState
+from .model import Comparability, ExecutionState, transition
 
 
 def _cell(value: object) -> str:
@@ -164,8 +164,9 @@ def _prompt(results: dict) -> list[str]:
 def render_report(run_dir: Path) -> Path:
     manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8"))
     results = json.loads((run_dir / "results.json").read_text(encoding="utf-8"))
-    if manifest["state"] != ExecutionState.BENCHMARKED or results["state"] != ExecutionState.BENCHMARKED:
-        raise ValueError("report requires benchmarked manifest and results")
+    reportable = {ExecutionState.BENCHMARKED, ExecutionState.REPORTED}
+    if manifest["state"] not in reportable or results["state"] not in reportable:
+        raise ValueError("report requires benchmarked or reported manifest and results")
     if manifest["dataset_id"] != results["dataset_id"]:
         raise ValueError("manifest and results dataset mismatch")
     profile = results["profile"]
@@ -188,4 +189,14 @@ def render_report(run_dir: Path) -> Path:
     ]
     path = run_dir / "report.md"
     path.write_text("\n".join(lines), encoding="utf-8")
+    # LLM contract: BENCHMARKED -> REPORTED after the human-readable evidence exists.
+    for payload, json_path in (
+        (manifest, run_dir / "manifest.json"),
+        (results, run_dir / "results.json"),
+    ):
+        if payload["state"] == ExecutionState.BENCHMARKED:
+            payload["state"] = transition(ExecutionState.BENCHMARKED, ExecutionState.REPORTED)
+        json_path.write_text(
+            json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+        )
     return path

src/format_bench/workflow.py

@@ -64,6 +64,8 @@ def prepare_run(
     _write_json(input_dir / "manifest.json", effective)
 
     entries = []
+    # LLM contract: DISCOVERED -> ENCODED -> ROUNDTRIP_VERIFIED -> BENCHMARKED -> REPORTED.
+    # Active evidence may terminate as UNSUPPORTED or FAILED; terminal evidence never ranks.
     for adapter in selected or adapters():
         description = adapter.describe()
         artifact_path = destination / "artifacts" / (
@@ -116,6 +118,7 @@ def verify_run(run_dir: Path, selected: dict[str, FormatAdapter] | None = None)
         (run_dir / run_manifest["input"]["manifest"]).read_text(encoding="utf-8")
     )
     registered = selected or adapter_map()
+    # LLM contract: only ENCODED evidence can advance to ROUNDTRIP_VERIFIED here.
     for entry in run_manifest["formats"]:
         if entry["state"] != ExecutionState.ENCODED:
             continue

tests/test_model.py

@@ -27,6 +27,10 @@ def test_lifecycle_rejects_skipped_or_terminal_transitions() -> None:
         transition(ExecutionState.DISCOVERED, ExecutionState.BENCHMARKED)
     with pytest.raises(ValueError, match="illegal evidence transition"):
         transition(ExecutionState.FAILED, ExecutionState.DISCOVERED)
+    with pytest.raises(ValueError, match="illegal evidence transition"):
+        transition(ExecutionState.REPORTED, ExecutionState.FAILED)
+    with pytest.raises(ValueError, match="illegal evidence transition"):
+        transition(ExecutionState.REPORTED, ExecutionState.UNSUPPORTED)
 
 
 def test_failure_is_available_from_an_active_state() -> None:
@@ -45,6 +49,8 @@ def test_dataset_asset_path_stays_under_the_run_root() -> None:
         expected_counts={"rows": 1},
     )
     assert spec.asset_path(Path("datasets")) == Path("datasets/source.csv")
+    with pytest.raises(TypeError):
+        spec.expected_counts["rows"] = 2
 
     unsafe = DatasetSpec(**{**spec.__dict__, "asset_name": "../source.csv"})
     with pytest.raises(ValueError, match="safe relative path"):

tests/test_release.py

@@ -11,9 +11,9 @@
 def test_release_package_is_deterministic_and_relative(tmp_path: Path) -> None:
     run = tmp_path / "run"
     (run / "input").mkdir(parents=True)
-    manifest = {"state": "BENCHMARKED", "dataset_id": "fixture"}
+    manifest = {"state": "REPORTED", "dataset_id": "fixture"}
     results = {
-        "state": "BENCHMARKED",
+        "state": "REPORTED",
         "dataset_id": "fixture",
         "profile": "fair",
         "run_id": "run-1",

tests/test_report.py

@@ -48,6 +48,8 @@ def test_prompt_report_is_deterministic_and_includes_exact_tokens(tmp_path: Path
     first = path.read_text()
     assert "| compact_tsv | 10 | 4 | 14 | 3 | 4 |" in first
     assert "Direct token counts for binary formats are N/A." in first
+    assert json.loads((tmp_path / "manifest.json").read_text())["state"] == "REPORTED"
+    assert json.loads((tmp_path / "results.json").read_text())["state"] == "REPORTED"
     assert render_report(tmp_path).read_text() == first