Package complete release evidence

README.ja.md

@@ -36,17 +36,17 @@ DISCOVERED -> ENCODED -> ROUNDTRIP_VERIFIED -> BENCHMARKED -> REPORTED
 ```bash
 nix develop
 uv sync --frozen
-format-bench run --profile prompt --dataset github-stars-2026-07-03 --fixture
+uv run --frozen format-bench run --profile prompt --dataset github-stars-2026-07-03 --fixture
 ```
 
 `--fixture`は順位対象外のsmoke testです。公開Releaseの全データを使う場合は次の順です。
 
 ```bash
-format-bench dataset fetch github-stars-2026-07-03
-format-bench prepare --dataset github-stars-2026-07-03 --run-dir runs/fair-local
-format-bench verify --run-dir runs/fair-local
-format-bench run --profile fair --dataset github-stars-2026-07-03 --run-dir runs/fair-local
-format-bench report --run-dir runs/fair-local
+uv run --frozen format-bench dataset fetch github-stars-2026-07-03
+uv run --frozen format-bench prepare --dataset github-stars-2026-07-03 --run-dir runs/fair-local
+uv run --frozen format-bench verify --run-dir runs/fair-local
+uv run --frozen format-bench run --profile fair --dataset github-stars-2026-07-03 --run-dir runs/fair-local
+uv run --frozen format-bench report --run-dir runs/fair-local
 ```
 
 測定値そのものはCIの合否に使いません。公開測定はmacOS ARMとLinux x86_64を別runにし、入力hash、commit、flake lock、依存版、seed、writer設定、失敗理由とともに保存します。

README.md

@@ -42,17 +42,17 @@ Nix pins Python 3.12 and native tools. `uv.lock` pins the Python environment.
 ```bash
 nix develop
 uv sync --frozen
-format-bench run --profile prompt --dataset github-stars-2026-07-03 --fixture
+uv run --frozen format-bench run --profile prompt --dataset github-stars-2026-07-03 --fixture
 ```
 
 The fixture command is a non-rankable smoke test. For the full published dataset:
 
 ```bash
-format-bench dataset fetch github-stars-2026-07-03
-format-bench prepare --dataset github-stars-2026-07-03 --run-dir runs/fair-local
-format-bench verify --run-dir runs/fair-local
-format-bench run --profile fair --dataset github-stars-2026-07-03 --run-dir runs/fair-local
-format-bench report --run-dir runs/fair-local
+uv run --frozen format-bench dataset fetch github-stars-2026-07-03
+uv run --frozen format-bench prepare --dataset github-stars-2026-07-03 --run-dir runs/fair-local
+uv run --frozen format-bench verify --run-dir runs/fair-local
+uv run --frozen format-bench run --profile fair --dataset github-stars-2026-07-03 --run-dir runs/fair-local
+uv run --frozen format-bench report --run-dir runs/fair-local
 ```
 
 Run `claims` and `prompt` in separate run directories. Omitting `--run-dir` makes `run` prepare and verify a new timestamped directory automatically.

src/format_bench/release.py

@@ -18,6 +18,7 @@
     "report.md",
     "input/manifest.json",
 )
+ARTIFACT_ROOTS = ("artifacts", "claims", "prompt")
 
 
 def _safe_slug(value: str) -> str:
@@ -26,6 +27,58 @@ def _safe_slug(value: str) -> str:
     return value
 
 
+def _artifact_references(manifest: dict, results: dict) -> list[str]:
+    references = [
+        entry["artifact"]
+        for entry in manifest.get("formats", [])
+        if isinstance(entry.get("artifact"), str)
+    ]
+    for observation in results.get("results", {}).values():
+        if not isinstance(observation, dict):
+            continue
+        evidence = observation.get("evidence", {})
+        for source in (observation, evidence):
+            if not isinstance(source, dict):
+                continue
+            if isinstance(source.get("artifact"), str):
+                references.append(source["artifact"])
+            if isinstance(source.get("artifacts"), dict):
+                references.extend(
+                    item for item in source["artifacts"].values() if isinstance(item, str)
+                )
+    return references
+
+
+def _release_files(run_dir: Path, manifest: dict, results: dict) -> list[Path]:
+    required = [run_dir / relative for relative in EVIDENCE_FILES]
+    missing = [str(path.relative_to(run_dir)) for path in required if not path.is_file()]
+    if missing:
+        raise FileNotFoundError(f"release evidence missing: {', '.join(missing)}")
+
+    run_root = run_dir.resolve()
+    referenced_files = set()
+    for value in _artifact_references(manifest, results):
+        relative = Path(value)
+        if relative.is_absolute() or ".." in relative.parts:
+            raise ValueError(f"release artifact path is unsafe: {value}")
+        target = run_dir / relative
+        if not target.exists() or not target.resolve().is_relative_to(run_root):
+            raise FileNotFoundError(f"release artifact missing or unsafe: {value}")
+        if target.is_file():
+            referenced_files.add(target)
+        else:
+            referenced_files.update(path for path in target.rglob("*") if path.is_file())
+
+    files = set(required) | referenced_files
+    for name in ARTIFACT_ROOTS:
+        root = run_dir / name
+        if root.exists():
+            files.update(path for path in root.rglob("*") if path.is_file())
+    if any(not path.resolve().is_relative_to(run_root) for path in files):
+        raise ValueError("release artifact resolves outside the run directory")
+    return sorted(files, key=lambda path: path.relative_to(run_dir).as_posix())
+
+
 def package_run(run_dir: Path, output: Path, platform: str) -> Path:
     manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8"))
     results = json.loads((run_dir / "results.json").read_text(encoding="utf-8"))
@@ -36,10 +89,7 @@ def package_run(run_dir: Path, output: Path, platform: str) -> Path:
     if manifest["dataset_id"] != results["dataset_id"]:
         raise ValueError("release manifest and results dataset mismatch")
 
-    files = [run_dir / relative for relative in EVIDENCE_FILES]
-    missing = [str(path.relative_to(run_dir)) for path in files if not path.is_file()]
-    if missing:
-        raise FileNotFoundError(f"release evidence missing: {', '.join(missing)}")
+    files = _release_files(run_dir, manifest, results)
 
     output.mkdir(parents=True, exist_ok=True)
     name = f"data-format-lab-{results['profile']}-{_safe_slug(platform)}-{results['run_id']}"

src/format_bench/report.py

@@ -87,6 +87,7 @@ def _fair(manifest: dict, results: dict) -> list[str]:
                 evidence["warm"]["p95_ms"],
                 evidence["warm"]["iqr_ms"],
                 evidence["result"],
+                evidence["evidence"]["normalized_hash"],
                 evidence["max_rss_bytes_p50"],
             ]
         )
@@ -96,7 +97,17 @@ def _fair(manifest: dict, results: dict) -> list[str]:
             "## Fair Operations",
             "",
             *_table(
-                ["Format", "Operation", "Fresh p50 ms", "Warm p50 ms", "Warm p95 ms", "IQR ms", "Rows", "RSS bytes"],
+                [
+                    "Format",
+                    "Operation",
+                    "Fresh p50 ms",
+                    "Warm p50 ms",
+                    "Warm p95 ms",
+                    "IQR ms",
+                    "Rows",
+                    "Result hash",
+                    "RSS bytes",
+                ],
                 timings,
             ),
         ]
@@ -130,6 +141,7 @@ def _prompt(results: dict) -> list[str]:
             name,
             item["payload_bytes"],
             item["taxonomy_bytes"],
+            item.get("schema_bytes", 0),
             item["total_bytes"],
             item["tokens"]["o200k_base"],
             item["tokens"]["cl100k_base"],
@@ -151,7 +163,18 @@ def _prompt(results: dict) -> list[str]:
     return [
         "## Prompt Corpus",
         "",
-        *_table(["Format", "Payload bytes", "Taxonomy bytes", "Total bytes", "o200k", "cl100k"], corpus),
+        *_table(
+            [
+                "Format",
+                "Payload bytes",
+                "Taxonomy bytes",
+                "Schema bytes",
+                "Total bytes",
+                "o200k",
+                "cl100k",
+            ],
+            corpus,
+        ),
         "",
         "## Retrieval Payload",
         "",

tests/test_release.py

@@ -5,18 +5,27 @@
 
 import zstandard as zstd
 
-from format_bench.release import EVIDENCE_FILES, package_run
+from format_bench.release import package_run
 
 
 def test_release_package_is_deterministic_and_relative(tmp_path: Path) -> None:
     run = tmp_path / "run"
     (run / "input").mkdir(parents=True)
-    manifest = {"state": "REPORTED", "dataset_id": "fixture"}
+    manifest = {
+        "state": "REPORTED",
+        "dataset_id": "fixture",
+        "formats": [{"artifact": "artifacts/value.bin"}],
+    }
     results = {
         "state": "REPORTED",
         "dataset_id": "fixture",
         "profile": "fair",
         "run_id": "run-1",
+        "results": {
+            "negative_research": {
+                "source_commits": {"artifact": "not-a-run-path"}
+            }
+        },
     }
     payloads = {
         "manifest.json": json.dumps(manifest),
@@ -26,6 +35,10 @@ def test_release_package_is_deterministic_and_relative(tmp_path: Path) -> None:
     }
     for relative, payload in payloads.items():
         (run / relative).write_text(payload)
+    (run / "artifacts").mkdir()
+    (run / "artifacts" / "value.bin").write_bytes(b"artifact")
+    (run / "claims" / "nested").mkdir(parents=True)
+    (run / "claims" / "nested" / "evidence.bin").write_bytes(b"claim")
 
     first = package_run(run, tmp_path / "first", "linux-x86_64")
     second = package_run(run, tmp_path / "second", "linux-x86_64")
@@ -35,5 +48,45 @@ def test_release_package_is_deterministic_and_relative(tmp_path: Path) -> None:
 
     tar_bytes = zstd.ZstdDecompressor().decompress(first.read_bytes())
     with tarfile.open(fileobj=__import__("io").BytesIO(tar_bytes)) as archive:
-        assert archive.getnames() == [f"run-1/{name}" for name in EVIDENCE_FILES]
+        assert archive.getnames() == [
+            "run-1/artifacts/value.bin",
+            "run-1/claims/nested/evidence.bin",
+            "run-1/input/manifest.json",
+            "run-1/manifest.json",
+            "run-1/report.md",
+            "run-1/results.json",
+        ]
         assert all(not Path(name).is_absolute() for name in archive.getnames())
+
+
+def test_release_rejects_missing_referenced_artifact(tmp_path: Path) -> None:
+    run = tmp_path / "run"
+    (run / "input").mkdir(parents=True)
+    (run / "manifest.json").write_text(
+        json.dumps(
+            {
+                "state": "REPORTED",
+                "dataset_id": "fixture",
+                "formats": [{"artifact": "artifacts/missing.bin"}],
+            }
+        )
+    )
+    (run / "results.json").write_text(
+        json.dumps(
+            {
+                "state": "REPORTED",
+                "dataset_id": "fixture",
+                "profile": "fair",
+                "run_id": "run-1",
+            }
+        )
+    )
+    (run / "report.md").write_text("# report\n")
+    (run / "input" / "manifest.json").write_text('{}\n')
+
+    try:
+        package_run(run, tmp_path / "output", "linux-x86_64")
+    except FileNotFoundError as error:
+        assert "artifacts/missing.bin" in str(error)
+    else:
+        raise AssertionError("missing artifact was accepted")

tests/test_report.py

@@ -27,7 +27,8 @@ def test_prompt_report_is_deterministic_and_includes_exact_tokens(tmp_path: Path
                         "compact_tsv": {
                             "payload_bytes": 10,
                             "taxonomy_bytes": 4,
-                            "total_bytes": 14,
+                            "schema_bytes": 2,
+                            "total_bytes": 16,
                             "tokens": {"o200k_base": 3, "cl100k_base": 4},
                         }
                     },
@@ -46,7 +47,7 @@ def test_prompt_report_is_deterministic_and_includes_exact_tokens(tmp_path: Path
     (tmp_path / "results.json").write_text(json.dumps(results))
     path = render_report(tmp_path)
     first = path.read_text()
-    assert "| compact_tsv | 10 | 4 | 14 | 3 | 4 |" in first
+    assert "| compact_tsv | 10 | 4 | 2 | 16 | 3 | 4 |" in first
     assert "Direct token counts for binary formats are N/A." in first
     assert json.loads((tmp_path / "manifest.json").read_text())["state"] == "REPORTED"
     assert json.loads((tmp_path / "results.json").read_text())["state"] == "REPORTED"
@@ -58,3 +59,49 @@ def test_report_rejects_unbenchmarked_evidence(tmp_path: Path) -> None:
     (tmp_path / "results.json").write_text('{"state":"BENCHMARKED"}')
     with pytest.raises(ValueError, match="requires benchmarked"):
         render_report(tmp_path)
+
+
+def test_fair_report_includes_normalized_result_hash(tmp_path: Path) -> None:
+    manifest = {
+        "state": "BENCHMARKED",
+        "dataset_id": "fixture",
+        "rankable": True,
+        "formats": [
+            {
+                "format": "csv",
+                "comparability": "FULL_COMPARABLE",
+                "state": "BENCHMARKED",
+                "native_bytes": 10,
+                "transport_zstd_bytes": 8,
+            }
+        ],
+    }
+    results = {
+        "state": "BENCHMARKED",
+        "dataset_id": "fixture",
+        "run_id": "fair-fixture",
+        "profile": "fair",
+        "environment": {
+            "git_commit": "abc",
+            "flake_lock_sha256": "def",
+            "platform": "test-os",
+            "machine": "test-cpu",
+            "python": "3.12.0",
+        },
+        "results": {
+            "csv/read_all": {
+                "status": "MEASURED",
+                "fresh_process": {"p50_ms": 1},
+                "warm": {"p50_ms": 1, "p95_ms": 2, "iqr_ms": 1},
+                "result": 4,
+                "evidence": {"normalized_hash": "abc123"},
+                "max_rss_bytes_p50": 100,
+            }
+        },
+    }
+    (tmp_path / "manifest.json").write_text(json.dumps(manifest))
+    (tmp_path / "results.json").write_text(json.dumps(results))
+
+    report = render_report(tmp_path).read_text()
+
+    assert "| csv | read_all | 1 | 1 | 2 | 1 | 4 | abc123 | 100 |" in report