diff --git a/README.ja.md b/README.ja.md index c49884f..211f0c6 100644 --- a/README.ja.md +++ b/README.ja.md @@ -36,17 +36,17 @@ DISCOVERED -> ENCODED -> ROUNDTRIP_VERIFIED -> BENCHMARKED -> REPORTED ```bash nix develop uv sync --frozen -format-bench run --profile prompt --dataset github-stars-2026-07-03 --fixture +uv run --frozen format-bench run --profile prompt --dataset github-stars-2026-07-03 --fixture ``` `--fixture`は順位対象外のsmoke testです。公開Releaseの全データを使う場合は次の順です。 ```bash -format-bench dataset fetch github-stars-2026-07-03 -format-bench prepare --dataset github-stars-2026-07-03 --run-dir runs/fair-local -format-bench verify --run-dir runs/fair-local -format-bench run --profile fair --dataset github-stars-2026-07-03 --run-dir runs/fair-local -format-bench report --run-dir runs/fair-local +uv run --frozen format-bench dataset fetch github-stars-2026-07-03 +uv run --frozen format-bench prepare --dataset github-stars-2026-07-03 --run-dir runs/fair-local +uv run --frozen format-bench verify --run-dir runs/fair-local +uv run --frozen format-bench run --profile fair --dataset github-stars-2026-07-03 --run-dir runs/fair-local +uv run --frozen format-bench report --run-dir runs/fair-local ``` 測定値そのものはCIの合否に使いません。公開測定はmacOS ARMとLinux x86_64を別runにし、入力hash、commit、flake lock、依存版、seed、writer設定、失敗理由とともに保存します。 diff --git a/README.md b/README.md index 706b850..edc837c 100644 --- a/README.md +++ b/README.md @@ -42,17 +42,17 @@ Nix pins Python 3.12 and native tools. `uv.lock` pins the Python environment. ```bash nix develop uv sync --frozen -format-bench run --profile prompt --dataset github-stars-2026-07-03 --fixture +uv run --frozen format-bench run --profile prompt --dataset github-stars-2026-07-03 --fixture ``` The fixture command is a non-rankable smoke test. For the full published dataset: ```bash -format-bench dataset fetch github-stars-2026-07-03 -format-bench prepare --dataset github-stars-2026-07-03 --run-dir runs/fair-local -format-bench verify --run-dir runs/fair-local -format-bench run --profile fair --dataset github-stars-2026-07-03 --run-dir runs/fair-local -format-bench report --run-dir runs/fair-local +uv run --frozen format-bench dataset fetch github-stars-2026-07-03 +uv run --frozen format-bench prepare --dataset github-stars-2026-07-03 --run-dir runs/fair-local +uv run --frozen format-bench verify --run-dir runs/fair-local +uv run --frozen format-bench run --profile fair --dataset github-stars-2026-07-03 --run-dir runs/fair-local +uv run --frozen format-bench report --run-dir runs/fair-local ``` Run `claims` and `prompt` in separate run directories. Omitting `--run-dir` makes `run` prepare and verify a new timestamped directory automatically. diff --git a/src/format_bench/release.py b/src/format_bench/release.py index f158bb9..6066413 100644 --- a/src/format_bench/release.py +++ b/src/format_bench/release.py @@ -18,6 +18,7 @@ "report.md", "input/manifest.json", ) +ARTIFACT_ROOTS = ("artifacts", "claims", "prompt") def _safe_slug(value: str) -> str: @@ -26,6 +27,58 @@ def _safe_slug(value: str) -> str: return value +def _artifact_references(manifest: dict, results: dict) -> list[str]: + references = [ + entry["artifact"] + for entry in manifest.get("formats", []) + if isinstance(entry.get("artifact"), str) + ] + for observation in results.get("results", {}).values(): + if not isinstance(observation, dict): + continue + evidence = observation.get("evidence", {}) + for source in (observation, evidence): + if not isinstance(source, dict): + continue + if isinstance(source.get("artifact"), str): + references.append(source["artifact"]) + if isinstance(source.get("artifacts"), dict): + references.extend( + item for item in source["artifacts"].values() if isinstance(item, str) + ) + return references + + +def _release_files(run_dir: Path, manifest: dict, results: dict) -> list[Path]: + required = [run_dir / relative for relative in EVIDENCE_FILES] + missing = [str(path.relative_to(run_dir)) for path in required if not path.is_file()] + if missing: + raise FileNotFoundError(f"release evidence missing: {', '.join(missing)}") + + run_root = run_dir.resolve() + referenced_files = set() + for value in _artifact_references(manifest, results): + relative = Path(value) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError(f"release artifact path is unsafe: {value}") + target = run_dir / relative + if not target.exists() or not target.resolve().is_relative_to(run_root): + raise FileNotFoundError(f"release artifact missing or unsafe: {value}") + if target.is_file(): + referenced_files.add(target) + else: + referenced_files.update(path for path in target.rglob("*") if path.is_file()) + + files = set(required) | referenced_files + for name in ARTIFACT_ROOTS: + root = run_dir / name + if root.exists(): + files.update(path for path in root.rglob("*") if path.is_file()) + if any(not path.resolve().is_relative_to(run_root) for path in files): + raise ValueError("release artifact resolves outside the run directory") + return sorted(files, key=lambda path: path.relative_to(run_dir).as_posix()) + + def package_run(run_dir: Path, output: Path, platform: str) -> Path: manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8")) results = json.loads((run_dir / "results.json").read_text(encoding="utf-8")) @@ -36,10 +89,7 @@ def package_run(run_dir: Path, output: Path, platform: str) -> Path: if manifest["dataset_id"] != results["dataset_id"]: raise ValueError("release manifest and results dataset mismatch") - files = [run_dir / relative for relative in EVIDENCE_FILES] - missing = [str(path.relative_to(run_dir)) for path in files if not path.is_file()] - if missing: - raise FileNotFoundError(f"release evidence missing: {', '.join(missing)}") + files = _release_files(run_dir, manifest, results) output.mkdir(parents=True, exist_ok=True) name = f"data-format-lab-{results['profile']}-{_safe_slug(platform)}-{results['run_id']}" diff --git a/src/format_bench/report.py b/src/format_bench/report.py index 1c17e35..b9ac18a 100644 --- a/src/format_bench/report.py +++ b/src/format_bench/report.py @@ -87,6 +87,7 @@ def _fair(manifest: dict, results: dict) -> list[str]: evidence["warm"]["p95_ms"], evidence["warm"]["iqr_ms"], evidence["result"], + evidence["evidence"]["normalized_hash"], evidence["max_rss_bytes_p50"], ] ) @@ -96,7 +97,17 @@ def _fair(manifest: dict, results: dict) -> list[str]: "## Fair Operations", "", *_table( - ["Format", "Operation", "Fresh p50 ms", "Warm p50 ms", "Warm p95 ms", "IQR ms", "Rows", "RSS bytes"], + [ + "Format", + "Operation", + "Fresh p50 ms", + "Warm p50 ms", + "Warm p95 ms", + "IQR ms", + "Rows", + "Result hash", + "RSS bytes", + ], timings, ), ] @@ -130,6 +141,7 @@ def _prompt(results: dict) -> list[str]: name, item["payload_bytes"], item["taxonomy_bytes"], + item.get("schema_bytes", 0), item["total_bytes"], item["tokens"]["o200k_base"], item["tokens"]["cl100k_base"], @@ -151,7 +163,18 @@ def _prompt(results: dict) -> list[str]: return [ "## Prompt Corpus", "", - *_table(["Format", "Payload bytes", "Taxonomy bytes", "Total bytes", "o200k", "cl100k"], corpus), + *_table( + [ + "Format", + "Payload bytes", + "Taxonomy bytes", + "Schema bytes", + "Total bytes", + "o200k", + "cl100k", + ], + corpus, + ), "", "## Retrieval Payload", "", diff --git a/tests/test_release.py b/tests/test_release.py index cc5743f..e6e98b3 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -5,18 +5,27 @@ import zstandard as zstd -from format_bench.release import EVIDENCE_FILES, package_run +from format_bench.release import package_run def test_release_package_is_deterministic_and_relative(tmp_path: Path) -> None: run = tmp_path / "run" (run / "input").mkdir(parents=True) - manifest = {"state": "REPORTED", "dataset_id": "fixture"} + manifest = { + "state": "REPORTED", + "dataset_id": "fixture", + "formats": [{"artifact": "artifacts/value.bin"}], + } results = { "state": "REPORTED", "dataset_id": "fixture", "profile": "fair", "run_id": "run-1", + "results": { + "negative_research": { + "source_commits": {"artifact": "not-a-run-path"} + } + }, } payloads = { "manifest.json": json.dumps(manifest), @@ -26,6 +35,10 @@ def test_release_package_is_deterministic_and_relative(tmp_path: Path) -> None: } for relative, payload in payloads.items(): (run / relative).write_text(payload) + (run / "artifacts").mkdir() + (run / "artifacts" / "value.bin").write_bytes(b"artifact") + (run / "claims" / "nested").mkdir(parents=True) + (run / "claims" / "nested" / "evidence.bin").write_bytes(b"claim") first = package_run(run, tmp_path / "first", "linux-x86_64") second = package_run(run, tmp_path / "second", "linux-x86_64") @@ -35,5 +48,45 @@ def test_release_package_is_deterministic_and_relative(tmp_path: Path) -> None: tar_bytes = zstd.ZstdDecompressor().decompress(first.read_bytes()) with tarfile.open(fileobj=__import__("io").BytesIO(tar_bytes)) as archive: - assert archive.getnames() == [f"run-1/{name}" for name in EVIDENCE_FILES] + assert archive.getnames() == [ + "run-1/artifacts/value.bin", + "run-1/claims/nested/evidence.bin", + "run-1/input/manifest.json", + "run-1/manifest.json", + "run-1/report.md", + "run-1/results.json", + ] assert all(not Path(name).is_absolute() for name in archive.getnames()) + + +def test_release_rejects_missing_referenced_artifact(tmp_path: Path) -> None: + run = tmp_path / "run" + (run / "input").mkdir(parents=True) + (run / "manifest.json").write_text( + json.dumps( + { + "state": "REPORTED", + "dataset_id": "fixture", + "formats": [{"artifact": "artifacts/missing.bin"}], + } + ) + ) + (run / "results.json").write_text( + json.dumps( + { + "state": "REPORTED", + "dataset_id": "fixture", + "profile": "fair", + "run_id": "run-1", + } + ) + ) + (run / "report.md").write_text("# report\n") + (run / "input" / "manifest.json").write_text('{}\n') + + try: + package_run(run, tmp_path / "output", "linux-x86_64") + except FileNotFoundError as error: + assert "artifacts/missing.bin" in str(error) + else: + raise AssertionError("missing artifact was accepted") diff --git a/tests/test_report.py b/tests/test_report.py index 3493594..720fed2 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -27,7 +27,8 @@ def test_prompt_report_is_deterministic_and_includes_exact_tokens(tmp_path: Path "compact_tsv": { "payload_bytes": 10, "taxonomy_bytes": 4, - "total_bytes": 14, + "schema_bytes": 2, + "total_bytes": 16, "tokens": {"o200k_base": 3, "cl100k_base": 4}, } }, @@ -46,7 +47,7 @@ def test_prompt_report_is_deterministic_and_includes_exact_tokens(tmp_path: Path (tmp_path / "results.json").write_text(json.dumps(results)) path = render_report(tmp_path) first = path.read_text() - assert "| compact_tsv | 10 | 4 | 14 | 3 | 4 |" in first + assert "| compact_tsv | 10 | 4 | 2 | 16 | 3 | 4 |" in first assert "Direct token counts for binary formats are N/A." in first assert json.loads((tmp_path / "manifest.json").read_text())["state"] == "REPORTED" assert json.loads((tmp_path / "results.json").read_text())["state"] == "REPORTED" @@ -58,3 +59,49 @@ def test_report_rejects_unbenchmarked_evidence(tmp_path: Path) -> None: (tmp_path / "results.json").write_text('{"state":"BENCHMARKED"}') with pytest.raises(ValueError, match="requires benchmarked"): render_report(tmp_path) + + +def test_fair_report_includes_normalized_result_hash(tmp_path: Path) -> None: + manifest = { + "state": "BENCHMARKED", + "dataset_id": "fixture", + "rankable": True, + "formats": [ + { + "format": "csv", + "comparability": "FULL_COMPARABLE", + "state": "BENCHMARKED", + "native_bytes": 10, + "transport_zstd_bytes": 8, + } + ], + } + results = { + "state": "BENCHMARKED", + "dataset_id": "fixture", + "run_id": "fair-fixture", + "profile": "fair", + "environment": { + "git_commit": "abc", + "flake_lock_sha256": "def", + "platform": "test-os", + "machine": "test-cpu", + "python": "3.12.0", + }, + "results": { + "csv/read_all": { + "status": "MEASURED", + "fresh_process": {"p50_ms": 1}, + "warm": {"p50_ms": 1, "p95_ms": 2, "iqr_ms": 1}, + "result": 4, + "evidence": {"normalized_hash": "abc123"}, + "max_rss_bytes_p50": 100, + } + }, + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + (tmp_path / "results.json").write_text(json.dumps(results)) + + report = render_report(tmp_path).read_text() + + assert "| csv | read_all | 1 | 1 | 2 | 1 | 4 | abc123 | 100 |" in report