Allow terminal artifact absence

src/format_bench/release.py

@@ -27,9 +27,10 @@ def _safe_slug(value: str) -> str:
     return value
 
 
-def _artifact_references(manifest: dict, results: dict) -> list[str]:
+def _artifact_references(manifest: dict, results: dict) -> list[tuple[str, bool]]:
+    terminal_failures = {ExecutionState.FAILED, ExecutionState.UNSUPPORTED}
     references = [
-        entry["artifact"]
+        (entry["artifact"], entry.get("state") not in terminal_failures)
         for entry in manifest.get("formats", [])
         if isinstance(entry.get("artifact"), str)
     ]
@@ -41,35 +42,41 @@ def _artifact_references(manifest: dict, results: dict) -> list[str]:
             if not isinstance(source, dict):
                 continue
             if isinstance(source.get("artifact"), str):
-                references.append(source["artifact"])
+                references.append((source["artifact"], True))
             if isinstance(source.get("artifacts"), dict):
                 references.extend(
-                    item for item in source["artifacts"].values() if isinstance(item, str)
+                    (item, True)
+                    for item in source["artifacts"].values()
+                    if isinstance(item, str)
                 )
     return references
 
 
 def _release_files(run_dir: Path, manifest: dict, results: dict) -> list[Path]:
-    required = [run_dir / relative for relative in EVIDENCE_FILES]
-    missing = [str(path.relative_to(run_dir)) for path in required if not path.is_file()]
+    required_files = [run_dir / relative for relative in EVIDENCE_FILES]
+    missing = [
+        str(path.relative_to(run_dir)) for path in required_files if not path.is_file()
+    ]
     if missing:
         raise FileNotFoundError(f"release evidence missing: {', '.join(missing)}")
 
     run_root = run_dir.resolve()
     referenced_files = set()
-    for value in _artifact_references(manifest, results):
+    for value, must_exist in _artifact_references(manifest, results):
         relative = Path(value)
         if relative.is_absolute() or ".." in relative.parts:
             raise ValueError(f"release artifact path is unsafe: {value}")
         target = run_dir / relative
+        if not target.exists() and not must_exist:
+            continue
         if not target.exists() or not target.resolve().is_relative_to(run_root):
             raise FileNotFoundError(f"release artifact missing or unsafe: {value}")
         if target.is_file():
             referenced_files.add(target)
         else:
             referenced_files.update(path for path in target.rglob("*") if path.is_file())
 
-    files = set(required) | referenced_files
+    files = set(required_files) | referenced_files
     for name in ARTIFACT_ROOTS:
         root = run_dir / name
         if root.exists():

tests/test_release.py

@@ -3,6 +3,7 @@
 import tarfile
 from pathlib import Path
 
+import pytest
 import zstandard as zstd
 
 from format_bench.release import package_run
@@ -67,7 +68,9 @@ def test_release_rejects_missing_referenced_artifact(tmp_path: Path) -> None:
             {
                 "state": "REPORTED",
                 "dataset_id": "fixture",
-                "formats": [{"artifact": "artifacts/missing.bin"}],
+                "formats": [
+                    {"artifact": "artifacts/missing.bin", "state": "BENCHMARKED"}
+                ],
             }
         )
     )
@@ -90,3 +93,36 @@ def test_release_rejects_missing_referenced_artifact(tmp_path: Path) -> None:
         assert "artifacts/missing.bin" in str(error)
     else:
         raise AssertionError("missing artifact was accepted")
+
+
+@pytest.mark.parametrize("state", ["FAILED", "UNSUPPORTED"])
+def test_release_allows_missing_terminal_format_artifact(
+    tmp_path: Path, state: str
+) -> None:
+    run = tmp_path / state.lower()
+    (run / "input").mkdir(parents=True)
+    (run / "manifest.json").write_text(
+        json.dumps(
+            {
+                "state": "REPORTED",
+                "dataset_id": "fixture",
+                "formats": [{"artifact": "artifacts/missing.bin", "state": state}],
+            }
+        )
+    )
+    (run / "results.json").write_text(
+        json.dumps(
+            {
+                "state": "REPORTED",
+                "dataset_id": "fixture",
+                "profile": "fair",
+                "run_id": state.lower(),
+            }
+        )
+    )
+    (run / "report.md").write_text("# report\n")
+    (run / "input" / "manifest.json").write_text('{}\n')
+
+    archive = package_run(run, tmp_path / "output", "linux-x86_64")
+
+    assert archive.is_file()