Supersede the legacy promotion to retire a stale expected failure

.github/workflows/freeze.yml

@@ -141,6 +141,7 @@ jobs:
           cargo run --locked -- validate benchmarks/cases
           cargo run --locked -- validate-evaluation benchmarks/cases/evaluation/real-project-v2
           cargo run --locked -- validate-legacy-promotion benchmarks/promotion/legacy-v1/manifest.json
+          cargo run --locked -- validate-legacy-promotion benchmarks/promotion/legacy-v2/manifest.json
           bash scripts/validate-reproduction-contract.sh
           python3 -m unittest tests/test_freeze_shards.py
       - uses: citation-file-format/cffconvert-github-action@4cf11baa70a673bfdf9dad0acc7ee33b3f4b6084 # 2.0.0

RELEASES.md

@@ -143,9 +143,13 @@ estimates, cross-language ranking, causal defect claims, and latency, memory,
 or cold-start claims.
 
 The `legacy-promoted` path is a separate retrospective release contract. It
-is bound to `benchmarks/promotion/legacy-v1/manifest.json`, stages an
-execution-only corpus containing exactly its 110 `balanced_core` IDs across
-30 documents, and keeps overflow and control cases outside the run. Two
+is bound to the current promotion manifest,
+`benchmarks/promotion/legacy-v2/manifest.json`, stages an execution-only
+corpus containing exactly its 110 `balanced_core` IDs across 30 documents, and
+keeps overflow and control cases outside the run. Superseded manifests stay in
+the tree and stay valid: corrections are append-only, so a later manifest is a
+new `legacy-vN` directory carrying a hash-bound `supersedes` link rather than
+an edit to its predecessor. Two
 checksum-bound canonical reference reports (Bifrost and gopls) run on Ubuntu;
 the nine remaining advertised profiles run one at a time on the selected
 native macOS runner. Candidate reports remain language-scoped; the frozen

benchmarks/promotion/legacy-v2/retirements/cpp-parity-function-like-macro-expanded-call.json

@@ -0,0 +1,28 @@
+{
+  "caseId": "cpp-parity-function-like-macro-expanded-call",
+  "caseFile": "benchmarks/cases/cpp-lsp-parity.yaml",
+  "supersededReason": "Bifrost finds the PARITY_CALL function-name argument as a line-level usage, but reverse navigation from that macro argument does not resolve to the expanded_label definition.",
+  "finding": "Reverse navigation from the macro argument now resolves to the authored declaration, so the annotation describes a defect the analyzer no longer has.",
+  "observedIn": {
+    "release": "v0.3.1",
+    "releaseTarball": "usagebench-v0.3.1.tar.gz",
+    "releaseTarballSha256": "0ad2e02017b064066bbc674b5d183d24a5f2b2fd80083094db3de6905bc9bb60",
+    "report": "evidence/bifrost.json",
+    "reportSha256": "593cc3580d5a8e97ca6d3852629c181f0d362eff1f9ae17e3d14c5a43e99cd7a",
+    "executionMode": "container",
+    "platformScope": "canonical_reference",
+    "referenceEnvironmentDigest": "sha256:3859af74db43f64499f05045bad3e8962471203b25309c31f3ea225631772c5c"
+  },
+  "observation": {
+    "status": "improved",
+    "declarationToUsages": "passed",
+    "usageToDeclaration": "passed",
+    "queries": 2,
+    "successfulQueries": 2,
+    "exactSetQueries": 2,
+    "falsePositives": 0,
+    "falseNegatives": 0,
+    "resolvedDeclaration": "include/parity.h:45:20"
+  },
+  "notes": "The frozen v0.3.1 legacy snapshot is the reference-container run the retirement rests on, so no fresh native measurement is relied upon. `improved` is precisely the runner's signal for a case that passes while annotated expectedFailure. Retirement removes the annotation from the execution-only staged corpus; benchmarks/cases/cpp-lsp-parity.yaml stays byte-identical and legacy-promotion-v1-balanced-core continues to validate against it."
+}

benchmarks/review-protocol/README.md

@@ -61,10 +61,12 @@ status and never changes this source-derived contract.
 
 The completed review contains milestones 1 through 10: 110 balanced-core
 cases, 220 fresh provider-native sessions, mechanical comparison, and accountable
-human adjudication. The final hash-bound
+human adjudication. The hash-bound
 `benchmarks/promotion/legacy-v1/manifest.json` publishes that complete cohort as
 the corpus-bounded `legacy_promoted` tier without changing the immutable
 `N = 10` denominator or its retrospective selection provenance.
+`benchmarks/promotion/legacy-v2/manifest.json` supersedes it, carrying the same
+reviewed membership and retiring one stale `expectedFailure` annotation.
 
 Validate each adjudicated milestone with:
 
@@ -78,4 +80,5 @@ Validate the complete promotion manifest with:
 
 ```bash
 cargo run -- validate-legacy-promotion benchmarks/promotion/legacy-v1/manifest.json
+cargo run -- validate-legacy-promotion benchmarks/promotion/legacy-v2/manifest.json
 ```

docs/legacy-promotion-manifest.md

@@ -32,3 +32,37 @@ reviewed entry in a later manifest and cannot change an already frozen N.
 Allowed wording is “reviewed conformance on the named promoted legacy corpus.”
 The tier does not support claims of preregistration, general accuracy,
 language-wide accuracy, ecosystem coverage, or analyzer superiority.
+
+## Retiring a stale expected failure
+
+`expectedFailure` in a historical case document records an analyzer outcome,
+not reviewed ground truth, so it can outlive the defect it describes. The
+runner already surfaces that: a case that passes while annotated is reported
+`improved` rather than counted as a pass.
+
+The annotation cannot be edited away. The document is content-addressed by
+every manifest that binds it, and history is not rewritten. A superseding
+manifest retires it instead, through `retiredExpectedFailure` on the case
+entry:
+
+- `supersededReason` repeats the annotation verbatim, so the retirement is a
+  record of what was withdrawn rather than a deletion, and cannot drift from
+  the frozen text.
+- `evidence` binds the artifact showing the annotated navigation now succeeds.
+- The manifest must carry `supersedes`. A retirement is a correction, and
+  corrections are append-only.
+
+Execution staging drops the annotation from its filtered, execution-only copy
+of the corpus, so the case is scored as an ordinary pass. The historical YAML
+stays byte-identical and every earlier manifest keeps validating against it.
+
+Retirement is one-way and only ever tightens the corpus: it removes an excuse
+and can never add one. A superseding manifest may retire more expectations, but
+may not restore one its predecessor retired — the validator rejects that,
+so a later manifest cannot quietly reinstate an excuse for a case already held
+to an ordinary pass.
+
+`legacy-promotion-v2-balanced-core` supersedes
+`legacy-promotion-v1-balanced-core` on exactly these terms. It carries the same
+110 reviewed balanced-core cases and retires one annotation, on
+`cpp-parity-function-like-macro-expanded-call`.

schema/legacy-promotion.schema.json

@@ -31,6 +31,7 @@
   "$defs": {
     "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
     "artifactLink": { "type": "object", "additionalProperties": false, "required": ["file", "sha256"], "properties": { "file": { "type": "string", "minLength": 1 }, "sha256": { "$ref": "#/$defs/sha256" } } },
+    "retiredExpectedFailure": { "type": "object", "additionalProperties": false, "required": ["supersededReason", "evidence"], "properties": { "supersededReason": { "type": "string", "minLength": 1 }, "evidence": { "$ref": "#/$defs/artifactLink" } } },
     "document": {
       "type": "object", "additionalProperties": false, "required": ["caseFile", "sourceSha256", "language", "cases"],
       "properties": {
@@ -46,7 +47,8 @@
         "controlStatus": { "enum": ["unsupported", "not_planned"] },
         "strata": { "type": "object", "minProperties": 6, "additionalProperties": { "type": "string", "minLength": 1 } },
         "reviewRecords": { "type": "array", "minItems": 2, "uniqueItems": true, "items": { "$ref": "#/$defs/artifactLink" } },
-        "adjudication": { "$ref": "#/$defs/artifactLink" }
+        "adjudication": { "$ref": "#/$defs/artifactLink" },
+        "retiredExpectedFailure": { "$ref": "#/$defs/retiredExpectedFailure" }
       },
       "allOf": [
         { "if": { "properties": { "membership": { "const": "control" } }, "required": ["membership"] }, "then": { "required": ["controlStatus"] }, "else": { "not": { "required": ["controlStatus"] } } }

scripts/generate-docs-evidence.py

@@ -394,6 +394,10 @@ def development_case_ids() -> list[str]:
 
 
 def build_legacy(bundles: dict[str, dict[str, Any]]) -> dict[str, Any]:
+    # The published legacy result is the authority for this slice, so the map
+    # describes the manifest that snapshot was frozen under, not the one the
+    # next freeze will use. Move this forward together with the legacy release
+    # that supersedes it.
     manifest_path = "benchmarks/promotion/legacy-v1/manifest.json"
     cohort_path = "benchmarks/promotion/legacy-v1/cohort.json"
     manifest = read_json(manifest_path)

scripts/resolve-freeze-scope.sh

@@ -40,7 +40,7 @@ case "$snapshot_kind" in
       '{casePath: $case_path, candidates: $candidates}'
     ;;
   legacy-promoted)
-    promotion_manifest="benchmarks/promotion/legacy-v1/manifest.json"
+    promotion_manifest="benchmarks/promotion/legacy-v2/manifest.json"
     promotion_path="$repo_root/$promotion_manifest"
     [[ -f "$promotion_path" ]] || {
       echo "legacy promotion manifest is missing: $promotion_manifest" >&2

scripts/run-freeze-candidates.sh

@@ -194,9 +194,12 @@ freeze_args=(
 if [[ "$SNAPSHOT_KIND" == "evaluation" ]]; then
   freeze_args+=(--evaluation-corpus "$corpus_root/$case_path")
 elif [[ "$SNAPSHOT_KIND" == "legacy-promoted" ]]; then
-  promotion_manifest="${FREEZE_PROMOTION_MANIFEST:-benchmarks/promotion/legacy-v1/manifest.json}"
-  [[ "$promotion_manifest" == "benchmarks/promotion/legacy-v1/manifest.json" ]] || {
-    echo "legacy-promoted freezes are bound to benchmarks/promotion/legacy-v1/manifest.json" >&2
+  promotion_manifest="${FREEZE_PROMOTION_MANIFEST:-benchmarks/promotion/legacy-v2/manifest.json}"
+  # Corrections are append-only, so a superseding manifest is a new legacy-vN
+  # directory rather than an edit. The rail keeps freezes bound to a promotion
+  # manifest under the promotion root.
+  [[ "$promotion_manifest" =~ ^benchmarks/promotion/legacy-v[0-9]+/manifest\.json$ ]] || {
+    echo "legacy-promoted freezes are bound to benchmarks/promotion/legacy-vN/manifest.json" >&2
     exit 1
   }
   [[ -f "$corpus_root/$promotion_manifest" ]] || {

scripts/stage-legacy-promotion-corpus.py

@@ -21,13 +21,17 @@
 
 CASE_ITEM = re.compile(r"^  - id: ([A-Za-z0-9][A-Za-z0-9_-]*)\s*$", re.MULTILINE)
 EXPECTED_CASE_COUNT = 110
+CASE_KEY = re.compile(r"^    (?=\S)", re.MULTILINE)
+EXPECTED_FAILURE_KEY = "    expectedFailure:"
 
 
 def fail(message: str) -> None:
     raise SystemExit(message)
 
 
-def balanced_case_ids(manifest_path: Path) -> tuple[dict[str, set[str]], dict[str, str]]:
+def balanced_case_ids(
+    manifest_path: Path,
+) -> tuple[dict[str, set[str]], dict[str, str], set[str]]:
     try:
         manifest = json.loads(manifest_path.read_text())
     except (OSError, UnicodeError, json.JSONDecodeError) as error:
@@ -38,6 +42,7 @@ def balanced_case_ids(manifest_path: Path) -> tuple[dict[str, set[str]], dict[st
 
     by_file: dict[str, set[str]] = {}
     source_hashes: dict[str, str] = {}
+    retired: set[str] = set()
     all_ids: set[str] = set()
     for document in documents:
         if not isinstance(document, dict):
@@ -74,6 +79,8 @@ def balanced_case_ids(manifest_path: Path) -> tuple[dict[str, set[str]], dict[st
                 fail(f"legacy promotion case ID is duplicated: {case_id}")
             selected.add(case_id)
             all_ids.add(case_id)
+            if "retiredExpectedFailure" in case:
+                retired.add(case_id)
         if not selected:
             fail(f"legacy promotion document has no balanced_core cases: {case_file}")
         if case_file in by_file:
@@ -86,10 +93,39 @@ def balanced_case_ids(manifest_path: Path) -> tuple[dict[str, set[str]], dict[st
             "legacy promotion execution requires exactly "
             f"{EXPECTED_CASE_COUNT} balanced_core cases, got {len(all_ids)}"
         )
-    return by_file, source_hashes
+    return by_file, source_hashes, retired
 
 
-def filter_document(source: Path, destination: Path, expected_ids: set[str]) -> None:
+def retire_expected_failure(block: str, case_id: str) -> str:
+    """Drop the ``expectedFailure`` mapping from one staged case block.
+
+    The historical document is content-addressed and is never edited.  A
+    promotion manifest that retires the annotation removes it from this
+    execution-only copy, so the case is scored as an ordinary pass instead of
+    reporting ``improved`` against an expectation it no longer fails.
+    """
+
+    keys = [match.start() for match in CASE_KEY.finditer(block)]
+    starts = [start for start in keys if block.startswith(EXPECTED_FAILURE_KEY, start)]
+    if len(starts) != 1:
+        fail(
+            f"case {case_id} does not author exactly one expectedFailure to retire: "
+            f"found {len(starts)}"
+        )
+    start = starts[0]
+    following = [key for key in keys if key > start]
+    end = following[0] if following else len(block)
+    retired = block[:start] + block[end:]
+    if EXPECTED_FAILURE_KEY in retired:
+        fail(f"retiring the expectedFailure for {case_id} left the annotation behind")
+    if CASE_ITEM.search(block[start:end]):
+        fail(f"retiring the expectedFailure for {case_id} would remove another case")
+    return retired
+
+
+def filter_document(
+    source: Path, destination: Path, expected_ids: set[str], retired_ids: set[str]
+) -> None:
     text = source.read_text()
     matches = list(CASE_ITEM.finditer(text))
     if not matches:
@@ -103,11 +139,16 @@ def filter_document(source: Path, destination: Path, expected_ids: set[str]) ->
             f"expected at least {sorted(expected_ids)}, got {sorted(actual_ids)}"
         )
     header = text[: matches[0].start()]
-    blocks = [
-        text[match.start() : (matches[index + 1].start() if index + 1 < len(matches) else len(text))]
-        for index, match in enumerate(matches)
-        if match.group(1) in expected_ids
-    ]
+    blocks = []
+    for index, match in enumerate(matches):
+        case_id = match.group(1)
+        if case_id not in expected_ids:
+            continue
+        end = matches[index + 1].start() if index + 1 < len(matches) else len(text)
+        block = text[match.start() : end]
+        if case_id in retired_ids:
+            block = retire_expected_failure(block, case_id)
+        blocks.append(block)
     destination.parent.mkdir(parents=True, exist_ok=True)
     destination.write_text(header + "".join(blocks))
     staged_ids = [match.group(1) for match in CASE_ITEM.finditer(destination.read_text())]
@@ -146,10 +187,16 @@ def main() -> int:
         relative_manifest = manifest.relative_to(source_root)
     except ValueError:
         fail("promotion manifest must be inside the source corpus root")
-    if relative_manifest.as_posix() != "benchmarks/promotion/legacy-v1/manifest.json":
-        fail("legacy execution is bound to benchmarks/promotion/legacy-v1/manifest.json")
-
-    selected, source_hashes = balanced_case_ids(manifest)
+    # Corrections to the promotion tier are append-only, so a superseding
+    # manifest lives beside its predecessor under a new legacy-vN directory.
+    # The rail stays: execution is bound to a promotion manifest, never to an
+    # arbitrary path.
+    if not re.fullmatch(
+        r"benchmarks/promotion/legacy-v[0-9]+/manifest\.json", relative_manifest.as_posix()
+    ):
+        fail("legacy execution is bound to benchmarks/promotion/legacy-vN/manifest.json")
+
+    selected, source_hashes, retired = balanced_case_ids(manifest)
     shutil.copytree(source_root, destination, symlinks=False, dirs_exist_ok=True)
     case_root = destination / "benchmarks/cases"
     selected_files = set(selected)
@@ -167,7 +214,7 @@ def main() -> int:
         source_digest = hashlib.sha256(source.read_bytes()).hexdigest()
         if source_digest != source_hashes[case_file]:
             fail(f"legacy promotion source hash does not match manifest: {case_file}")
-        filter_document(source, target, case_ids)
+        filter_document(source, target, case_ids, retired)
     staged_files = {
         path.relative_to(destination).as_posix()
         for path in case_root.rglob("*.yaml")

scripts/validate-reproduction-contract.sh

@@ -50,6 +50,7 @@ python3 "$repo_root/scripts/build-real-project-v1-publication-review.py" --check
 python3 "$repo_root/scripts/build-real-project-v2-publication-review.py" --check
 python3 -m unittest "$repo_root/tests/test_publication_bundle.py"
 cargo run --locked -- validate-legacy-promotion "$repo_root/benchmarks/promotion/legacy-v1/manifest.json"
+cargo run --locked -- validate-legacy-promotion "$repo_root/benchmarks/promotion/legacy-v2/manifest.json"
 
 jq -e '.schemaVersion == 3' "$registry" >/dev/null
 jq -e '.schemaVersion == 3' "$v030_registry" >/dev/null
@@ -144,15 +145,15 @@ development_candidates="$(jq -c '.candidates' <<< "$development_scope")"
   echo "legacy-promoted case scope does not use the development case corpus" >&2
   exit 1
 }
-[[ "$(jq -r '.promotionManifest' <<< "$legacy_scope")" == "benchmarks/promotion/legacy-v1/manifest.json" ]] || {
-  echo "legacy-promoted scope is not bound to the v1 promotion manifest" >&2
+[[ "$(jq -r '.promotionManifest' <<< "$legacy_scope")" == "benchmarks/promotion/legacy-v2/manifest.json" ]] || {
+  echo "legacy-promoted scope is not bound to the current promotion manifest" >&2
   exit 1
 }
-[[ "$(jq -r '[.documents[].cases[] | select(.membership == "balanced_core")] | length' benchmarks/promotion/legacy-v1/manifest.json)" == "110" ]] || {
+[[ "$(jq -r '[.documents[].cases[] | select(.membership == "balanced_core")] | length' benchmarks/promotion/legacy-v2/manifest.json)" == "110" ]] || {
   echo "legacy promotion manifest does not contain exactly 110 balanced-core cases" >&2
   exit 1
 }
-[[ "$(jq -r '[.documents[].cases[] | select(.membership != "balanced_core")] | length' benchmarks/promotion/legacy-v1/manifest.json)" == "0" ]] || {
+[[ "$(jq -r '[.documents[].cases[] | select(.membership != "balanced_core")] | length' benchmarks/promotion/legacy-v2/manifest.json)" == "0" ]] || {
   echo "legacy freeze manifest contains overflow or control cases" >&2
   exit 1
 }

src/main.rs

@@ -290,6 +290,12 @@ fn main() -> Result<()> {
                 "validated legacy promotion {} with {} balanced-core cases",
                 audit.promotion_id, audit.balanced_core_case_count
             );
+            if !audit.retired_expected_failures.is_empty() {
+                println!(
+                    "retired expected failures: {}",
+                    audit.retired_expected_failures.join(", ")
+                );
+            }
         }
         Command::GenerateLegacyPromotionCohort { cases, output } => {
             let cohort = usagebench::promotion_cohort::generate(&cases, &output)?;