Promote C++ macro benchmark to expected pass

.github/workflows/benchmark.yml

@@ -105,6 +105,7 @@ jobs:
             --bifrost-working-tree
             --work-dir target/usagebench
             --scan-usages-max-duration-secs 300
+            --expected-passes benchmarks/expectations/bifrost-expected-passes.yaml
             --output "benchmark-output/run-${GITHUB_RUN_ID}.json"
           )
           if [ "$INCLUDE_UNSUPPORTED" = "true" ]; then

README.md

@@ -274,7 +274,8 @@ The workflow:
 * validates `benchmarks/cases`
 * checks out `BrokkAi/bifrost`
 * builds `usagebench`
-* runs `usagebench run-bifrost benchmarks/cases`
+* runs `usagebench run-bifrost benchmarks/cases` with the versioned current
+  expected-pass overlay
 * uploads the JSON report from `benchmark-output`
 * publishes a GitHub step summary
 * optionally posts a payload to Slack

benchmarks/README.md

@@ -218,6 +218,10 @@ Each case supports both benchmark directions:
 - `expectedFailure.reason` keeps a known analyzer gap in the baseline while
   still running the case and reporting it as improved if it unexpectedly starts
   passing.
+- `benchmarks/expectations/bifrost-expected-passes.yaml` can promote a frozen
+  historical `expectedFailure` to a current Bifrost required pass without
+  mutating content-addressed legacy benchmark documents. A matching case must
+  pass; any regression is reported as a normal failure.
 - `notPlanned.reason` keeps runtime-dynamic or generated-code expectations in
   the corpus and runs them without including them in the planned-case total.
 - `unsupported.reason` documents out-of-boundary cases and reports them as

benchmarks/expectations/bifrost-expected-passes.yaml

@@ -0,0 +1,6 @@
+schemaVersion: 1
+expectedPasses:
+  - caseId: cpp-parity-function-like-macro-expanded-call
+    reason: >
+      Promoted after scheduled runs 32240334240 and 32356646488 passed, then a
+      focused replay passed exactly against Bifrost 79ea68cf5e2e7cda9cd90de4188836bb02fe0d6c.

src/evaluation.rs

@@ -4124,6 +4124,7 @@ mod tests {
                 }],
                 type_lookups: Vec::new(),
                 expected_failure: None,
+                expected_pass_reason: None,
                 not_planned: None,
                 unsupported: None,
                 verification: None,

src/lib.rs

@@ -194,6 +194,11 @@ pub struct BenchmarkCase {
     pub type_lookups: Vec<TypeLookup>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub expected_failure: Option<ExpectedFailure>,
+    /// Current-run expectation supplied by the Bifrost expected-pass overlay.
+    /// This is intentionally not part of an authored benchmark document: some
+    /// legacy documents are content-addressed by frozen promotion evidence.
+    #[serde(skip)]
+    pub expected_pass_reason: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub not_planned: Option<NotPlannedReason>,
     #[serde(default, skip_serializing_if = "Option::is_none")]

src/main.rs

@@ -197,6 +197,9 @@ enum Command {
         /// Run only benchmark documents for this language.
         #[arg(long)]
         language: Option<String>,
+        /// Versioned overlay that promotes historical expected failures to current required passes.
+        #[arg(long)]
+        expected_passes: Option<PathBuf>,
     },
     /// Run benchmark cases against a versioned language-server profile.
     RunLsp {
@@ -448,6 +451,7 @@ fn main() -> Result<()> {
             keep_worktrees,
             case_id,
             language,
+            expected_passes,
         } => {
             let mut options = RunBifrostOptions::with_defaults(path);
             options.bifrost_repo = bifrost_repo;
@@ -462,6 +466,7 @@ fn main() -> Result<()> {
             options.keep_worktrees = keep_worktrees;
             options.case_id = case_id;
             options.language = language;
+            options.expected_passes = expected_passes;
             let report = run_bifrost(options)?;
             println!(
                 "ran {} planned case(s) ({} development, {} evaluation): {} passed, {} near miss(es), {} position-unverified, {} improved, {} failed, {} expected failure(s), {} not planned, {} unsupported, {} skipped, {} error(s)",

src/promotion.rs

@@ -720,6 +720,7 @@ mod tests {
                 required_destination_status: None,
                 location_metrics: None,
                 expected_failure_reason: None,
+                expected_pass_reason: None,
                 not_planned_reason: None,
                 unsupported_reason: None,
                 declaration_to_usages: None,

src/publication.rs

@@ -1171,6 +1171,7 @@ mod tests {
                 required_destination_status: Some(RequiredDestinationStatus::Found),
                 location_metrics: None,
                 expected_failure_reason: None,
+                expected_pass_reason: None,
                 not_planned_reason: None,
                 unsupported_reason: None,
                 declaration_to_usages: None,
@@ -1185,6 +1186,7 @@ mod tests {
                 required_destination_status: Some(RequiredDestinationStatus::Unsupported),
                 location_metrics: None,
                 expected_failure_reason: None,
+                expected_pass_reason: None,
                 not_planned_reason: None,
                 unsupported_reason: Some("unsupported".into()),
                 declaration_to_usages: None,

src/results.rs

@@ -2233,6 +2233,7 @@ mod tests {
                                 },
                             }),
                             expected_failure_reason: None,
+                            expected_pass_reason: None,
                             not_planned_reason: None,
                             unsupported_reason: None,
                             declaration_to_usages: None,

src/runners/bifrost.rs

@@ -59,6 +59,7 @@ pub struct RunBifrostOptions {
     pub keep_worktrees: bool,
     pub case_id: Option<String>,
     pub language: Option<String>,
+    pub expected_passes: Option<PathBuf>,
 }
 
 impl RunBifrostOptions {
@@ -78,10 +79,25 @@ impl RunBifrostOptions {
             keep_worktrees: false,
             case_id: None,
             language: None,
+            expected_passes: None,
         }
     }
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct ExpectedPassManifest {
+    schema_version: u32,
+    expected_passes: Vec<ExpectedPass>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct ExpectedPass {
+    case_id: String,
+    reason: String,
+}
+
 pub type BifrostRunReport = RunReport;
 
 pub fn generated_bifrost_report_schema_json() -> Result<String> {
@@ -105,7 +121,7 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result<BifrostRunReport> {
     let repo_root = find_repo_root_for_path(&options.case_path)?;
     let usagebench_provenance = resolve_usagebench_provenance(&repo_root)?;
     let case_files = crate::validate_path(&options.case_path)?;
-    let benchmark_documents = case_files
+    let mut benchmark_documents = case_files
         .iter()
         .map(|case_file| {
             let yaml = fs::read_to_string(case_file)
@@ -115,6 +131,11 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result<BifrostRunReport> {
             Ok((case_file.clone(), document))
         })
         .collect::<Result<Vec<_>>>()?;
+    apply_expected_passes(
+        &mut benchmark_documents,
+        &repo_root,
+        options.expected_passes.as_deref(),
+    )?;
     let requested_totals = requested_run_totals(
         benchmark_documents
             .iter()
@@ -382,6 +403,85 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result<BifrostRunReport> {
     Ok(report)
 }
 
+fn apply_expected_passes(
+    documents: &mut [(PathBuf, BenchmarkDocument)],
+    repo_root: &Path,
+    manifest_path: Option<&Path>,
+) -> Result<()> {
+    let Some(manifest_path) = manifest_path else {
+        return Ok(());
+    };
+    let manifest_path = if manifest_path.is_absolute() {
+        manifest_path.to_path_buf()
+    } else {
+        repo_root.join(manifest_path)
+    };
+    let source = fs::read_to_string(&manifest_path)
+        .with_context(|| format!("read expected-pass overlay {}", manifest_path.display()))?;
+    let manifest = serde_yaml::from_str::<ExpectedPassManifest>(&source)
+        .with_context(|| format!("parse expected-pass overlay {}", manifest_path.display()))?;
+    if manifest.schema_version != 1 {
+        bail!(
+            "expected-pass overlay {} has unsupported schemaVersion {}",
+            manifest_path.display(),
+            manifest.schema_version
+        );
+    }
+
+    let mut expected_passes = BTreeMap::new();
+    for expected_pass in manifest.expected_passes {
+        if expected_pass.case_id.trim().is_empty() || expected_pass.reason.trim().is_empty() {
+            bail!(
+                "expected-pass overlay {} requires non-empty caseId and reason",
+                manifest_path.display()
+            );
+        }
+        if expected_passes
+            .insert(expected_pass.case_id.clone(), expected_pass.reason)
+            .is_some()
+        {
+            bail!(
+                "expected-pass overlay {} repeats caseId {}",
+                manifest_path.display(),
+                expected_pass.case_id
+            );
+        }
+    }
+
+    let mut applied = BTreeSet::new();
+    for (_, document) in documents {
+        for case in &mut document.cases {
+            let Some(reason) = expected_passes.get(&case.id) else {
+                continue;
+            };
+            if case.expected_failure.is_none() {
+                bail!(
+                    "expected-pass overlay {} may only promote an authored expectedFailure: {}",
+                    manifest_path.display(),
+                    case.id
+                );
+            }
+            case.expected_failure = None;
+            case.expected_pass_reason = Some(reason.clone());
+            applied.insert(case.id.clone());
+        }
+    }
+
+    let missing = expected_passes
+        .keys()
+        .filter(|case_id| !applied.contains(*case_id))
+        .cloned()
+        .collect::<Vec<_>>();
+    if !missing.is_empty() {
+        bail!(
+            "expected-pass overlay {} references case IDs absent from this run: {}",
+            manifest_path.display(),
+            missing.join(", ")
+        );
+    }
+    Ok(())
+}
+
 fn document_failure_cases(
     document: &BenchmarkDocument,
     include_unsupported: bool,
@@ -867,6 +967,13 @@ fn run_case_with_scan_duration(
         .expected_failure
         .as_ref()
         .map(|expected_failure| expected_failure.reason.clone());
+    let expected_pass_reason = case.expected_pass_reason.clone();
+    if let Some(reason) = &expected_pass_reason {
+        diagnostics.push(RunDiagnostic {
+            kind: "expected_pass_override".to_string(),
+            message: format!("current Bifrost expected-pass overlay applied: {reason}"),
+        });
+    }
     let not_planned_reason = case
         .not_planned
         .as_ref()
@@ -898,6 +1005,7 @@ fn run_case_with_scan_duration(
         id: case.id.clone(),
         status,
         expected_failure_reason,
+        expected_pass_reason,
         not_planned_reason,
         unsupported_reason: case
             .unsupported
@@ -3148,6 +3256,7 @@ for line in sys.stdin:
                     status: CaseStatus::Passed,
                     location_metrics: None,
                     expected_failure_reason: None,
+                    expected_pass_reason: None,
                     not_planned_reason: None,
                     unsupported_reason: None,
                     declaration_to_usages: Some(DeclarationUsageReport {
@@ -4168,6 +4277,92 @@ for line in sys.stdin:
         assert_eq!(totals.improved, 1);
     }
 
+    #[test]
+    fn expected_pass_override_is_reported_as_passed() {
+        let mut case = benchmark_case();
+        case.expected_pass_reason = Some("verified current behavior".to_string());
+        let mut client = MockClient::new(vec![
+            tool(
+                "search_symbols",
+                search_symbols_json("src/service.rs", "example.build_service", 30),
+            ),
+            tool(
+                "scan_usages_by_location",
+                scan_usages_json(vec![("src/lib.rs", 8)], false),
+            ),
+        ]);
+
+        let report = run_case(
+            &case,
+            PositionEncoding::Utf16,
+            ReferencePolicy::BindingsOptional,
+            None,
+            &mut client,
+            false,
+            false,
+        );
+
+        assert_eq!(report.status, CaseStatus::Passed);
+        assert_eq!(
+            report.expected_pass_reason.as_deref(),
+            Some("verified current behavior")
+        );
+        assert_eq!(report.diagnostics[0].kind, "expected_pass_override");
+    }
+
+    #[test]
+    fn expected_pass_override_does_not_mask_a_regression() {
+        let mut case = benchmark_case();
+        case.expected_pass_reason = Some("verified current behavior".to_string());
+        let mut client = MockClient::new(vec![
+            tool(
+                "search_symbols",
+                search_symbols_json("src/service.rs", "example.build_service", 30),
+            ),
+            tool(
+                "scan_usages_by_location",
+                scan_usages_json(Vec::new(), false),
+            ),
+        ]);
+
+        let report = run_case(
+            &case,
+            PositionEncoding::Utf16,
+            ReferencePolicy::BindingsOptional,
+            None,
+            &mut client,
+            false,
+            false,
+        );
+
+        assert_eq!(report.status, CaseStatus::Failed);
+        assert_eq!(report.diagnostics[0].kind, "expected_pass_override");
+    }
+
+    #[test]
+    fn expected_pass_overlay_promotes_only_authored_expected_failures() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let manifest_path = tempdir.path().join("expected-passes.yaml");
+        fs::write(
+            &manifest_path,
+            "schemaVersion: 1\nexpectedPasses:\n  - caseId: expected-case\n    reason: verified\n",
+        )
+        .unwrap();
+        let mut documents = vec![(
+            PathBuf::from("benchmarks/cases/sample.yaml"),
+            serde_yaml::from_str::<BenchmarkDocument>(
+                "schemaVersion: 2\ncorpus:\n  partition: development\n  selection: analyzer_informed\ngroundTruth:\n  status: legacy_unattributed\n  reviewers: []\nreferencePolicy: bindings_optional\nsource:\n  kind: fixture\n  path: fixtures/sample\nlanguage: rust\ncases:\n  - id: expected-case\n    expectedFailure:\n      reason: historical gap\n",
+            )
+            .unwrap(),
+        )];
+
+        apply_expected_passes(&mut documents, tempdir.path(), Some(&manifest_path)).unwrap();
+
+        let case = &documents[0].1.cases[0];
+        assert!(case.expected_failure.is_none());
+        assert_eq!(case.expected_pass_reason.as_deref(), Some("verified"));
+    }
+
     #[test]
     fn unsupported_case_reports_boundary_status_by_default() {
         let mut case = benchmark_case();
@@ -5153,6 +5348,7 @@ for line in sys.stdin:
             }],
             type_lookups: Vec::new(),
             expected_failure: None,
+            expected_pass_reason: None,
             not_planned: None,
             unsupported: None,
             verification: None,