Mark current Bifrost regressions as expected

.github/workflows/benchmark.yml

@@ -105,6 +105,7 @@ jobs:
             --bifrost-working-tree
             --work-dir target/usagebench
             --scan-usages-max-duration-secs 300
+            --expected-failures benchmarks/expectations/bifrost-expected-failures.yaml
             --expected-passes benchmarks/expectations/bifrost-expected-passes.yaml
             --output "benchmark-output/run-${GITHUB_RUN_ID}.json"
           )

benchmarks/README.md

@@ -218,6 +218,9 @@ Each case supports both benchmark directions:
 - `expectedFailure.reason` keeps a known analyzer gap in the baseline while
   still running the case and reporting it as improved if it unexpectedly starts
   passing.
+- `benchmarks/expectations/bifrost-expected-failures.yaml` records current
+  Bifrost regressions without changing content-addressed legacy or evaluation
+  case documents. The scheduled benchmark applies this overlay before scoring.
 - `benchmarks/expectations/bifrost-expected-passes.yaml` can promote a frozen
   historical `expectedFailure` to a current Bifrost required pass without
   mutating content-addressed legacy benchmark documents. A matching case must

benchmarks/expectations/bifrost-expected-failures.yaml

@@ -0,0 +1,40 @@
+schemaVersion: 1
+expectedFailures:
+  - caseId: real-project-v2-cpp-01-2
+    reason: >
+      Bifrost cannot resolve the ImGui_ImplSDL3_CloseGamepads declaration symbol,
+      although forward definition lookup succeeds.
+  - caseId: real-project-v2-cpp-02-1
+    reason: >
+      Bifrost does not navigate from the TessPageIteratorDelete header declaration
+      to its implementation body.
+  - caseId: real-project-v2-cpp-02-3
+    reason: >
+      Bifrost does not navigate from the TessResultIteratorSymbolIsSubscript header
+      declaration to its implementation body.
+  - caseId: real-project-v2-cpp-04-3
+    reason: >
+      Bifrost cannot resolve the MountFilesystem declaration symbol, although
+      forward definition lookup succeeds.
+  - caseId: real-project-v2-rust-04-3
+    reason: >
+      Bifrost cannot resolve the ServerEvents usage across the indexed Rust
+      crate boundary.
+  - caseId: rust-struct-construction
+    reason: >
+      Bifrost declaration-to-usages misses the two Self references.
+  - caseId: rust-parity-module-declaration-definition
+    reason: >
+      Bifrost module declaration lookup returns a zero-width range instead of
+      the authored module token.
+  - caseId: rust-parity-macro-generated-function-reference
+    reason: >
+      Bifrost does not index or resolve the macro-generated function.
+  - caseId: scala-parity-case-class-generated-construction-and-copy
+    reason: >
+      Bifrost reverse usage includes the synthetic copy call as an unexpected
+      extra location.
+  - caseId: ts-parity-interface-property-access
+    reason: >
+      Bifrost cannot resolve the user.name interface property usage to its
+      declaration and omits it from reverse usages.

src/main.rs

@@ -197,6 +197,9 @@ enum Command {
         /// Run only benchmark documents for this language.
         #[arg(long)]
         language: Option<String>,
+        /// Current Bifrost failures applied without mutating frozen benchmark documents.
+        #[arg(long)]
+        expected_failures: Option<PathBuf>,
         /// Versioned overlay that promotes historical expected failures to current required passes.
         #[arg(long)]
         expected_passes: Option<PathBuf>,
@@ -457,6 +460,7 @@ fn main() -> Result<()> {
             keep_worktrees,
             case_id,
             language,
+            expected_failures,
             expected_passes,
         } => {
             let mut options = RunBifrostOptions::with_defaults(path);
@@ -472,6 +476,7 @@ fn main() -> Result<()> {
             options.keep_worktrees = keep_worktrees;
             options.case_id = case_id;
             options.language = language;
+            options.expected_failures = expected_failures;
             options.expected_passes = expected_passes;
             let report = run_bifrost(options)?;
             println!(

src/runners/bifrost.rs

@@ -108,6 +108,7 @@ pub struct RunBifrostOptions {
     pub keep_worktrees: bool,
     pub case_id: Option<String>,
     pub language: Option<String>,
+    pub expected_failures: Option<PathBuf>,
     pub expected_passes: Option<PathBuf>,
 }
 
@@ -128,6 +129,7 @@ impl RunBifrostOptions {
             keep_worktrees: false,
             case_id: None,
             language: None,
+            expected_failures: None,
             expected_passes: None,
         }
     }
@@ -147,6 +149,20 @@ struct ExpectedPass {
     reason: String,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct ExpectedFailureManifest {
+    schema_version: u32,
+    expected_failures: Vec<ExpectedFailureOverride>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct ExpectedFailureOverride {
+    case_id: String,
+    reason: String,
+}
+
 pub type BifrostRunReport = RunReport;
 
 pub fn generated_bifrost_report_schema_json() -> Result<String> {
@@ -185,6 +201,11 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result<BifrostRunReport> {
         &repo_root,
         options.expected_passes.as_deref(),
     )?;
+    apply_expected_failures(
+        &mut benchmark_documents,
+        &repo_root,
+        options.expected_failures.as_deref(),
+    )?;
     let requested_totals = requested_run_totals(
         benchmark_documents
             .iter()
@@ -452,6 +473,90 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result<BifrostRunReport> {
     Ok(report)
 }
 
+fn apply_expected_failures(
+    documents: &mut [(PathBuf, BenchmarkDocument)],
+    repo_root: &Path,
+    manifest_path: Option<&Path>,
+) -> Result<()> {
+    let Some(manifest_path) = manifest_path else {
+        return Ok(());
+    };
+    let manifest_path = if manifest_path.is_absolute() {
+        manifest_path.to_path_buf()
+    } else {
+        repo_root.join(manifest_path)
+    };
+    let source = fs::read_to_string(&manifest_path)
+        .with_context(|| format!("read expected-failure overlay {}", manifest_path.display()))?;
+    let manifest = serde_yaml::from_str::<ExpectedFailureManifest>(&source)
+        .with_context(|| format!("parse expected-failure overlay {}", manifest_path.display()))?;
+    if manifest.schema_version != 1 {
+        bail!(
+            "expected-failure overlay {} has unsupported schemaVersion {}",
+            manifest_path.display(),
+            manifest.schema_version
+        );
+    }
+
+    let mut expected_failures = BTreeMap::new();
+    for expected_failure in manifest.expected_failures {
+        if expected_failure.case_id.trim().is_empty() || expected_failure.reason.trim().is_empty() {
+            bail!(
+                "expected-failure overlay {} requires non-empty caseId and reason",
+                manifest_path.display()
+            );
+        }
+        if expected_failures
+            .insert(expected_failure.case_id.clone(), expected_failure.reason)
+            .is_some()
+        {
+            bail!(
+                "expected-failure overlay {} repeats caseId {}",
+                manifest_path.display(),
+                expected_failure.case_id
+            );
+        }
+    }
+
+    let mut applied = BTreeSet::new();
+    for (_, document) in documents {
+        for case in &mut document.cases {
+            let Some(reason) = expected_failures.get(&case.id) else {
+                continue;
+            };
+            if case.expected_failure.is_some()
+                || case.expected_pass_reason.is_some()
+                || case.not_planned.is_some()
+                || case.unsupported.is_some()
+            {
+                bail!(
+                    "expected-failure overlay {} may only mark an unclassified case: {}",
+                    manifest_path.display(),
+                    case.id
+                );
+            }
+            case.expected_failure = Some(crate::ExpectedFailure {
+                reason: reason.clone(),
+            });
+            applied.insert(case.id.clone());
+        }
+    }
+
+    let missing = expected_failures
+        .keys()
+        .filter(|case_id| !applied.contains(*case_id))
+        .cloned()
+        .collect::<Vec<_>>();
+    if !missing.is_empty() {
+        bail!(
+            "expected-failure overlay {} references case IDs absent from this run: {}",
+            manifest_path.display(),
+            missing.join(", ")
+        );
+    }
+    Ok(())
+}
+
 fn apply_expected_passes(
     documents: &mut [(PathBuf, BenchmarkDocument)],
     repo_root: &Path,
@@ -4618,6 +4723,99 @@ for line in sys.stdin:
         assert_eq!(report.diagnostics[0].kind, "expected_pass_override");
     }
 
+    #[test]
+    fn expected_failure_overlay_marks_an_unclassified_case() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let manifest_path = tempdir.path().join("expected-failures.yaml");
+        fs::write(
+            &manifest_path,
+            "schemaVersion: 1\nexpectedFailures:\n  - caseId: current-gap\n    reason: current regression\n",
+        )
+        .unwrap();
+        let mut documents = vec![(
+            PathBuf::from("benchmarks/cases/sample.yaml"),
+            serde_yaml::from_str::<BenchmarkDocument>(
+                "schemaVersion: 2\ncorpus:\n  partition: development\n  selection: analyzer_informed\ngroundTruth:\n  status: legacy_unattributed\n  reviewers: []\nreferencePolicy: bindings_optional\nsource:\n  kind: fixture\n  path: fixtures/sample\nlanguage: rust\ncases:\n  - id: current-gap\n",
+            )
+            .unwrap(),
+        )];
+
+        apply_expected_failures(&mut documents, tempdir.path(), Some(&manifest_path)).unwrap();
+
+        assert_eq!(
+            documents[0].1.cases[0]
+                .expected_failure
+                .as_ref()
+                .map(|expected| expected.reason.as_str()),
+            Some("current regression")
+        );
+    }
+
+    #[test]
+    fn expected_failure_overlay_does_not_replace_an_authored_marker() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let manifest_path = tempdir.path().join("expected-failures.yaml");
+        fs::write(
+            &manifest_path,
+            "schemaVersion: 1\nexpectedFailures:\n  - caseId: existing-gap\n    reason: replacement\n",
+        )
+        .unwrap();
+        let mut documents = vec![(
+            PathBuf::from("benchmarks/cases/sample.yaml"),
+            serde_yaml::from_str::<BenchmarkDocument>(
+                "schemaVersion: 2\ncorpus:\n  partition: development\n  selection: analyzer_informed\ngroundTruth:\n  status: legacy_unattributed\n  reviewers: []\nreferencePolicy: bindings_optional\nsource:\n  kind: fixture\n  path: fixtures/sample\nlanguage: rust\ncases:\n  - id: existing-gap\n    expectedFailure:\n      reason: authored gap\n",
+            )
+            .unwrap(),
+        )];
+
+        let error = apply_expected_failures(&mut documents, tempdir.path(), Some(&manifest_path))
+            .unwrap_err();
+
+        assert!(error
+            .to_string()
+            .contains("may only mark an unclassified case"));
+    }
+
+    #[test]
+    fn checked_in_expected_failure_overlay_marks_all_current_regressions() {
+        let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        let case_files = [
+            "benchmarks/cases/evaluation/real-project-v2/cpp-01.yaml",
+            "benchmarks/cases/evaluation/real-project-v2/cpp-02.yaml",
+            "benchmarks/cases/evaluation/real-project-v2/cpp-04.yaml",
+            "benchmarks/cases/evaluation/real-project-v2/rust-04.yaml",
+            "benchmarks/cases/rust-baseline.yaml",
+            "benchmarks/cases/rust-lsp-parity.yaml",
+            "benchmarks/cases/scala-lsp-parity.yaml",
+            "benchmarks/cases/typescript-lsp-parity.yaml",
+        ];
+        let mut documents = case_files
+            .iter()
+            .map(|case_file| {
+                let path = repo_root.join(case_file);
+                let source = fs::read_to_string(&path).unwrap();
+                let document = serde_yaml::from_str::<BenchmarkDocument>(&source).unwrap();
+                (path, document)
+            })
+            .collect::<Vec<_>>();
+
+        apply_expected_failures(
+            &mut documents,
+            &repo_root,
+            Some(Path::new(
+                "benchmarks/expectations/bifrost-expected-failures.yaml",
+            )),
+        )
+        .unwrap();
+
+        let overlaid = documents
+            .iter()
+            .flat_map(|(_, document)| &document.cases)
+            .filter(|case| case.expected_failure.is_some())
+            .count();
+        assert_eq!(overlaid, 10);
+    }
+
     #[test]
     fn expected_pass_overlay_promotes_only_authored_expected_failures() {
         let tempdir = tempfile::tempdir().unwrap();