diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index eb750b1..cce6977 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -105,6 +105,7 @@ jobs: --bifrost-working-tree --work-dir target/usagebench --scan-usages-max-duration-secs 300 + --expected-passes benchmarks/expectations/bifrost-expected-passes.yaml --output "benchmark-output/run-${GITHUB_RUN_ID}.json" ) if [ "$INCLUDE_UNSUPPORTED" = "true" ]; then diff --git a/README.md b/README.md index d7b1645..dd3d63c 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,8 @@ The workflow: * validates `benchmarks/cases` * checks out `BrokkAi/bifrost` * builds `usagebench` -* runs `usagebench run-bifrost benchmarks/cases` +* runs `usagebench run-bifrost benchmarks/cases` with the versioned current + expected-pass overlay * uploads the JSON report from `benchmark-output` * publishes a GitHub step summary * optionally posts a payload to Slack diff --git a/benchmarks/README.md b/benchmarks/README.md index 3afed2d..1950740 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -218,6 +218,10 @@ Each case supports both benchmark directions: - `expectedFailure.reason` keeps a known analyzer gap in the baseline while still running the case and reporting it as improved if it unexpectedly starts passing. +- `benchmarks/expectations/bifrost-expected-passes.yaml` can promote a frozen + historical `expectedFailure` to a current Bifrost required pass without + mutating content-addressed legacy benchmark documents. A matching case must + pass; any regression is reported as a normal failure. - `notPlanned.reason` keeps runtime-dynamic or generated-code expectations in the corpus and runs them without including them in the planned-case total. - `unsupported.reason` documents out-of-boundary cases and reports them as diff --git a/benchmarks/expectations/bifrost-expected-passes.yaml b/benchmarks/expectations/bifrost-expected-passes.yaml new file mode 100644 index 0000000..65906ae --- /dev/null +++ b/benchmarks/expectations/bifrost-expected-passes.yaml @@ -0,0 +1,6 @@ +schemaVersion: 1 +expectedPasses: + - caseId: cpp-parity-function-like-macro-expanded-call + reason: > + Promoted after scheduled runs 32240334240 and 32356646488 passed, then a + focused replay passed exactly against Bifrost 79ea68cf5e2e7cda9cd90de4188836bb02fe0d6c. diff --git a/src/evaluation.rs b/src/evaluation.rs index ce55ec9..1363a47 100644 --- a/src/evaluation.rs +++ b/src/evaluation.rs @@ -4124,6 +4124,7 @@ mod tests { }], type_lookups: Vec::new(), expected_failure: None, + expected_pass_reason: None, not_planned: None, unsupported: None, verification: None, diff --git a/src/lib.rs b/src/lib.rs index 37fb578..cff4aa3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -194,6 +194,11 @@ pub struct BenchmarkCase { pub type_lookups: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_failure: Option, + /// Current-run expectation supplied by the Bifrost expected-pass overlay. + /// This is intentionally not part of an authored benchmark document: some + /// legacy documents are content-addressed by frozen promotion evidence. + #[serde(skip)] + pub expected_pass_reason: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub not_planned: Option, #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/src/main.rs b/src/main.rs index 941a9e4..e4752db 100644 --- a/src/main.rs +++ b/src/main.rs @@ -197,6 +197,9 @@ enum Command { /// Run only benchmark documents for this language. #[arg(long)] language: Option, + /// Versioned overlay that promotes historical expected failures to current required passes. + #[arg(long)] + expected_passes: Option, }, /// Run benchmark cases against a versioned language-server profile. RunLsp { @@ -448,6 +451,7 @@ fn main() -> Result<()> { keep_worktrees, case_id, language, + expected_passes, } => { let mut options = RunBifrostOptions::with_defaults(path); options.bifrost_repo = bifrost_repo; @@ -462,6 +466,7 @@ fn main() -> Result<()> { options.keep_worktrees = keep_worktrees; options.case_id = case_id; options.language = language; + options.expected_passes = expected_passes; let report = run_bifrost(options)?; println!( "ran {} planned case(s) ({} development, {} evaluation): {} passed, {} near miss(es), {} position-unverified, {} improved, {} failed, {} expected failure(s), {} not planned, {} unsupported, {} skipped, {} error(s)", diff --git a/src/promotion.rs b/src/promotion.rs index 07089f3..758d704 100644 --- a/src/promotion.rs +++ b/src/promotion.rs @@ -720,6 +720,7 @@ mod tests { required_destination_status: None, location_metrics: None, expected_failure_reason: None, + expected_pass_reason: None, not_planned_reason: None, unsupported_reason: None, declaration_to_usages: None, diff --git a/src/publication.rs b/src/publication.rs index 747fa1f..c29b7e1 100644 --- a/src/publication.rs +++ b/src/publication.rs @@ -1171,6 +1171,7 @@ mod tests { required_destination_status: Some(RequiredDestinationStatus::Found), location_metrics: None, expected_failure_reason: None, + expected_pass_reason: None, not_planned_reason: None, unsupported_reason: None, declaration_to_usages: None, @@ -1185,6 +1186,7 @@ mod tests { required_destination_status: Some(RequiredDestinationStatus::Unsupported), location_metrics: None, expected_failure_reason: None, + expected_pass_reason: None, not_planned_reason: None, unsupported_reason: Some("unsupported".into()), declaration_to_usages: None, diff --git a/src/results.rs b/src/results.rs index 613d404..a307f14 100644 --- a/src/results.rs +++ b/src/results.rs @@ -2233,6 +2233,7 @@ mod tests { }, }), expected_failure_reason: None, + expected_pass_reason: None, not_planned_reason: None, unsupported_reason: None, declaration_to_usages: None, diff --git a/src/runners/bifrost.rs b/src/runners/bifrost.rs index 3d18e31..2a741a0 100644 --- a/src/runners/bifrost.rs +++ b/src/runners/bifrost.rs @@ -59,6 +59,7 @@ pub struct RunBifrostOptions { pub keep_worktrees: bool, pub case_id: Option, pub language: Option, + pub expected_passes: Option, } impl RunBifrostOptions { @@ -78,10 +79,25 @@ impl RunBifrostOptions { keep_worktrees: false, case_id: None, language: None, + expected_passes: None, } } } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ExpectedPassManifest { + schema_version: u32, + expected_passes: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ExpectedPass { + case_id: String, + reason: String, +} + pub type BifrostRunReport = RunReport; pub fn generated_bifrost_report_schema_json() -> Result { @@ -105,7 +121,7 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result { let repo_root = find_repo_root_for_path(&options.case_path)?; let usagebench_provenance = resolve_usagebench_provenance(&repo_root)?; let case_files = crate::validate_path(&options.case_path)?; - let benchmark_documents = case_files + let mut benchmark_documents = case_files .iter() .map(|case_file| { let yaml = fs::read_to_string(case_file) @@ -115,6 +131,11 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result { Ok((case_file.clone(), document)) }) .collect::>>()?; + apply_expected_passes( + &mut benchmark_documents, + &repo_root, + options.expected_passes.as_deref(), + )?; let requested_totals = requested_run_totals( benchmark_documents .iter() @@ -382,6 +403,85 @@ pub fn run_bifrost(options: RunBifrostOptions) -> Result { Ok(report) } +fn apply_expected_passes( + documents: &mut [(PathBuf, BenchmarkDocument)], + repo_root: &Path, + manifest_path: Option<&Path>, +) -> Result<()> { + let Some(manifest_path) = manifest_path else { + return Ok(()); + }; + let manifest_path = if manifest_path.is_absolute() { + manifest_path.to_path_buf() + } else { + repo_root.join(manifest_path) + }; + let source = fs::read_to_string(&manifest_path) + .with_context(|| format!("read expected-pass overlay {}", manifest_path.display()))?; + let manifest = serde_yaml::from_str::(&source) + .with_context(|| format!("parse expected-pass overlay {}", manifest_path.display()))?; + if manifest.schema_version != 1 { + bail!( + "expected-pass overlay {} has unsupported schemaVersion {}", + manifest_path.display(), + manifest.schema_version + ); + } + + let mut expected_passes = BTreeMap::new(); + for expected_pass in manifest.expected_passes { + if expected_pass.case_id.trim().is_empty() || expected_pass.reason.trim().is_empty() { + bail!( + "expected-pass overlay {} requires non-empty caseId and reason", + manifest_path.display() + ); + } + if expected_passes + .insert(expected_pass.case_id.clone(), expected_pass.reason) + .is_some() + { + bail!( + "expected-pass overlay {} repeats caseId {}", + manifest_path.display(), + expected_pass.case_id + ); + } + } + + let mut applied = BTreeSet::new(); + for (_, document) in documents { + for case in &mut document.cases { + let Some(reason) = expected_passes.get(&case.id) else { + continue; + }; + if case.expected_failure.is_none() { + bail!( + "expected-pass overlay {} may only promote an authored expectedFailure: {}", + manifest_path.display(), + case.id + ); + } + case.expected_failure = None; + case.expected_pass_reason = Some(reason.clone()); + applied.insert(case.id.clone()); + } + } + + let missing = expected_passes + .keys() + .filter(|case_id| !applied.contains(*case_id)) + .cloned() + .collect::>(); + if !missing.is_empty() { + bail!( + "expected-pass overlay {} references case IDs absent from this run: {}", + manifest_path.display(), + missing.join(", ") + ); + } + Ok(()) +} + fn document_failure_cases( document: &BenchmarkDocument, include_unsupported: bool, @@ -867,6 +967,13 @@ fn run_case_with_scan_duration( .expected_failure .as_ref() .map(|expected_failure| expected_failure.reason.clone()); + let expected_pass_reason = case.expected_pass_reason.clone(); + if let Some(reason) = &expected_pass_reason { + diagnostics.push(RunDiagnostic { + kind: "expected_pass_override".to_string(), + message: format!("current Bifrost expected-pass overlay applied: {reason}"), + }); + } let not_planned_reason = case .not_planned .as_ref() @@ -898,6 +1005,7 @@ fn run_case_with_scan_duration( id: case.id.clone(), status, expected_failure_reason, + expected_pass_reason, not_planned_reason, unsupported_reason: case .unsupported @@ -3148,6 +3256,7 @@ for line in sys.stdin: status: CaseStatus::Passed, location_metrics: None, expected_failure_reason: None, + expected_pass_reason: None, not_planned_reason: None, unsupported_reason: None, declaration_to_usages: Some(DeclarationUsageReport { @@ -4168,6 +4277,92 @@ for line in sys.stdin: assert_eq!(totals.improved, 1); } + #[test] + fn expected_pass_override_is_reported_as_passed() { + let mut case = benchmark_case(); + case.expected_pass_reason = Some("verified current behavior".to_string()); + let mut client = MockClient::new(vec![ + tool( + "search_symbols", + search_symbols_json("src/service.rs", "example.build_service", 30), + ), + tool( + "scan_usages_by_location", + scan_usages_json(vec![("src/lib.rs", 8)], false), + ), + ]); + + let report = run_case( + &case, + PositionEncoding::Utf16, + ReferencePolicy::BindingsOptional, + None, + &mut client, + false, + false, + ); + + assert_eq!(report.status, CaseStatus::Passed); + assert_eq!( + report.expected_pass_reason.as_deref(), + Some("verified current behavior") + ); + assert_eq!(report.diagnostics[0].kind, "expected_pass_override"); + } + + #[test] + fn expected_pass_override_does_not_mask_a_regression() { + let mut case = benchmark_case(); + case.expected_pass_reason = Some("verified current behavior".to_string()); + let mut client = MockClient::new(vec![ + tool( + "search_symbols", + search_symbols_json("src/service.rs", "example.build_service", 30), + ), + tool( + "scan_usages_by_location", + scan_usages_json(Vec::new(), false), + ), + ]); + + let report = run_case( + &case, + PositionEncoding::Utf16, + ReferencePolicy::BindingsOptional, + None, + &mut client, + false, + false, + ); + + assert_eq!(report.status, CaseStatus::Failed); + assert_eq!(report.diagnostics[0].kind, "expected_pass_override"); + } + + #[test] + fn expected_pass_overlay_promotes_only_authored_expected_failures() { + let tempdir = tempfile::tempdir().unwrap(); + let manifest_path = tempdir.path().join("expected-passes.yaml"); + fs::write( + &manifest_path, + "schemaVersion: 1\nexpectedPasses:\n - caseId: expected-case\n reason: verified\n", + ) + .unwrap(); + let mut documents = vec![( + PathBuf::from("benchmarks/cases/sample.yaml"), + serde_yaml::from_str::( + "schemaVersion: 2\ncorpus:\n partition: development\n selection: analyzer_informed\ngroundTruth:\n status: legacy_unattributed\n reviewers: []\nreferencePolicy: bindings_optional\nsource:\n kind: fixture\n path: fixtures/sample\nlanguage: rust\ncases:\n - id: expected-case\n expectedFailure:\n reason: historical gap\n", + ) + .unwrap(), + )]; + + apply_expected_passes(&mut documents, tempdir.path(), Some(&manifest_path)).unwrap(); + + let case = &documents[0].1.cases[0]; + assert!(case.expected_failure.is_none()); + assert_eq!(case.expected_pass_reason.as_deref(), Some("verified")); + } + #[test] fn unsupported_case_reports_boundary_status_by_default() { let mut case = benchmark_case(); @@ -5153,6 +5348,7 @@ for line in sys.stdin: }], type_lookups: Vec::new(), expected_failure: None, + expected_pass_reason: None, not_planned: None, unsupported: None, verification: None, diff --git a/src/runners/lsp.rs b/src/runners/lsp.rs index 42e6c0e..32070ee 100644 --- a/src/runners/lsp.rs +++ b/src/runners/lsp.rs @@ -464,6 +464,7 @@ fn run_case( id: case.id.clone(), status: CaseStatus::Unsupported, expected_failure_reason: None, + expected_pass_reason: None, not_planned_reason: case.not_planned.as_ref().map(|item| item.reason.clone()), unsupported_reason: Some(unsupported.reason.clone()), declaration_to_usages: None, @@ -569,6 +570,7 @@ fn run_case( id: case.id.clone(), status, expected_failure_reason: None, + expected_pass_reason: None, not_planned_reason: case.not_planned.as_ref().map(|item| item.reason.clone()), unsupported_reason: case.unsupported.as_ref().map(|item| item.reason.clone()), declaration_to_usages, @@ -1972,6 +1974,7 @@ mod tests { required_destination_status: Some(destination_status), location_metrics: None, expected_failure_reason: None, + expected_pass_reason: None, not_planned_reason: None, unsupported_reason: None, declaration_to_usages: None, diff --git a/src/runners/mod.rs b/src/runners/mod.rs index cf54dc5..7f23569 100644 --- a/src/runners/mod.rs +++ b/src/runners/mod.rs @@ -682,6 +682,8 @@ pub struct CaseRunReport { pub location_metrics: Option, #[serde(skip_serializing_if = "Option::is_none")] pub expected_failure_reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expected_pass_reason: Option, #[serde(skip_serializing_if = "Option::is_none")] pub not_planned_reason: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -858,6 +860,7 @@ pub(crate) fn runner_failure_case( .expected_failure .as_ref() .map(|item| item.reason.clone()), + expected_pass_reason: case.expected_pass_reason.clone(), not_planned_reason: case.not_planned.as_ref().map(|item| item.reason.clone()), unsupported_reason: case.unsupported.as_ref().map(|item| item.reason.clone()), declaration_to_usages: None, @@ -886,6 +889,7 @@ pub(crate) fn excluded_case(case: &BenchmarkCase, status: CaseStatus) -> CaseRun .expected_failure .as_ref() .map(|item| item.reason.clone()), + expected_pass_reason: case.expected_pass_reason.clone(), not_planned_reason: case.not_planned.as_ref().map(|item| item.reason.clone()), unsupported_reason: case.unsupported.as_ref().map(|item| item.reason.clone()), declaration_to_usages: None, diff --git a/src/runners/report_compare.rs b/src/runners/report_compare.rs index 17db0b5..52f067f 100644 --- a/src/runners/report_compare.rs +++ b/src/runners/report_compare.rs @@ -500,6 +500,7 @@ mod tests { id: "sample-case".to_string(), status: super::super::CaseStatus::Passed, expected_failure_reason: None, + expected_pass_reason: None, not_planned_reason: None, unsupported_reason: None, declaration_to_usages: None,