From bcde86e371e13d39df4f3caea4d4e33efbe923eb Mon Sep 17 00:00:00 2001 From: Eric Conklin Date: Thu, 4 Jun 2026 13:44:42 -0500 Subject: [PATCH] Add --- scripts/compare_prod_like_oracle_findings.py | 82 +++++++++++++------- tests/test_prod_like_oracle_comparison.py | 10 +++ 2 files changed, 64 insertions(+), 28 deletions(-) diff --git a/scripts/compare_prod_like_oracle_findings.py b/scripts/compare_prod_like_oracle_findings.py index f0dc9f9..009df87 100644 --- a/scripts/compare_prod_like_oracle_findings.py +++ b/scripts/compare_prod_like_oracle_findings.py @@ -12,6 +12,14 @@ SANDBOX_PREFIX = "iamscope-prodlike-v1-" +ORACLE_I001_BOUNDARY_TRIAGE_NOTE = ( + "emitted blocked due complete-confidence boundary evidence; likely oracle/fixture expectation conflict, " + "not automatically an IAMScope false positive" +) +UNCERTAINTY_PROBE_EXTRA_TRIAGE_NOTE = ( + "extra blocked path induced by uncertainty-probe boundary/policy shape; " + "not part of deterministic oracle mapping" +) NON_CLAIMS = [ "not broad IAMScope correctness", @@ -169,6 +177,18 @@ def _mapping_key(spec: dict[str, str]) -> tuple[str, str, str]: return (spec["pattern_id"], spec["source_name"], spec["target_name"]) +def _oracle_mismatch_triage_note(row_id: str, expected_verdict: str, emitted_verdicts: list[str]) -> str | None: + if row_id == "oracle-i-001" and expected_verdict == "inconclusive" and "blocked" in emitted_verdicts: + return ORACLE_I001_BOUNDARY_TRIAGE_NOTE + return None + + +def _unmapped_sandbox_extra_triage_note(finding: FindingView) -> str | None: + if finding.source_name == f"{SANDBOX_PREFIX}uncertainty-probe" and finding.verdict == "blocked": + return UNCERTAINTY_PROBE_EXTRA_TRIAGE_NOTE + return None + + def compare(oracle_payload: Any, findings_payload: Any) -> dict[str, Any]: oracle_rows = _oracle_rows(oracle_payload) raw_findings = _findings(findings_payload) @@ -249,23 +269,25 @@ def compare(oracle_payload: Any, findings_payload: Any) -> dict[str, Any]: emitted_verdicts = sorted({candidate.verdict for candidate in candidates}) expected_verdict = mapping["expected_verdict"] category = "oracle_match" if expected_verdict in emitted_verdicts else "oracle_mismatch" - rows.append( - { - **base, - "comparison_category": category, - "expected_verdict": expected_verdict, - "emitted_verdict": emitted_verdicts[0] if len(emitted_verdicts) == 1 else emitted_verdicts, - "pattern_id": mapping["pattern_id"], - "source_name": mapping["source_name"], - "target_name": mapping["target_name"], - "finding_ids": [candidate.finding_id for candidate in candidates], - "reason": ( - "emitted verdict matched expected oracle category" - if category == "oracle_match" - else "emitted verdict differed from expected oracle category" - ), - } - ) + row = { + **base, + "comparison_category": category, + "expected_verdict": expected_verdict, + "emitted_verdict": emitted_verdicts[0] if len(emitted_verdicts) == 1 else emitted_verdicts, + "pattern_id": mapping["pattern_id"], + "source_name": mapping["source_name"], + "target_name": mapping["target_name"], + "finding_ids": [candidate.finding_id for candidate in candidates], + "reason": ( + "emitted verdict matched expected oracle category" + if category == "oracle_match" + else "emitted verdict differed from expected oracle category" + ), + } + triage_note = _oracle_mismatch_triage_note(row_id, expected_verdict, emitted_verdicts) + if triage_note is not None: + row["triage_note"] = triage_note + rows.append(row) environmental_extras: list[dict[str, Any]] = [] unmapped_sandbox_extras: list[dict[str, Any]] = [] @@ -285,17 +307,19 @@ def compare(oracle_payload: Any, findings_payload: Any) -> dict[str, Any]: } ) elif finding.source_has_sandbox_prefix and finding.mapping_key not in supported_mapping_keys: - unmapped_sandbox_extras.append( - { - "comparison_category": "unmapped_sandbox_extra", - "extra_type": "sandbox_source_has_no_deterministic_oracle_mapping", - "finding_id": finding.finding_id, - "pattern_id": finding.pattern_id, - "verdict": finding.verdict, - "source_name": finding.source_name, - "target_name": finding.target_name, - } - ) + extra = { + "comparison_category": "unmapped_sandbox_extra", + "extra_type": "sandbox_source_has_no_deterministic_oracle_mapping", + "finding_id": finding.finding_id, + "pattern_id": finding.pattern_id, + "verdict": finding.verdict, + "source_name": finding.source_name, + "target_name": finding.target_name, + } + triage_note = _unmapped_sandbox_extra_triage_note(finding) + if triage_note is not None: + extra["triage_note"] = triage_note + unmapped_sandbox_extras.append(extra) comparison_category_counts = Counter(row["comparison_category"] for row in rows) comparison_category_counts.update(extra["comparison_category"] for extra in environmental_extras) @@ -385,6 +409,7 @@ def render_summary(result: dict[str, Any]) -> str: "Oracle Mismatches", [ f"{row['oracle_row_id']}: expected {row.get('expected_verdict')}, emitted {row.get('emitted_verdict')} for {row.get('source_name')} -> {row.get('target_name')}" + + (f"; triage: {row['triage_note']}" if row.get("triage_note") else "") for row in rows if row["comparison_category"] == "oracle_mismatch" ], @@ -424,6 +449,7 @@ def render_summary(result: dict[str, Any]) -> str: "Unmapped Sandbox Extras", [ f"{extra['finding_id']}: {extra['extra_type']} {extra['pattern_id']} {extra['verdict']} {extra['source_name']} -> {extra['target_name']}" + + (f"; triage: {extra['triage_note']}" if extra.get("triage_note") else "") for extra in result["unmapped_sandbox_extras"] ], ) diff --git a/tests/test_prod_like_oracle_comparison.py b/tests/test_prod_like_oracle_comparison.py index fe82e42..c56158b 100644 --- a/tests/test_prod_like_oracle_comparison.py +++ b/tests/test_prod_like_oracle_comparison.py @@ -139,6 +139,10 @@ def test_comparison_classifies_matches_mismatch_missing_unsupported_and_not_comp assert _row_by_id(result, "oracle-b-001")["comparison_category"] == "oracle_match" assert _row_by_id(result, "oracle-i-001")["comparison_category"] == "oracle_mismatch" assert _row_by_id(result, "oracle-i-001")["emitted_verdict"] == "blocked" + assert _row_by_id(result, "oracle-i-001")["triage_note"] == ( + "emitted blocked due complete-confidence boundary evidence; likely oracle/fixture expectation conflict, " + "not automatically an IAMScope false positive" + ) assert _row_by_id(result, "oracle-v-006")["comparison_category"] == "oracle_missing" assert _row_by_id(result, "oracle-v-003")["comparison_category"] == "not_currently_live_comparable" assert _row_by_id(result, "oracle-u-001")["comparison_category"] == "unsupported_static_only" @@ -175,6 +179,10 @@ def test_environmental_extra_uses_sanitized_source_and_target_names_only() -> No assert unmapped[0]["extra_type"] == "sandbox_source_has_no_deterministic_oracle_mapping" assert unmapped[0]["source_name"] == "iamscope-prodlike-v1-uncertainty-probe" assert unmapped[0]["target_name"] == "iamscope-prodlike-v1-service-mediated-target" + assert unmapped[0]["triage_note"] == ( + "extra blocked path induced by uncertainty-probe boundary/policy shape; " + "not part of deterministic oracle mapping" + ) def test_output_json_contains_no_raw_account_ids_or_iam_arns(tmp_path: Path) -> None: @@ -198,6 +206,8 @@ def test_summary_contains_non_claims_without_machine_score_or_pass_fail_fields(t assert "no composite benchmark score" in summary assert "no pass/fail benchmark label" in summary + assert "likely oracle/fixture expectation conflict" in summary + assert "extra blocked path induced by uncertainty-probe boundary/policy shape" in summary assert "composite_score" not in summary assert "benchmark_passed" not in summary assert "pass_fail" not in summary