diff --git a/CHANGELOG.md b/CHANGELOG.md index b7f54637b..cab83fa7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `assert-ai results matrix` for behavior-by-arm comparison across multiple runs and suites, with count-weighted prompt/scenario pooling and safe fallback when the impermissible split has no denominator. - Clean-install checks for every documented ASSERT/example environment, including `pip check` and credential-free target imports. ### Changed diff --git a/assert_ai/cli.py b/assert_ai/cli.py index 17e12bbca..3d88c520c 100644 --- a/assert_ai/cli.py +++ b/assert_ai/cli.py @@ -153,6 +153,17 @@ def _fmt_percent(value: Optional[float]) -> str: _PERMISSIBILITY_SPLIT_RATE_KEYS = tuple(_DERIVED_PERMISSIBILITY_RATE_KEYS.values()) +_PERMISSIBILITY_METRIC_ALIASES = { + rate_key: metric + for metric, rate_key in _DERIVED_PERMISSIBILITY_RATE_KEYS.items() +} + +#: The half of the split the matrix leads with. ``policy_violation`` unions +#: permissible and impermissible behaviors, so ranking behaviors by it can order +#: them by the wrong thing entirely -- a behavior can carry a high union rate +#: made up almost wholly of mishandled *permissible* work while another with a +#: lower union rate is nearly all genuine impermissible failure. +_MATRIX_SPLIT_METRIC = _POLICY_VIOLATION_NOT_PERMISSIBLE def _has_permissibility_split(*metric_sets: Any) -> bool: @@ -497,10 +508,7 @@ def _resolve_compare_metric(metric: str | None, run_summaries: Iterable[dict[str return metric summaries = list(run_summaries) if summaries and all( - has_permissibility_split_data( - run_summary.get("prompt_metrics") or {}, - run_summary.get("scenario_metrics") or {}, - ) + _run_dimension_rate(run_summary, _POLICY_VIOLATION_NOT_PERMISSIBLE) is not None for run_summary in summaries ): return _POLICY_VIOLATION_NOT_PERMISSIBLE @@ -667,8 +675,46 @@ def _compute_scenario_metrics( return metrics -def _load_behavior_categories(suite_dir: Path) -> list[dict[str, Any]]: - taxonomy = load_json(suite_dir / "taxonomy.json") +def _manifest_artifact_path( + suite_dir: Path, + manifest: dict[str, Any] | None, + stage_name: str, +) -> tuple[bool, Path | None]: + """Resolve a run's versioned artifact path without leaving its suite.""" + artifact_versions = manifest.get("artifact_versions") if isinstance(manifest, dict) else None + stage_ref = artifact_versions.get(stage_name) if isinstance(artifact_versions, dict) else None + if not isinstance(stage_ref, dict): + return False, None + + raw_path = stage_ref.get("path") or stage_ref.get("relative_path") + if not isinstance(raw_path, str) or not raw_path.strip(): + return True, None + + suite_root = suite_dir.resolve() + candidate = Path(raw_path) + try: + resolved = (candidate if candidate.is_absolute() else suite_dir / candidate).resolve() + resolved.relative_to(suite_root) + except (OSError, RuntimeError, ValueError): + return True, None + return True, resolved + + +def _load_behavior_categories( + suite_dir: Path, + manifest: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + has_versioned_ref, versioned_path = _manifest_artifact_path( + suite_dir, + manifest, + "systematize", + ) + if has_versioned_ref: + if versioned_path is None or not versioned_path.is_file(): + return [] + taxonomy = load_json(versioned_path) + else: + taxonomy = load_json(suite_dir / "taxonomy.json") behavior_categories = (taxonomy or {}).get("behavior_categories") if not isinstance(behavior_categories, list): return [] @@ -678,7 +724,7 @@ def _load_behavior_categories(suite_dir: Path) -> list[dict[str, Any]]: def _load_run_summary(run_dir: Path) -> dict[str, Any] | None: manifest = load_json(run_dir / "manifest.json") score_rows = load_jsonl(run_dir / "scores.jsonl") - behavior_categories = _load_behavior_categories(run_dir.parent) + behavior_categories = _load_behavior_categories(run_dir.parent, manifest) prompt_rows = [row for row in score_rows if not row.get("tester_model")] scenario_rows = [row for row in score_rows if row.get("tester_model")] @@ -701,11 +747,137 @@ def _load_run_summary(run_dir: Path) -> dict[str, Any] | None: "ended_at": (manifest or {}).get("ended_at"), "prompt_metrics": _compute_prompt_metrics(prompt_rows, behavior_categories), "scenario_metrics": _compute_scenario_metrics(scenario_rows, behavior_categories), + "behavior_categories": behavior_categories, "prompt_rows": prompt_rows, "scenario_rows": scenario_rows, } +def _run_behavior_name(run_dir: Path, suite_id: str) -> str: + config_path = run_dir / "config.yaml" + config = None + if config_path.exists(): + try: + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, yaml.YAMLError): + pass + behavior = config.get("behavior") if isinstance(config, dict) else None + if isinstance(behavior, dict) and isinstance(behavior.get("name"), str) and behavior.get("name"): + return behavior["name"] + + manifest = load_json(run_dir / "manifest.json") + manifest_candidates: list[Any] = [] + if isinstance(manifest, dict): + manifest_behavior = manifest.get("behavior") + manifest_candidates.append(manifest.get("behavior_name")) + if isinstance(manifest_behavior, dict): + manifest_candidates.append(manifest_behavior.get("name")) + manifest_config = manifest.get("config") + if isinstance(manifest_config, dict): + manifest_behavior = manifest_config.get("behavior") + if isinstance(manifest_behavior, dict): + manifest_candidates.append(manifest_behavior.get("name")) + for candidate in manifest_candidates: + if isinstance(candidate, str) and candidate: + return candidate + return suite_id + + +def _run_arm_label(run_id: str, suite_id: str) -> str: + prefix = f"{suite_id}-" + if run_id.startswith(prefix): + return run_id[len(prefix):] or run_id + return run_id + + +def _ordered_arm_labels(arms: Iterable[str]) -> list[str]: + known_order = {"baseline": 0, "prompted": 1, "acs": 2} + return sorted( + arms, + key=lambda arm: ( + 0 if arm.lower() in known_order else 1, + known_order.get(arm.lower(), 0), + arm.lower(), + ), + ) + + +def _pooled_rate(pairs: Iterable[tuple[int, int]]) -> float | None: + """Pool ``(flagged, scored)`` pairs instead of averaging rates.""" + flagged = scored = 0 + for hit, total in pairs: + flagged += hit + scored += total + return flagged / scored if scored else None + + +def _binary_counts(summary: Any) -> tuple[int, int] | None: + """Return ``(flagged, scored)`` for a binary dimension summary.""" + if not isinstance(summary, dict) or summary.get("kind") == "ordinal": + return None + counts = summary.get("counts") + if not isinstance(counts, dict): + return None + flagged = counts.get(1, counts.get("1", 0)) + passed = counts.get(0, counts.get("0", 0)) + if not isinstance(flagged, int) or not isinstance(passed, int): + return None + total = flagged + passed + return (flagged, total) if total else None + + +def _run_dimension_rate(run_summary: dict[str, Any], metric: str) -> float | None: + """Pool one metric across a run's prompt and scenario rows. + + The permissibility split is derived and stored in top-level bucket summaries, + not in ``dimensions``. All rates are recombined from counts when possible. + """ + prompt_metrics = run_summary.get("prompt_metrics") or {} + scenario_metrics = run_summary.get("scenario_metrics") or {} + halves = [metrics for metrics in (prompt_metrics, scenario_metrics) if isinstance(metrics, dict)] + + canonical_metric = _PERMISSIBILITY_METRIC_ALIASES.get(metric, metric) + split_key = _DERIVED_PERMISSIBILITY_RATE_KEYS.get(canonical_metric) + if split_key is not None: + bucket_key = _DERIVED_PERMISSIBILITY_SUMMARY_KEYS[canonical_metric] + pairs = [ + counts + for metrics in halves + if (counts := _binary_counts(metrics.get(bucket_key))) is not None + ] + if pairs: + return _pooled_rate(pairs) + for metrics in halves: + rate = metrics.get(split_key) + if isinstance(rate, (int, float)): + return float(rate) + return None + + pairs = [ + counts + for metrics in halves + if isinstance(metrics.get("dimensions"), dict) + and (counts := _binary_counts(metrics["dimensions"].get(metric))) is not None + ] + if pairs: + return _pooled_rate(pairs) + + prompt_rate = _dimension_rate(prompt_metrics, metric) + if prompt_rate is not None: + return prompt_rate + return _dimension_rate(scenario_metrics, metric) + + +def _parse_suite_run_arg(suite_run: str) -> tuple[str, str]: + parts = suite_run.strip("/").split("/") + if len(parts) == 1: + return parts[0], "run-1" + if len(parts) == 2: + return parts[0], parts[1] + _error(f"Invalid format: '{suite_run}'. Use SUITE/RUN (e.g., my-suite/run-1).") + raise AssertionError("unreachable") + + def _count_test_case_types(path: Path) -> tuple[int, int]: rows = load_jsonl(path) prompt_count = 0 @@ -1358,16 +1530,15 @@ def _run_within_suite_compare( behavior_category_deltas: list[dict[str, Any]] = [] if all(run_summary.get("prompt_rows") for run_summary in run_summaries): - behavior_categories = _load_behavior_categories(suite_dir) first_map = _behavior_category_metric_map( run_summaries[0]["prompt_rows"], metric, - behavior_categories, + run_summaries[0].get("behavior_categories") or [], ) last_map = _behavior_category_metric_map( run_summaries[-1]["prompt_rows"], metric, - behavior_categories, + run_summaries[-1].get("behavior_categories") or [], ) for behavior_category in sorted(set(first_map) | set(last_map)): first = first_map.get(behavior_category) @@ -1472,6 +1643,191 @@ def _run_within_suite_compare( console.print(delta_table) +@results.command("matrix", short_help="Compare behaviors across arms as a matrix") +@click.argument("suite_runs", nargs=-1) +@click.option( + "--suite", + "suites", + multiple=True, + shell_complete=_complete_suite, + help="Suite ID under artifacts/results. May be repeated; expands to all runs with scores.", +) +@click.option( + "--results-dir", + type=click.Path(path_type=Path), + default=DEFAULT_RESULTS_DIR, + show_default=True, + help="Results root to inspect.", +) +@click.option( + "--metric", + default=None, + shell_complete=_complete_metric, + help=( + "Judge dimension to compare. Defaults to the impermissible half of the " + "permissibility split when every run reports it, otherwise " + f"'{DEFAULT_COMPARE_METRIC}'." + ), +) +@click.option("--json", "as_json", is_flag=True, help="Emit machine-readable JSON instead of tables.") +@click.option("--no-color", is_flag=True, help="Disable colored terminal output.") +def results_matrix( + suite_runs: tuple[str, ...], + suites: tuple[str, ...], + results_dir: Path, + metric: str | None, + as_json: bool, + no_color: bool, +): + """Render a behavior x arm pivot over multiple runs.""" + if not suites and len(suite_runs) < 2: + _error("Provide at least two SUITE/RUN arguments to compare, or use --suite SUITE.") + + results_root = _resolve_results_dir(results_dir) + resolved_suite_runs: list[tuple[str, str]] = [] + seen_suite_runs: set[tuple[str, str]] = set() + for suite_run in suite_runs: + parsed = _parse_suite_run_arg(suite_run) + if parsed not in seen_suite_runs: + resolved_suite_runs.append(parsed) + seen_suite_runs.add(parsed) + for suite in suites: + suite_dir = results_root / suite + if not suite_dir.exists(): + _error(f"Suite not found: {suite}") + for child in sorted(suite_dir.iterdir()): + if child.is_dir() and (child / "scores.jsonl").exists(): + parsed = (suite, child.name) + if parsed not in seen_suite_runs: + resolved_suite_runs.append(parsed) + seen_suite_runs.add(parsed) + + if len(resolved_suite_runs) < 2: + _error("Provide at least two runs with scores to compare.") + + run_summaries: list[dict[str, Any]] = [] + behaviors: list[str] = [] + arms: list[str] = [] + cells: dict[str, dict[str, float | None]] = {} + cell_sources: dict[tuple[str, str], str] = {} + seen_behaviors: set[str] = set() + seen_arms: set[str] = set() + + loaded: list[tuple[str, str, dict[str, Any]]] = [] + for suite_id, run_id in resolved_suite_runs: + run_dir = results_root / suite_id / run_id + if not run_dir.exists(): + _error(f"Not found: {suite_id}/{run_id}") + run_summary = _load_run_summary(run_dir) + if run_summary is None: + _error(f"No scores in {suite_id}/{run_id}") + matrix_summary = { + "prompt_metrics": run_summary.get("prompt_metrics"), + "scenario_metrics": run_summary.get("scenario_metrics"), + } + run_summaries.append(matrix_summary) + + behavior = _run_behavior_name(run_dir, suite_id) + arm = _run_arm_label(run_id, suite_id) + source = f"{suite_id}/{run_id}" + cell_key = (behavior, arm) + if previous_source := cell_sources.get(cell_key): + _error( + f"Runs '{previous_source}' and '{source}' both resolve to " + f"behavior '{behavior}' and arm '{arm}'. Use distinct run IDs " + "or select only one run for that cell." + ) + cell_sources[cell_key] = source + if behavior not in seen_behaviors: + behaviors.append(behavior) + seen_behaviors.add(behavior) + if arm not in seen_arms: + arms.append(arm) + seen_arms.add(arm) + loaded.append((behavior, arm, matrix_summary)) + + split_by_run = [ + _run_dimension_rate(run_summary, _MATRIX_SPLIT_METRIC) is not None + for run_summary in run_summaries + ] + + # Require the exact default bucket to have a denominator in every run. + # A populated permissible bucket does not make the impermissible rate usable. + if metric is None: + metric = ( + _MATRIX_SPLIT_METRIC + if split_by_run and all(split_by_run) + else DEFAULT_COMPARE_METRIC + ) + metric = _PERMISSIBILITY_METRIC_ALIASES.get(metric, metric) + + available_metrics: set[str] = set() + for run_summary in run_summaries: + prompt_metrics = run_summary.get("prompt_metrics") + scenario_metrics = run_summary.get("scenario_metrics") + for metrics in (prompt_metrics, scenario_metrics): + dimensions = metrics.get("dimensions") if isinstance(metrics, dict) else None + if isinstance(dimensions, dict): + available_metrics.update(dimensions) + if any(split_by_run): + available_metrics.update(_DERIVED_PERMISSIBILITY_RATE_KEYS) + if metric not in available_metrics: + _error( + f"Metric '{metric}' was not found in the compared judgments. " + f"Available: {sorted(available_metrics)}" + ) + + for behavior, arm, run_summary in loaded: + cells.setdefault(behavior, {})[arm] = _run_dimension_rate(run_summary, metric) + + _reject_ordinal_compare(run_summaries, metric) + arms = _ordered_arm_labels(arms) + + if as_json: + _echo_json({ + "metric": metric, + "behaviors": behaviors, + "arms": arms, + "cells": { + behavior: { + arm: cells.get(behavior, {}).get(arm) + for arm in arms + } + for behavior in behaviors + }, + }) + return + + console = _console(no_color=no_color) + table = Table( + title=f"Behavior × arm matrix ({_metric_label(metric)})", + box=None, + show_header=True, + show_edge=False, + pad_edge=False, + ) + table.add_column("Behavior", style="cyan", no_wrap=True) + for arm in arms: + table.add_column(arm, style="white", no_wrap=True) + for behavior in behaviors: + row = [behavior] + for arm in arms: + row.append(_fmt_percent(cells.get(behavior, {}).get(arm))) + table.add_row(*row) + console.print(table) + + if metric in _DERIVED_PERMISSIBILITY_RATE_KEYS: + # Each half is scored only over the rows where a behavior in that bucket + # was relevant, so the two halves have different denominators from each + # other and from `policy_violation`. Say so, or a reader will try to add + # them and find they do not reconcile to the union. + console.print( + "[dim]Rate is over rows where a behavior in this bucket was relevant, " + "not all scored rows. The two halves of the split therefore have " + "different denominators and do not sum to policy_violation.[/dim]" + ) + + @results.command("compare-suites", short_help="Compare runs across different suites (e.g., approach A vs B vs C)") @click.argument("suite_runs", nargs=-1) @click.option( diff --git a/assert_ai/results.py b/assert_ai/results.py index 86059a895..efa1f9183 100644 --- a/assert_ai/results.py +++ b/assert_ai/results.py @@ -222,18 +222,21 @@ def compute_policy_violation_by_permissibility( if not isinstance(violated, bool): continue - node_index = node.get("node_index") - if ( - isinstance(node_index, int) - and not isinstance(node_index, bool) - and node_index in permissible_by_index - ): - permissible = permissible_by_index[node_index] - else: - node_name = str(node.get("node_name") or "").strip() + raw_node_name = node.get("node_name") + node_name = raw_node_name.strip() if isinstance(raw_node_name, str) else "" + if node_name: if node_name not in permissible_by_name: continue permissible = permissible_by_name[node_name] + else: + node_index = node.get("node_index") + if ( + not isinstance(node_index, int) + or isinstance(node_index, bool) + or node_index not in permissible_by_index + ): + continue + permissible = permissible_by_index[node_index] row_applicable[permissible] = True row_violated[permissible] = row_violated[permissible] or violated diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 682cb728c..180a4c7bf 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -119,7 +119,7 @@ assert-ai results compare / / [suite3/run3 ...] [OPT Options: - `--results-dir ` optional -- `--metric ` optional; defaults to `policy_violation_not_permissible` when every compared run has permissibility-split data, otherwise `policy_violation` +- `--metric ` optional; defaults to `policy_violation_not_permissible` when that exact metric has a non-empty denominator in every compared run, otherwise `policy_violation` - `--limit ` optional, default `8` - `--json` optional flag - `--no-color` optional flag @@ -135,10 +135,35 @@ assert-ai results compare-suites / / [OPTIONS] Options: - `--results-dir ` optional -- `--metric ` optional; defaults to `policy_violation_not_permissible` when every compared run has permissibility-split data, otherwise `policy_violation` +- `--metric ` optional; defaults to `policy_violation_not_permissible` when that exact metric has a non-empty denominator in every compared run, otherwise `policy_violation` - `--json` optional flag - `--no-color` optional flag +## `results matrix` + +Compare multiple behavior suites and target arms in one table. + +```bash +assert-ai results matrix / / [suite3/run3 ...] [OPTIONS] +assert-ai results matrix --suite [--suite ...] [OPTIONS] +``` + +Rows are behavior names, columns are arm labels derived from run IDs, and cells +are pooled rates across the run's prompt and scenario judgments. `--suite` +expands every scored run in that suite and may be repeated. + +Options: + +- `--results-dir ` optional +- `--suite ` repeatable; expands all scored runs in the suite +- `--metric ` optional; defaults to `policy_violation_not_permissible` only when that exact metric has a non-empty denominator in every run, otherwise `policy_violation` +- `--json` optional flag +- `--no-color` optional flag + +Permissibility-split rates use only rows where a behavior in that bucket was +relevant. The permissible and not-permissible rates can therefore have +different denominators and do not sum to the union `policy_violation` rate. + ## `analysis test-set-metrics` Compute test-set coverage/diversity metrics. diff --git a/docs/guides/results.md b/docs/guides/results.md index 6fc859e1d..b99e5d69d 100644 --- a/docs/guides/results.md +++ b/docs/guides/results.md @@ -56,10 +56,44 @@ assert-ai results status assert-ai results status assert-ai results compare assert-ai results compare-suites / / +assert-ai results matrix --suite --suite ``` See [CLI Commands](../cli/commands.md) for full options. +## Compare behaviors across arms + +Use `results matrix` when each behavior has multiple runs representing arms +such as `baseline`, `prompted`, and `acs`: + +```bash +assert-ai results matrix \ + --suite travel-budget \ + --suite travel-grounding +``` + +The command renders one row per behavior and one column per arm. It pools prompt +and scenario judgments from each run using their scored counts rather than +averaging two rates. You can also select runs explicitly: + +```bash +assert-ai results matrix \ + travel-budget/travel-budget-baseline \ + travel-budget/travel-budget-prompted \ + travel-grounding/travel-grounding-baseline \ + travel-grounding/travel-grounding-prompted +``` + +By default, the matrix shows `policy_violation_not_permissible` only when every +selected run has a non-empty denominator for that exact metric. Otherwise it +falls back to the union `policy_violation` rate, avoiding an all-empty table for +one-sided taxonomies. Use `--metric ` to choose another Boolean +judge dimension and `--json` for machine-readable output. + +Historical runs use the versioned taxonomy recorded in each run's +`manifest.json`. This keeps permissibility labels stable if the suite's current +taxonomy is regenerated or reordered later. + ## View evaluation suite artifacts and run results in a local UI app Access a rich inspector and editing application to view run status, evaluation suite artifacts such as richly rendered taxonomy of behavior categories and their associated policy labels. diff --git a/tests/test_cli_results_matrix.py b/tests/test_cli_results_matrix.py new file mode 100644 index 000000000..82b845882 --- /dev/null +++ b/tests/test_cli_results_matrix.py @@ -0,0 +1,900 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +from pathlib import Path +from typing import Any + +import pytest +from click.testing import CliRunner + +from assert_ai.cli import cli + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + +def _score_row(policy_violation: bool) -> dict[str, Any]: + return { + "judge_status": "ok", + "target": "test-target", + "judge_model": "test-judge", + "verdict": { + "dimensions": { + "policy_violation": policy_violation, + "overrefusal": False, + }, + "node_judgments": [], + }, + } + + +def _make_run( + results_root: Path, + suite_id: str, + run_id: str, + behavior_name: str | None, + policy_violations: list[bool], +) -> None: + run_dir = results_root / suite_id / run_id + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), + encoding="utf-8", + ) + (run_dir / "config.yaml").write_text( + "\n".join([ + "behavior:", + f" name: {behavior_name}", + ]) if behavior_name is not None else "behavior: {}\n", + encoding="utf-8", + ) + _write_jsonl(run_dir / "scores.jsonl", [_score_row(value) for value in policy_violations]) + + +def test_results_matrix_json_renders_two_behaviors_by_two_arms(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "behavior-a", "behavior-a-baseline", "behavior_a", [True, False]) + _make_run(results_root, "behavior-a", "behavior-a-prompted", "behavior_a", [False, False]) + _make_run(results_root, "behavior-b", "behavior-b-baseline", "behavior_b", [True, True]) + _make_run(results_root, "behavior-b", "behavior-b-prompted", "behavior_b", [False, True]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "behavior-a/behavior-a-baseline", + "behavior-a/behavior-a-prompted", + "behavior-b/behavior-b-baseline", + "behavior-b/behavior-b-prompted", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == { + "metric": "policy_violation", + "behaviors": ["behavior_a", "behavior_b"], + "arms": ["baseline", "prompted"], + "cells": { + "behavior_a": {"baseline": 0.5, "prompted": 0.0}, + "behavior_b": {"baseline": 1.0, "prompted": 0.5}, + }, + } + + +def test_results_matrix_missing_cell_renders_null_and_dash(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "behavior-a", "behavior-a-baseline", "behavior_a", [True]) + _make_run(results_root, "behavior-a", "behavior-a-prompted", "behavior_a", [False]) + _make_run(results_root, "behavior-b", "behavior-b-baseline", "behavior_b", [False]) + + args = [ + "results", + "matrix", + "behavior-a/behavior-a-baseline", + "behavior-a/behavior-a-prompted", + "behavior-b/behavior-b-baseline", + "--results-dir", + str(results_root), + ] + runner = CliRunner() + + json_result = runner.invoke(cli, [*args, "--json"]) + assert json_result.exit_code == 0, json_result.output + payload = json.loads(json_result.output) + assert payload["cells"]["behavior_b"]["prompted"] is None + + text_result = runner.invoke(cli, [*args, "--no-color"]) + assert text_result.exit_code == 0, text_result.output + behavior_b_row = next(line for line in text_result.output.splitlines() if "behavior_b" in line) + assert behavior_b_row.rstrip().endswith("-") + + +def test_results_matrix_suite_auto_expand_matches_explicit_args(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "suite-a", "suite-a-baseline", "behavior_a", [True, False]) + _make_run(results_root, "suite-a", "suite-a-prompted", "behavior_a", [False, False]) + + runner = CliRunner() + explicit = runner.invoke( + cli, + [ + "results", + "matrix", + "suite-a/suite-a-baseline", + "suite-a/suite-a-prompted", + "--results-dir", + str(results_root), + "--json", + ], + ) + expanded = runner.invoke( + cli, + [ + "results", + "matrix", + "--suite", + "suite-a", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert explicit.exit_code == 0, explicit.output + assert expanded.exit_code == 0, expanded.output + assert json.loads(explicit.output) == json.loads(expanded.output) + + +def test_results_matrix_repeated_suite_expands_multiple_suites_with_known_arm_order(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "suite-a", "suite-a-acs", "behavior_a", [False]) + _make_run(results_root, "suite-a", "suite-a-baseline", "behavior_a", [True]) + _make_run(results_root, "suite-b", "suite-b-prompted", "behavior_b", [True, False]) + _make_run(results_root, "suite-b", "suite-b-acs", "behavior_b", [False, False]) + _make_run(results_root, "suite-b", "suite-b-baseline", "behavior_b", [True, True]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "suite-a", + "--suite", + "suite-b", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["behaviors"] == ["behavior_a", "behavior_b"] + assert payload["arms"] == ["baseline", "prompted", "acs"] + assert payload["cells"] == { + "behavior_a": {"baseline": 1.0, "prompted": None, "acs": 0.0}, + "behavior_b": {"baseline": 1.0, "prompted": 0.5, "acs": 0.0}, + } + + +def test_results_matrix_behavior_name_falls_back_to_suite_id(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "fallback-suite", "fallback-suite-baseline", None, [True]) + _make_run(results_root, "fallback-suite", "fallback-suite-prompted", None, [False]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "fallback-suite/fallback-suite-baseline", + "fallback-suite/fallback-suite-prompted", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["behaviors"] == ["fallback-suite"] + + +def test_results_matrix_preserves_full_non_prefixed_run_ids(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run( + results_root, + "suite-a", + "variant-c-baseline-prompt", + "behavior_a", + [True], + ) + _make_run( + results_root, + "suite-b", + "baseline-weak-prompt", + "behavior_b", + [False], + ) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "suite-a/variant-c-baseline-prompt", + "suite-b/baseline-weak-prompt", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + arms = json.loads(result.output)["arms"] + assert arms == ["baseline-weak-prompt", "variant-c-baseline-prompt"] + assert "prompt" not in arms + + +def test_results_matrix_rejects_duplicate_behavior_arm_cells(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "suite-a", "suite-a-baseline", "shared_behavior", [True]) + _make_run(results_root, "suite-b", "suite-b-baseline", "shared_behavior", [False]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "suite-a/suite-a-baseline", + "suite-b/suite-b-baseline", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 1 + assert "suite-a/suite-a-baseline" in result.output + assert "suite-b/suite-b-baseline" in result.output + assert "behavior 'shared_behavior' and arm 'baseline'" in result.output + + +def test_results_matrix_rejects_unknown_metric(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "beh", "beh-baseline", "beh", [True]) + _make_run(results_root, "beh", "beh-prompted", "beh", [False]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "beh", + "--results-dir", + str(results_root), + "--metric", + "policy_violaton", + "--json", + ], + ) + + assert result.exit_code == 1 + assert "Metric 'policy_violaton' was not found" in result.output + assert "policy_violation" in result.output + + +def test_results_matrix_malformed_config_falls_back_to_manifest(tmp_path: Path) -> None: + results_root = tmp_path / "results" + for run_id, flagged in (("suite-a-baseline", True), ("suite-a-prompted", False)): + _make_run(results_root, "suite-a", run_id, None, [flagged]) + run_dir = results_root / "suite-a" / run_id + (run_dir / "config.yaml").write_text("behavior: [\n", encoding="utf-8") + (run_dir / "manifest.json").write_text( + json.dumps({ + "status": "completed", + "stages": {"judge": "completed"}, + "behavior_name": "manifest_behavior", + }), + encoding="utf-8", + ) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "suite-a", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["behaviors"] == ["manifest_behavior"] + + +def test_results_matrix_deduplicates_explicit_and_suite_expansion(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "suite-a", "suite-a-baseline", "behavior_a", [True]) + _make_run(results_root, "suite-a", "suite-a-prompted", "behavior_a", [False]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "suite-a/suite-a-baseline", + "--suite", + "suite-a", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["cells"] == { + "behavior_a": {"baseline": 1.0, "prompted": 0.0} + } + + +def test_results_matrix_missing_suite_is_a_cli_error(tmp_path: Path) -> None: + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "missing-suite", + "--results-dir", + str(tmp_path / "results"), + ], + ) + + assert result.exit_code == 1 + assert "Suite not found: missing-suite" in result.output + + +# --- permissibility split ------------------------------------------------- +# +# `policy_violation` unions permissible and impermissible behaviors, so ranking +# behaviors by it can order them by the wrong thing entirely. The matrix +# therefore leads with the impermissible half when every run reports it. +# +# The split is derived from node judgments plus the taxonomy and is stored as a +# top-level rate, not under `dimensions`. Asking for it used to render an empty +# cell under a correct-looking "Impermissible behavior violated" heading, which +# reads as "no violations" rather than "not wired up". + + +def _split_score_row(*, impermissible: bool, permissible: bool) -> dict[str, Any]: + """A row whose node judgments cover both buckets.""" + return { + "judge_status": "ok", + "target": "test-target", + "judge_model": "test-judge", + "verdict": { + "dimensions": { + "policy_violation": impermissible or permissible, + "overrefusal": False, + }, + "node_judgments": [ + {"node_index": 0, "node_name": "must never", "relevant": True, "violated": impermissible}, + {"node_index": 1, "node_name": "allowed", "relevant": True, "violated": permissible}, + ], + }, + } + + +def _make_split_run( + results_root: Path, + suite_id: str, + run_id: str, + behavior_name: str, + rows: list[tuple[bool, bool]], + *, with_taxonomy: bool = True, +) -> None: + run_dir = results_root / suite_id / run_id + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), encoding="utf-8" + ) + (run_dir / "config.yaml").write_text(f"behavior:\n name: {behavior_name}\n", encoding="utf-8") + if with_taxonomy: + (run_dir.parent / "taxonomy.json").write_text( + json.dumps({ + "behavior_categories": [ + {"name": "must never", "permissible": False}, + {"name": "allowed", "permissible": True}, + ] + }), + encoding="utf-8", + ) + _write_jsonl( + run_dir / "scores.jsonl", + [_split_score_row(impermissible=i, permissible=p) for i, p in rows], + ) + + +def _set_systematize_artifact(run_dir: Path, path: str) -> None: + manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8")) + manifest["artifact_versions"] = { + "systematize": { + "version": "v0001", + "path": path, + } + } + (run_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + + +def test_matrix_defaults_to_the_impermissible_half_when_every_run_has_the_split(tmp_path: Path) -> None: + results_root = tmp_path / "results" + # 1 of 4 impermissible, 3 of 4 permissible: the union would rank this high + # for the wrong reason. + _make_split_run(results_root, "beh", "beh-baseline", "beh", + [(True, True), (False, True), (False, True), (False, False)]) + _make_split_run(results_root, "beh", "beh-governed", "beh", + [(False, False), (False, False), (False, True), (False, False)]) + + result = CliRunner().invoke( + cli, ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), "--json"] + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation_not_permissible" + assert payload["cells"]["beh"]["baseline"] == 0.25 + assert payload["cells"]["beh"]["governed"] == 0.0 + + +def test_matrix_falls_back_to_union_for_all_permissible_taxonomy(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_split_run( + results_root, + "beh", + "beh-baseline", + "allowed_behavior", + [(False, True), (False, False)], + ) + _make_split_run( + results_root, + "beh", + "beh-governed", + "allowed_behavior", + [(False, False), (False, False)], + ) + (results_root / "beh" / "taxonomy.json").write_text( + json.dumps({ + "behavior_categories": [ + {"name": "allowed", "permissible": True}, + ] + }), + encoding="utf-8", + ) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), "--json"], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation" + assert payload["cells"]["allowed_behavior"] == { + "baseline": 0.5, + "governed": 0.0, + } + + +def test_matrix_uses_each_runs_versioned_taxonomy_after_suite_reordering(tmp_path: Path) -> None: + results_root = tmp_path / "results" + for run_id, violated in (("beh-baseline", True), ("beh-governed", False)): + _make_split_run(results_root, "beh", run_id, "beh", [(violated, False)]) + + suite_dir = results_root / "beh" + versioned_path = suite_dir / "artifacts" / "systematize" / "v0001" / "taxonomy.json" + versioned_path.parent.mkdir(parents=True) + versioned_path.write_text( + json.dumps({ + "behavior_categories": [ + {"name": "must never", "permissible": False}, + {"name": "allowed", "permissible": True}, + ] + }), + encoding="utf-8", + ) + for run_id in ("beh-baseline", "beh-governed"): + _set_systematize_artifact( + suite_dir / run_id, + "artifacts/systematize/v0001/taxonomy.json", + ) + + (suite_dir / "taxonomy.json").write_text( + json.dumps({ + "behavior_categories": [ + {"name": "allowed", "permissible": True}, + {"name": "must never", "permissible": False}, + ] + }), + encoding="utf-8", + ) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), "--json"], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation_not_permissible" + assert payload["cells"]["beh"] == { + "baseline": 1.0, + "governed": 0.0, + } + + +def test_matrix_uses_valid_node_name_over_stale_node_index(tmp_path: Path) -> None: + results_root = tmp_path / "results" + for run_id, violated in (("beh-baseline", True), ("beh-governed", False)): + _make_split_run(results_root, "beh", run_id, "beh", [(violated, False)]) + scores_path = results_root / "beh" / run_id / "scores.jsonl" + row = _split_score_row(impermissible=violated, permissible=False) + row["verdict"]["node_judgments"][0]["node_index"] = 1 + row["verdict"]["node_judgments"][1]["node_index"] = 0 + _write_jsonl(scores_path, [row]) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), "--json"], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation_not_permissible" + assert payload["cells"]["beh"]["baseline"] == 1.0 + + +def test_matrix_rejects_versioned_taxonomy_path_outside_suite(tmp_path: Path) -> None: + results_root = tmp_path / "results" + outside_taxonomy = results_root / "outside-taxonomy.json" + outside_taxonomy.parent.mkdir(parents=True) + outside_taxonomy.write_text( + json.dumps({ + "behavior_categories": [ + {"name": "must never", "permissible": False}, + {"name": "allowed", "permissible": True}, + ] + }), + encoding="utf-8", + ) + for run_id, violated in (("beh-baseline", True), ("beh-governed", False)): + _make_split_run( + results_root, + "beh", + run_id, + "beh", + [(violated, False)], + with_taxonomy=False, + ) + _set_systematize_artifact( + results_root / "beh" / run_id, + "../outside-taxonomy.json", + ) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), "--json"], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation" + assert payload["cells"]["beh"]["baseline"] == 1.0 + + +def test_matrix_split_cells_are_populated_not_dashes(tmp_path: Path) -> None: + """The regression: the metric resolved and labelled, but every cell was None.""" + results_root = tmp_path / "results" + _make_split_run(results_root, "beh", "beh-baseline", "beh", [(True, False), (False, True)]) + _make_split_run(results_root, "beh", "beh-governed", "beh", [(False, False), (False, True)]) + + for metric in ("policy_violation_not_permissible", "policy_violation_permissible"): + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), + "--metric", metric, "--json"], + ) + assert result.exit_code == 0, result.output + cells = json.loads(result.output)["cells"]["beh"] + assert all(value is not None for value in cells.values()), (metric, cells) + + +def test_matrix_accepts_the_artifact_key_spelling_of_the_split(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_split_run(results_root, "beh", "beh-baseline", "beh", [(True, False), (False, False)]) + _make_split_run(results_root, "beh", "beh-governed", "beh", [(False, False), (False, False)]) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), + "--metric", "not_permissible_policy_violation_rate", "--json"], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation_not_permissible" + assert payload["cells"]["beh"]["baseline"] == 0.5 + + +def test_matrix_falls_back_to_policy_violation_without_a_taxonomy(tmp_path: Path) -> None: + """Quality suites repurpose policy_violation and have no taxonomy; they must + keep reporting the union rather than a table of blanks.""" + results_root = tmp_path / "results" + _make_split_run(results_root, "beh", "beh-baseline", "beh", + [(True, False), (False, False)], with_taxonomy=False) + _make_split_run(results_root, "beh", "beh-governed", "beh", + [(False, False), (False, False)], with_taxonomy=False) + + result = CliRunner().invoke( + cli, ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), "--json"] + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation" + assert payload["cells"]["beh"]["baseline"] == 0.5 + + +def test_matrix_falls_back_when_taxonomy_no_longer_matches_judgments(tmp_path: Path) -> None: + results_root = tmp_path / "results" + for run_id, violated in (("beh-baseline", True), ("beh-governed", False)): + _make_split_run( + results_root, + "beh", + run_id, + "beh", + [(violated, False)], + ) + row = _split_score_row(impermissible=violated, permissible=False) + for index, node in enumerate(row["verdict"]["node_judgments"]): + node["node_index"] = index + 10 + node["node_name"] = f"stale-{index}" + _write_jsonl(results_root / "beh" / run_id / "scores.jsonl", [row]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "beh", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation" + assert payload["cells"]["beh"]["baseline"] == 1.0 + + +def test_matrix_does_not_mix_halves_when_only_some_runs_have_the_split(tmp_path: Path) -> None: + """One run contributing an impermissible-only rate while another contributes + the union would put non-comparable numbers in one table.""" + results_root = tmp_path / "results" + _make_split_run(results_root, "with-tax", "with-tax-baseline", "beh_a", [(True, False)]) + _make_split_run(results_root, "no-tax", "no-tax-baseline", "beh_b", + [(True, False)], with_taxonomy=False) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "with-tax/with-tax-baseline", "no-tax/no-tax-baseline", + "--results-dir", str(results_root), "--json"], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["metric"] == "policy_violation" + + +def test_matrix_warns_that_split_halves_have_different_denominators(tmp_path: Path) -> None: + """Each half is scored only over rows where its bucket was relevant, so the + halves do not sum to policy_violation.""" + results_root = tmp_path / "results" + _make_split_run(results_root, "beh", "beh-baseline", "beh", [(True, False), (False, True)]) + _make_split_run(results_root, "beh", "beh-governed", "beh", [(False, False), (False, True)]) + + result = CliRunner().invoke( + cli, ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root)] + ) + + assert result.exit_code == 0, result.output + assert "denominator" in result.output.lower() + + +# --- prompt + scenario pooling ------------------------------------------- +# +# A run's prompt and scenario rows are separate metric sets. Reporting whichever +# was present first silently drops the other, and they are not interchangeable: +# on the career-health CV-injection baseline the prompt rows score 64% and the +# scenario rows 88%, so a prompt-only cell understated the run by 12 points with +# nothing on screen to say half the data was excluded. + + +def _mixed_run( + results_root: Path, + suite_id: str, + run_id: str, + behavior_name: str, + prompt_flags: list[bool], + scenario_flags: list[bool], +) -> None: + run_dir = results_root / suite_id / run_id + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), encoding="utf-8" + ) + (run_dir / "config.yaml").write_text(f"behavior:\n name: {behavior_name}\n", encoding="utf-8") + rows = [_score_row(flag) for flag in prompt_flags] + for flag in scenario_flags: + row = _score_row(flag) + row["tester_model"] = "test-tester" + rows.append(row) + _write_jsonl(run_dir / "scores.jsonl", rows) + + +def test_matrix_pools_prompt_and_scenario_rows(tmp_path: Path) -> None: + results_root = tmp_path / "results" + # 1/4 prompt + 3/4 scenario = 4/8 pooled. Prompt-only would report 0.25. + _mixed_run(results_root, "beh", "beh-baseline", "beh", + [True, False, False, False], [True, True, True, False]) + _mixed_run(results_root, "beh", "beh-governed", "beh", + [False, False, False, False], [False, False, False, False]) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), + "--metric", "policy_violation", "--json"], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["cells"]["beh"]["baseline"] == 0.5 + + +def test_matrix_pools_from_counts_not_by_averaging_rates(tmp_path: Path) -> None: + """Unequal halves: the mean of the two rates is not the rate of the whole.""" + results_root = tmp_path / "results" + # 1/1 prompt (100%) + 1/9 scenario (11.1%) = 2/10 pooled (20%). + # Averaging the two rates would give 55.6%. + _mixed_run(results_root, "beh", "beh-baseline", "beh", + [True], [True] + [False] * 8) + _mixed_run(results_root, "beh", "beh-governed", "beh", [False], [False]) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), + "--metric", "policy_violation", "--json"], + ) + + assert result.exit_code == 0, result.output + baseline = json.loads(result.output)["cells"]["beh"]["baseline"] + assert baseline == pytest.approx(0.2), baseline + + +def test_matrix_pools_the_permissibility_split_across_both_halves(tmp_path: Path) -> None: + results_root = tmp_path / "results" + run_dir = results_root / "beh" / "beh-baseline" + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), encoding="utf-8" + ) + (run_dir / "config.yaml").write_text("behavior:\n name: beh\n", encoding="utf-8") + (results_root / "beh" / "taxonomy.json").write_text( + json.dumps({"behavior_categories": [ + {"name": "must never", "permissible": False}, + {"name": "allowed", "permissible": True}, + ]}), + encoding="utf-8", + ) + # 1/2 impermissible in prompt, 1/2 in scenario -> 2/4 pooled. + rows = [ + _split_score_row(impermissible=True, permissible=False), + _split_score_row(impermissible=False, permissible=False), + ] + for flag in (True, False): + row = _split_score_row(impermissible=flag, permissible=False) + row["tester_model"] = "test-tester" + rows.append(row) + _write_jsonl(run_dir / "scores.jsonl", rows) + + _make_split_run(results_root, "beh2", "beh2-baseline", "beh2", [(False, False)]) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "beh/beh-baseline", "beh2/beh2-baseline", + "--results-dir", str(results_root), "--json"], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation_not_permissible" + assert payload["cells"]["beh"]["baseline"] == 0.5 + + +def test_matrix_matches_cv_injection_prompt_scenario_totals(tmp_path: Path) -> None: + results_root = tmp_path / "results" + run_dir = results_root / "cv-injection" / "cv-injection-baseline" + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), + encoding="utf-8", + ) + (run_dir / "config.yaml").write_text( + "behavior:\n name: cv_injection\n", + encoding="utf-8", + ) + (run_dir.parent / "taxonomy.json").write_text( + json.dumps({ + "behavior_categories": [ + {"name": "must never", "permissible": False}, + {"name": "allowed", "permissible": True}, + ] + }), + encoding="utf-8", + ) + + rows: list[dict[str, Any]] = [] + for index in range(25): + row = _split_score_row( + impermissible=index < 4, + permissible=index < 16, + ) + if index >= 21: + row["verdict"]["node_judgments"][0]["relevant"] = False + rows.append(row) + for index in range(25): + row = _split_score_row( + impermissible=index < 18, + permissible=index < 22, + ) + row["tester_model"] = "test-tester" + rows.append(row) + _write_jsonl(run_dir / "scores.jsonl", rows) + _make_split_run( + results_root, + "control", + "control-baseline", + "control", + [(False, False)], + ) + + base_args = [ + "results", + "matrix", + "cv-injection/cv-injection-baseline", + "control/control-baseline", + "--results-dir", + str(results_root), + "--json", + ] + runner = CliRunner() + union_result = runner.invoke(cli, [*base_args, "--metric", "policy_violation"]) + split_result = runner.invoke(cli, base_args) + + assert union_result.exit_code == 0, union_result.output + assert split_result.exit_code == 0, split_result.output + union = json.loads(union_result.output) + split = json.loads(split_result.output) + assert union["cells"]["cv_injection"]["baseline"] == pytest.approx(38 / 50) + assert split["cells"]["cv_injection"]["baseline"] == pytest.approx(22 / 46) diff --git a/tests/test_results.py b/tests/test_results.py index 51e7ae0bb..115c712b3 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -520,6 +520,37 @@ def test_compare_defaults_to_impermissible_split_and_accepts_permissible_split(s self.assertEqual(permissible_result.exit_code, 0, permissible_result.output) self.assertIn("Run Comparison (metrics-suite, Permissible behavior violated)", permissible_result.output) + def test_compare_falls_back_to_union_when_impermissible_bucket_is_empty(self) -> None: + with TemporaryDirectory() as tmp_dir: + results_root = Path(tmp_dir) / "results" + _write_split_results(results_root) + (results_root / "metrics-suite" / "taxonomy.json").write_text( + json.dumps({ + "behavior_categories": [ + {"name": "allowed", "permissible": True}, + ] + }), + encoding="utf-8", + ) + + result = self.runner.invoke( + cli, + [ + "results", + "compare", + "metrics-suite", + "run-1", + "run-2", + "--results-dir", + str(results_root), + "--no-color", + ], + terminal_width=180, + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("Run Comparison (metrics-suite, Policy violation)", result.output) + def test_compare_falls_back_when_any_run_lacks_current_split_data(self) -> None: with TemporaryDirectory() as tmp_dir: results_root = Path(tmp_dir) / "results"