Add reporting visualization command

demo/reporting_example_results.jsonl

@@ -0,0 +1,6 @@
+{"key": "utt_001", "model": "baseline", "pesq": 2.1, "stoi": 0.72, "wer": 0.18, "spk_similarity": 0.81, "mcd": 6.4}
+{"key": "utt_002", "model": "baseline", "pesq": 2.4, "stoi": 0.75, "wer": 0.15, "spk_similarity": 0.84, "mcd": 5.9}
+{"key": "utt_003", "model": "baseline", "pesq": 1.9, "stoi": 0.69, "wer": 0.23, "spk_similarity": 0.79, "mcd": 7.1}
+{"key": "utt_004", "model": "candidate", "pesq": 3.2, "stoi": 0.86, "wer": 0.09, "spk_similarity": 0.88, "mcd": 4.8}
+{"key": "utt_005", "model": "candidate", "pesq": 3.4, "stoi": 0.88, "wer": 0.07, "spk_similarity": 0.91, "mcd": 4.5}
+{"key": "utt_006", "model": "candidate", "pesq": 2.8, "stoi": 0.82, "wer": 0.12, "spk_similarity": 0.86, "mcd": 5.2}

demo/run_reporting_example.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RESULTS="${SCRIPT_DIR}/reporting_example_results.jsonl"
+OUT_DIR="${TMPDIR:-/tmp}/versa-reporting-example"
+
+mkdir -p "${OUT_DIR}"
+
+versa-visualize "${RESULTS}" \
+  --out "${OUT_DIR}/report.html" \
+  --csv "${OUT_DIR}/report.csv" \
+  --markdown "${OUT_DIR}/report.md" \
+  --group-by model
+
+echo "Wrote example reports to ${OUT_DIR}"
+echo "Open ${OUT_DIR}/report.html to inspect the visualization report."

docs/visualization.md

@@ -1,4 +1,45 @@
 ## Interactive Visualization of Versa Results
+
+### Packaged Report Command
+
+VERSA can generate a first-class report directly from a scoring JSONL file or a
+directory of result files:
+
+```
+versa-visualize results.jsonl --out report.html
+```
+
+The HTML report includes a radar overview, category sunburst, summary tables,
+mean/std, 95% confidence intervals, failure counts, best/worst examples, and
+outlier examples. CSV and Markdown exports are also available:
+
+```
+versa-visualize results.jsonl --out report.html --csv report.csv --markdown report.md
+```
+
+For model comparisons, group records by a field and VERSA will add per-metric
+rankings:
+
+```
+versa-visualize results.jsonl --out report.html --group-by model
+```
+
+The aggregation command can also write summary reports:
+
+```
+versa-aggregate results.jsonl --out metrics_report.csv
+versa-aggregate results.jsonl --out metrics_report.md --format md
+```
+
+For a quick smoke test after installation, run the bundled toy example:
+
+```
+demo/run_reporting_example.sh
+```
+
+It reads ``demo/reporting_example_results.jsonl`` and writes an HTML report plus
+CSV/Markdown summaries to ``${TMPDIR:-/tmp}/versa-reporting-example``.
+
 ### Steps
 * Additional Package Dependency Installation
 ```

pyproject.toml

@@ -124,6 +124,8 @@ Homepage = "https://github.com/wavlab-speech/versa.git"
 
 [project.scripts]
 versa-score = "versa.bin.scorer:main"
+versa-aggregate = "versa.bin.aggregate_results:main"
+versa-visualize = "versa.bin.visualize:main"
 
 [tool.setuptools.packages.find]
 include = ["versa*"]

test/test_reporting.py

@@ -0,0 +1,74 @@
+import csv
+
+from versa.reporting import (
+    analyze_records,
+    metric_category,
+    read_result_records,
+    write_csv_report,
+    write_html_report,
+    write_markdown_report,
+)
+
+
+def test_analyze_records_computes_ci_failures_rankings_and_outliers():
+    records = [
+        {"key": "a", "model": "m1", "pesq": 2.0, "wer": 0.2},
+        {"key": "b", "model": "m1", "pesq": 2.5, "wer": 0.1},
+        {"key": "c", "model": "m2", "pesq": 4.0, "wer": 0.4},
+        {"key": "d", "model": "m2", "pesq": "bad", "wer": 2.0},
+        {"key": "e", "model": "m2", "wer": 0.3},
+    ]
+
+    analysis = analyze_records(records, group_by="model")
+    summaries = {summary.name: summary for summary in analysis["metrics"]}
+
+    assert analysis["record_count"] == 5
+    assert summaries["pesq"].count == 3
+    assert summaries["pesq"].missing == 1
+    assert summaries["pesq"].invalid == 1
+    assert summaries["pesq"].ci95_high > summaries["pesq"].ci95_low
+    assert summaries["pesq"].best_key == "c"
+    assert summaries["wer"].best_key == "b"
+    assert summaries["wer"].worst_key == "d"
+    assert analysis["groups"]["rankings"]["pesq"][0]["group"] == "m2"
+    assert analysis["groups"]["rankings"]["wer"][0]["group"] == "m1"
+
+
+def test_read_result_records_accepts_jsonl_and_python_literals(tmp_path):
+    result_file = tmp_path / "results.txt"
+    result_file.write_text(
+        '{"key": "a", "pesq": 2.0}\n' "{'key': 'b', 'sir': inf}\n",
+        encoding="utf-8",
+    )
+
+    records = read_result_records(str(result_file))
+
+    assert [record["key"] for record in records] == ["a", "b"]
+    assert records[0]["_source_file"] == "results.txt"
+
+
+def test_report_exports(tmp_path):
+    records = [
+        {"key": "a", "pesq": 2.0, "wer": 0.2},
+        {"key": "b", "pesq": 3.0, "wer": 0.1},
+    ]
+    analysis = analyze_records(records)
+    csv_path = tmp_path / "report.csv"
+    md_path = tmp_path / "report.md"
+    html_path = tmp_path / "report.html"
+
+    write_csv_report(analysis, str(csv_path))
+    write_markdown_report(analysis, str(md_path))
+    write_html_report(analysis, str(html_path))
+
+    with csv_path.open(newline="", encoding="utf-8") as handle:
+        rows = list(csv.DictReader(handle))
+    assert {row["metric"] for row in rows} == {"pesq", "wer"}
+    assert "VERSA Results Report" in md_path.read_text(encoding="utf-8")
+    assert "Radar Overview" in html_path.read_text(encoding="utf-8")
+    assert "Category Sunburst" in html_path.read_text(encoding="utf-8")
+
+
+def test_metric_category_strips_model_prefixes():
+    assert metric_category("arecho_pesq") == "speech_enhancement"
+    assert metric_category("custom_wer") == "asr_wer_cer"

versa/__init__.py

@@ -20,6 +20,13 @@
     flag in sys.argv
     for flag in ("--list-metrics", "--describe-metric", "--recommend-config")
 )
+_SKIP_OPTIONAL_METRIC_IMPORTS = _SKIP_OPTIONAL_METRIC_IMPORTS or any(
+    Path(arg).name
+    in {"versa-visualize", "versa-aggregate", "visualize.py", "aggregate_results.py"}
+    or arg.endswith("versa.bin.visualize")
+    or arg.endswith("versa.bin.aggregate_results")
+    for arg in sys.argv
+)
 
 
 def _optional_metric_import(module_name, names, install_hint=None):

versa/bin/aggregate_results.py

@@ -3,39 +3,70 @@
 # Copyright 2024 Jiatong Shi
 #  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
 
-"""Aggregate results."""
+"""Aggregate and report VERSA results."""
 
 import argparse
 import json
 import logging
+import os
 
 from tqdm import tqdm
 
+from versa.reporting import (
+    analyze_records,
+    read_result_records,
+    write_csv_report,
+    write_html_report,
+    write_markdown_report,
+)
+
 
 def get_parser() -> argparse.Namespace:
     """Get parser of aggregate results."""
     parser = argparse.ArgumentParser(
-        description="Aggregate results.",
+        description="Aggregate chunked results or generate polished result reports.",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
+    parser.add_argument(
+        "input",
+        nargs="?",
+        help="Input JSONL result file or directory. If omitted, --logdir/--scoredir/--nj mode is used.",
+    )
     parser.add_argument(
         "--logdir",
         type=str,
-        required=True,
+        default=None,
         help="Input log directory.",
     )
     parser.add_argument(
         "--scoredir",
         type=str,
-        required=True,
+        default=None,
         help="Output scoring directory.",
     )
     parser.add_argument(
         "--nj",
         type=int,
-        required=True,
+        default=None,
         help="Number of sub jobs",
     )
+    parser.add_argument(
+        "--out",
+        type=str,
+        default=None,
+        help="Output report path for input mode.",
+    )
+    parser.add_argument(
+        "--format",
+        choices=["auto", "csv", "md", "html"],
+        default="auto",
+        help="Report format for input mode.",
+    )
+    parser.add_argument(
+        "--group-by",
+        default=None,
+        help="Optional record field used for per-metric ranking.",
+    )
     return parser
 
 
@@ -61,11 +92,56 @@ def aggregate_results(logdir: str, scoredir: str, nj: int) -> None:
     logging.info("Done.")
 
 
+def generate_report(
+    input_path: str,
+    output_path: str,
+    report_format: str = "auto",
+    group_by: str = None,
+) -> None:
+    """Generate a CSV, Markdown, or HTML report from result records."""
+    records = read_result_records(input_path)
+    analysis = analyze_records(records, group_by=group_by)
+
+    if report_format == "auto":
+        suffix = os.path.splitext(output_path)[1].lower()
+        report_format = {
+            ".csv": "csv",
+            ".md": "md",
+            ".markdown": "md",
+            ".html": "html",
+            ".htm": "html",
+        }.get(suffix, "csv")
+
+    os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True)
+    if report_format == "csv":
+        write_csv_report(analysis, output_path)
+    elif report_format == "md":
+        write_markdown_report(analysis, output_path)
+    else:
+        write_html_report(analysis, output_path)
+
+    logging.info(
+        "Wrote %s report for %s utterances and %s metrics to %s",
+        report_format,
+        analysis["record_count"],
+        analysis["metric_count"],
+        output_path,
+    )
+
+
 def main() -> None:
     """Run main function."""
     parser = get_parser()
     args = parser.parse_args()
     logging.basicConfig(level=logging.INFO)
+    if args.input:
+        output_path = args.out or "metrics_report.csv"
+        generate_report(args.input, output_path, args.format, args.group_by)
+        return
+
+    if args.logdir is None or args.scoredir is None or args.nj is None:
+        parser.error("either provide input, or provide --logdir, --scoredir, and --nj")
+
     aggregate_results(args.logdir, args.scoredir, args.nj)