diff --git a/docs-site/benchmark/scoring.mdx b/docs-site/benchmark/scoring.mdx index 0707213..f7bb088 100644 --- a/docs-site/benchmark/scoring.mdx +++ b/docs-site/benchmark/scoring.mdx @@ -58,3 +58,10 @@ Libraries are grouped by overall fidelity + capability: | C | below 60% fidelity or limited scope | See [live dashboard](https://excelbench.vercel.app) for current tier assignments. + +## Interpreting failed vs unsupported +- **Failed** means the adapter attempted the operation but produced mismatched output. +- **Unsupported** means the adapter explicitly cannot perform that operation; these are visually distinct in HTML dashboards and should be triaged as coverage gaps first. + +## Delta Since Last Run +Dashboard and report outputs now include a **Delta Since Last Run** summary sourced from `history.jsonl` in each results directory. diff --git a/docs-site/cli/report.mdx b/docs-site/cli/report.mdx index 266a9c7..0704e0a 100644 --- a/docs-site/cli/report.mdx +++ b/docs-site/cli/report.mdx @@ -64,3 +64,8 @@ uv run excelbench scatter ``` Generates fidelity-vs-throughput scatter plots as PNG and SVG, broken down by tier. + +## Delta Since Last Run +`excelbench report` and dashboard layers read `results/*/history.jsonl` and emit a compact run-over-run delta summary: +- fidelity score changes (improvements vs regressions) +- perf median read/write throughput percentage changes when perf history exists diff --git a/src/excelbench/results/dashboard.py b/src/excelbench/results/dashboard.py index 4f544d3..0f2b73f 100644 --- a/src/excelbench/results/dashboard.py +++ b/src/excelbench/results/dashboard.py @@ -35,7 +35,12 @@ def render_dashboard( perf_data = json.load(f) perf_data = filter_report_data(perf_data) - lines = _build_dashboard(fidelity_data, perf_data) + lines = _build_dashboard( + fidelity_data, + perf_data, + fidelity_history_path=fidelity_json.parent / "history.jsonl", + perf_history_path=perf_json.parent / "history.jsonl" if perf_json else None, + ) output_path.parent.mkdir(parents=True, exist_ok=True) with open(output_path, "w") as f: @@ -45,6 +50,9 @@ def render_dashboard( def _build_dashboard( fidelity: dict[str, Any], perf: dict[str, Any] | None, + *, + fidelity_history_path: Path | None = None, + perf_history_path: Path | None = None, ) -> list[str]: """Build combined dashboard markdown lines.""" fidelity = filter_report_data(fidelity) @@ -61,6 +69,7 @@ def _build_dashboard( lines.append("> Combined fidelity and performance view. Fidelity shows correctness;") lines.append("> throughput shows speed. Use this to find the right library for your needs.") lines.append("") + lines.extend(_render_delta_since_last_run(fidelity_history_path, perf_history_path)) # ── Compute per-library fidelity stats ── lib_stats = _compute_fidelity_stats(fidelity) @@ -128,6 +137,30 @@ def _build_dashboard( return lines +def _render_delta_since_last_run( + fidelity_history_path: Path | None, + perf_history_path: Path | None, +) -> list[str]: + fidelity_delta = _compute_history_delta_summary(fidelity_history_path) + perf_delta = _compute_perf_delta_summary(perf_history_path) + if not fidelity_delta and not perf_delta: + return [] + lines = ["## Delta Since Last Run", ""] + if fidelity_delta: + lines.append( + "- Fidelity score changes: " + f"**{fidelity_delta['changes']}** (improvements: **{fidelity_delta['improvements']}**, " + f"regressions: **{fidelity_delta['regressions']}**)" + ) + if perf_delta: + if "read_ops_per_sec" in perf_delta: + lines.append(f"- Median read throughput: **{perf_delta['read_ops_per_sec']:+d}%**") + if "write_ops_per_sec" in perf_delta: + lines.append(f"- Median write throughput: **{perf_delta['write_ops_per_sec']:+d}%**") + lines.append("") + return lines + + def _compute_fidelity_stats(data: dict[str, Any]) -> dict[str, dict[str, Any]]: """Compute per-library fidelity statistics from results JSON.""" libs_info = data.get("libraries", {}) @@ -186,6 +219,99 @@ def _compute_fidelity_stats(data: dict[str, Any]) -> dict[str, dict[str, Any]]: return out +def _load_recent_history(history_path: Path | None) -> tuple[dict[str, Any], dict[str, Any]] | None: + if history_path is None or not history_path.exists(): + return None + entries: list[dict[str, Any]] = [] + for line in history_path.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + parsed = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + entries.append(parsed) + if len(entries) < 2: + return None + return entries[-2], entries[-1] + + +def _compute_history_delta_summary(history_path: Path | None) -> dict[str, int] | None: + pair = _load_recent_history(history_path) + if pair is None: + return None + previous, current = pair + changes = regressions = improvements = 0 + for item in _compute_fidelity_deltas(previous, current): + changes += 1 + if item["delta"] < 0: + regressions += 1 + elif item["delta"] > 0: + improvements += 1 + return {"changes": changes, "improvements": improvements, "regressions": regressions} + + +def _compute_perf_delta_summary(history_path: Path | None) -> dict[str, int] | None: + pair = _load_recent_history(history_path) + if pair is None: + return None + previous, current = pair + out: dict[str, int] = {} + for key, op_key in (("read_ops_per_sec", "read_p50"), ("write_ops_per_sec", "write_p50")): + deltas = _compute_perf_p50_deltas(previous, current, op_key) + if deltas: + out[key] = round(sorted(deltas)[len(deltas) // 2]) + return out or None + + +def _compute_perf_p50_deltas( + previous: dict[str, Any], current: dict[str, Any], op_key: str +) -> list[float]: + deltas: list[float] = [] + prev_wall = previous.get("p50_wall_ms", {}) + curr_wall = current.get("p50_wall_ms", {}) + if not isinstance(prev_wall, dict) or not isinstance(curr_wall, dict): + return deltas + for library in set(prev_wall) | set(curr_wall): + prev_lib = prev_wall.get(library, {}) + curr_lib = curr_wall.get(library, {}) + if not isinstance(prev_lib, dict) or not isinstance(curr_lib, dict): + continue + for feature in set(prev_lib) | set(curr_lib): + prev_val = (prev_lib.get(feature) or {}).get(op_key) + curr_val = (curr_lib.get(feature) or {}).get(op_key) + if ( + isinstance(prev_val, (int, float)) + and isinstance(curr_val, (int, float)) + and prev_val != 0 + ): + deltas.append((prev_val - curr_val) / prev_val * 100) + return deltas + + +def _compute_fidelity_deltas( + previous: dict[str, Any], current: dict[str, Any] +) -> list[dict[str, int]]: + deltas: list[dict[str, int]] = [] + prev_scores: dict[str, Any] = previous.get("scores", {}) + curr_scores: dict[str, Any] = current.get("scores", {}) + for library in sorted(set(prev_scores) | set(curr_scores)): + prev_lib = prev_scores.get(library, {}) + curr_lib = curr_scores.get(library, {}) + for feature in sorted(set(prev_lib) | set(curr_lib)): + prev_feature = prev_lib.get(feature, {}) + curr_feature = curr_lib.get(feature, {}) + for mode in ("read", "write"): + prev_value = prev_feature.get(mode) + curr_value = curr_feature.get(mode) + if prev_value is None or curr_value is None or prev_value == curr_value: + continue + deltas.append({"delta": int(curr_value) - int(prev_value)}) + return deltas + + def _compute_throughput(data: dict[str, Any]) -> dict[str, dict[str, float | None]]: """Extract representative throughput (cells/s) per library from perf results. diff --git a/src/excelbench/results/failure_explainer.py b/src/excelbench/results/failure_explainer.py index 2c78db9..20247d7 100644 --- a/src/excelbench/results/failure_explainer.py +++ b/src/excelbench/results/failure_explainer.py @@ -18,6 +18,7 @@ class FailureExplanation: summary: str probable_cause: str next_step: str + tag: str def to_json_dict(self) -> JSONDict: return { @@ -25,6 +26,7 @@ def to_json_dict(self) -> JSONDict: "summary": self.summary, "probable_cause": self.probable_cause, "next_step": self.next_step, + "tag": self.tag, } @@ -51,6 +53,7 @@ def explain_diagnostic( summary=diagnostic.probable_cause or diagnostic.adapter_message, probable_cause=diagnostic.probable_cause or diagnostic.adapter_message, next_step=diagnostic.suggested_next_step, + tag=diagnostic.root_cause_code or "uncategorized", ) message = f"{diagnostic.adapter_message} {diagnostic.probable_cause or ''}".lower() payload_text = f"{expected or {}} {actual or {}}".lower() @@ -107,6 +110,7 @@ def _classify_payload(expected: JSONDict, actual: JSONDict) -> FailureExplanatio "adapter raised instead of returning comparable workbook data", "adapter/runtime error interrupted the assertion path", "open the diagnostic adapter message and reproduce the adapter call directly", + "exception", ) if "formula" in expected_keys or "formula" in actual_keys or "cached" in text: return FailureExplanation( @@ -114,6 +118,7 @@ def _classify_payload(expected: JSONDict, actual: JSONDict) -> FailureExplanatio "formula text or cached formula result drifted", "formula preservation and cached-value handling differ between libraries", "inspect formula XML and cached value handling for the target cell", + "formula", ) if {"bg_color", "font_color", "number_format", "format"} & expected_keys or "style" in text: return FailureExplanation( @@ -122,6 +127,7 @@ def _classify_payload(expected: JSONDict, actual: JSONDict) -> FailureExplanatio "style normalization, default style handling, or writer formatting support " "is incomplete", "diff styles.xml and the adapter's cell-format read/write path", + "style", ) return _classify_text("", text) @@ -136,6 +142,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "adapter does not support this feature surface", "the library or adapter has no implementation for this operation", "check adapter capability gating before treating this as semantic drift", + "unsupported", ), ), ( @@ -145,6 +152,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "cell style metadata changed", "formatting was dropped, defaulted, or normalized differently", "inspect styles.xml and the adapter's style mapping", + "style", ), ), ( @@ -154,6 +162,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "table metadata changed", "table XML, totals-row state, or auto-filter metadata was not preserved", "inspect xl/tables/table*.xml and worksheet table relationships", + "table", ), ), ( @@ -163,6 +172,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "image or drawing relationship changed", "media part may exist but worksheet drawing rels or anchors do not match", "inspect drawing XML, drawing rels, and xl/media package parts", + "drawing", ), ), ( @@ -172,6 +182,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "named range metadata changed", "workbook-level vs sheet-level defined-name scope was lost or rewritten", "inspect workbook.xml definedNames and localSheetId values", + "named_range", ), ), ( @@ -181,6 +192,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "merged-cell metadata changed", "merged range XML or non-anchor cell handling differs", "inspect mergeCells in the worksheet XML and subordinate cell behavior", + "merge", ), ), ( @@ -190,6 +202,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "data validation metadata changed", "validation type, formula, or target range was dropped or rewritten", "inspect dataValidations in the worksheet XML", + "validation", ), ), ( @@ -199,6 +212,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "hyperlink target or display metadata changed", "hyperlink rel target, tooltip, or internal location was not preserved", "inspect worksheet hyperlinks and worksheet rels", + "hyperlink", ), ), ( @@ -208,6 +222,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "comment metadata changed", "legacy comment text, author, or VML relationship was not preserved", "inspect comments XML plus VML drawing relationships", + "comment", ), ), ( @@ -217,6 +232,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None "freeze pane settings changed", "pane split/top-left metadata was dropped or normalized incorrectly", "inspect sheetViews/pane in the worksheet XML", + "freeze_pane", ), ), ] diff --git a/src/excelbench/results/html_dashboard.py b/src/excelbench/results/html_dashboard.py index 76f1203..b19159e 100644 --- a/src/excelbench/results/html_dashboard.py +++ b/src/excelbench/results/html_dashboard.py @@ -257,6 +257,32 @@ def _fmt_mb(val: float | None) -> str: return f"{val:.1f}" +def _is_unsupported_case(case_data: dict[str, Any]) -> bool: + text_parts = [ + str(case_data.get("message", "")), + str(case_data.get("notes", "")), + str(case_data.get("label", "")), + ] + for diag in case_data.get("diagnostics", []): + if not isinstance(diag, dict): + continue + category = str(diag.get("category", "")).lower() + explanation = diag.get("explanation") + if category == "unsupported_feature": + return True + if ( + isinstance(explanation, dict) + and str(explanation.get("tag", "")).lower() == "unsupported" + ): + return True + text_parts.append(str(diag.get("adapter_message", ""))) + text_parts.append(str(diag.get("probable_cause", ""))) + text_parts.append(str(diag.get("root_cause_code", ""))) + haystack = " ".join(text_parts).lower() + markers = ("unsupported", "not implemented", "not supported", "read-only", "write-only") + return any(x in haystack for x in markers) + + def _safe_json(data: Any) -> str: """JSON for embedding inside .""" return json.dumps(data, ensure_ascii=False).replace(" tuple[dict[str, Any], dict[str, Any]] | None: + if history_path is None or not history_path.exists(): + return None + entries: list[dict[str, Any]] = [] + for line in history_path.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(row, dict): + entries.append(row) + if len(entries) < 2: + return None + return entries[-2], entries[-1] + + +def _section_delta_since_last_run(fidelity_history: Path | None, perf_history: Path | None) -> str: + items: list[str] = [] + pair = _load_recent_history(fidelity_history) + if pair: + prev, cur = pair + deltas = _compute_score_deltas(prev, cur) + if deltas: + improvements = sum(1 for d in deltas if d > 0) + regressions = sum(1 for d in deltas if d < 0) + items.append( + f"
  • Fidelity score changes: {len(deltas)} " + f"(improvements: {improvements}, regressions: {regressions})
    • " + ) + perf_pair = _load_recent_history(perf_history) + if perf_pair: + prev, cur = perf_pair + for op_key, label in ( + ("read_p50", "Median read throughput"), + ("write_p50", "Median write throughput"), + ): + perf_deltas = _compute_perf_p50_deltas(prev, cur, op_key) + if perf_deltas: + pct = round(sorted(perf_deltas)[len(perf_deltas) // 2]) + items.append(f"
  • {label}: {pct:+d}%
    • ") + if not items: + return "" + return ( + '

    Delta Since Last Run

    ' + f'
    ' + ) + + +def _compute_score_deltas(previous: dict[str, Any], current: dict[str, Any]) -> list[int]: + out: list[int] = [] + prev_scores = previous.get("scores", {}) + curr_scores = current.get("scores", {}) + for lib in set(prev_scores) | set(curr_scores): + for feat in set(prev_scores.get(lib, {})) | set(curr_scores.get(lib, {})): + for mode in ("read", "write"): + pv = prev_scores.get(lib, {}).get(feat, {}).get(mode) + cv = curr_scores.get(lib, {}).get(feat, {}).get(mode) + if pv is None or cv is None or pv == cv: + continue + out.append(int(cv) - int(pv)) + return out + + +def _compute_perf_p50_deltas( + previous: dict[str, Any], current: dict[str, Any], op_key: str +) -> list[float]: + out: list[float] = [] + prev_wall = previous.get("p50_wall_ms", {}) + curr_wall = current.get("p50_wall_ms", {}) + if not isinstance(prev_wall, dict) or not isinstance(curr_wall, dict): + return out + for library in set(prev_wall) | set(curr_wall): + prev_lib = prev_wall.get(library, {}) + curr_lib = curr_wall.get(library, {}) + if not isinstance(prev_lib, dict) or not isinstance(curr_lib, dict): + continue + for feature in set(prev_lib) | set(curr_lib): + prev_val = (prev_lib.get(feature) or {}).get(op_key) + curr_val = (curr_lib.get(feature) or {}).get(op_key) + if ( + isinstance(prev_val, (int, float)) + and isinstance(curr_val, (int, float)) + and prev_val != 0 + ): + out.append((prev_val - curr_val) / prev_val * 100) + return out + + # ==================================================================== # CSS # ==================================================================== @@ -686,6 +807,7 @@ def render_html_dashboard( .tc-table td,.tc-table th{font-size:.74rem;padding:.3rem .5rem} .tc-table .pass{color:#76d8a2} .tc-table .fail{color:#ff7683;font-weight:600} +.tc-table .unsupported{color:#f5c46b;font-weight:700} code.val{ font-family:var(--font-mono); font-size:.71rem; @@ -1386,7 +1508,14 @@ def _section_radar( perf: dict[str, Any] | None, ) -> str: """Render a 5-axis spider chart comparing top libraries.""" - import plotly.graph_objects as go + try: + import plotly.graph_objects as go + except ModuleNotFoundError: + return ( + '

    Strength Profiles

    ' + "

    Plotly is not installed in this environment.

    " + "
    " + ) from excelbench.results.scatter import ( _DARK_BG, @@ -1561,28 +1690,31 @@ def _section_scatter( # Interactive Plotly scatter charts (preferred when perf data exists) if perf: - from excelbench.results.scatter_interactive import ( - render_interactive_scatter_features_from_data, - render_interactive_scatter_tiers_from_data, - ) + try: + from excelbench.results.scatter_interactive import ( + render_interactive_scatter_features_from_data, + render_interactive_scatter_tiers_from_data, + ) - tiers_html = render_interactive_scatter_tiers_from_data(fidelity, perf) - features_html = render_interactive_scatter_features_from_data(fidelity, perf) + tiers_html = render_interactive_scatter_tiers_from_data(fidelity, perf) + features_html = render_interactive_scatter_features_from_data(fidelity, perf) - parts.append( - '

    By Feature Group

    ' - f'
    ' - f'' - f'
    {tiers_html}
    ' - ) - parts.append( - '

    Per Feature

    ' - f'
    ' - f'' - f'
    {features_html}
    ' - ) + parts.append( + '

    By Feature Group

    ' + f'
    ' + f'' + f'
    {tiers_html}
    ' + ) + parts.append( + '

    Per Feature

    ' + f'
    ' + f'' + f'
    {features_html}
    ' + ) + except ModuleNotFoundError: + parts.append('

    Interactive scatter unavailable.

    ') else: # Fallback: embed pre-rendered static SVGs when perf data is unavailable if scatter_svgs: @@ -1845,8 +1977,9 @@ def _section_features(fidelity: dict[str, Any]) -> str: lbl = d.get("label") or tc_id exp = _fmt_val(d.get("expected")) act = _fmt_val(d.get("actual")) - pcls = "pass" if passed else "fail" - psym = "\u2713" if passed else "\u2717" + is_unsupported = (not passed) and _is_unsupported_case(d) + pcls = "pass" if passed else ("unsupported" if is_unsupported else "fail") + psym = "✓" if passed else ("⊘" if is_unsupported else "✗") rows.append( f"{_esc(lbl)}{op}" f"{_esc(imp)}" diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 5ac7dc3..afc2c1b 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -1,3 +1,5 @@ +from pathlib import Path + from excelbench.results.dashboard import _build_dashboard @@ -88,3 +90,32 @@ def test_dashboard_filters_pyumya_and_shows_modify_column() -> None: assert "| wolfxl | R+W | Patch |" in doc assert "| openpyxl | R+W | Rewrite |" in doc assert "pyumya" not in doc + + +def test_dashboard_includes_delta_since_last_run(tmp_path: Path) -> None: + fidelity_history = tmp_path / "history.jsonl" + fidelity_history.write_text( + '{"scores":{"openpyxl":{"cell_values":{"read":2,"write":2}}}}\n' + '{"scores":{"openpyxl":{"cell_values":{"read":3,"write":1}}}}\n' + ) + perf_history = tmp_path / "perf_history.jsonl" + perf_history.write_text( + '{"p50_wall_ms":{"openpyxl":{"cell_values":{"read_p50":10,"write_p50":20}}}}\n' + '{"p50_wall_ms":{"openpyxl":{"cell_values":{"read_p50":8,"write_p50":25}}}}\n' + ) + fidelity = { + "metadata": {}, + "libraries": {"openpyxl": {"capabilities": ["read", "write"]}}, + "results": [], + } + doc = "\n".join( + _build_dashboard( + fidelity, + perf=None, + fidelity_history_path=fidelity_history, + perf_history_path=perf_history, + ) + ) + assert "## Delta Since Last Run" in doc + assert "Fidelity score changes" in doc + assert "Median read throughput" in doc diff --git a/tests/test_failure_explainer.py b/tests/test_failure_explainer.py index a7d5f92..27c67b1 100644 --- a/tests/test_failure_explainer.py +++ b/tests/test_failure_explainer.py @@ -29,6 +29,7 @@ def test_explain_diagnostic_classifies_style_drift() -> None: assert explanation is not None assert explanation.code == "style_drift" + assert explanation.tag == "style" def test_explain_test_failure_classifies_formula_payload() -> None: @@ -45,3 +46,4 @@ def test_explain_test_failure_classifies_formula_payload() -> None: assert explanation is not None assert explanation.code == "formula_cache_or_formula_drift" + assert explanation.tag == "formula" diff --git a/tests/test_results_html_dashboard.py b/tests/test_results_html_dashboard.py index 6ec0d1c..0586179 100644 --- a/tests/test_results_html_dashboard.py +++ b/tests/test_results_html_dashboard.py @@ -109,3 +109,42 @@ def test_compute_radar_data_uses_p50_when_op_count_missing() -> None: # pandas is half as fast as openpyxl in this fixture. assert by_lib["pandas"][1] == 25.0 assert by_lib["pandas"][2] == 25.0 + + +def test_render_html_dashboard_shows_delta_and_unsupported(tmp_path: Path) -> None: + fidelity = tmp_path / "results.json" + fidelity.write_text( + """ +{ + "metadata": {"profile": "xlsx", "run_date": "2026-01-01T00:00:00Z"}, + "libraries": {"openpyxl": {"capabilities": ["read", "write"]}}, + "results": [{ + "feature": "cell_values", + "library": "openpyxl", + "scores": {"read": 0, "write": 0}, + "test_cases": { + "tc1": { + "read": { + "passed": false, + "label": "unsupported check", + "diagnostics": [{"adapter_message": "feature not supported"}] + } + } + } + }] +} +""".strip() + ) + (tmp_path / "history.jsonl").write_text( + '{"scores":{"openpyxl":{"cell_values":{"read":1,"write":1}}}}\n' + '{"scores":{"openpyxl":{"cell_values":{"read":0,"write":0}}}}\n' + ) + (tmp_path / "perf_history.jsonl").write_text( + '{"p50_wall_ms":{"openpyxl":{"cell_values":{"read_p50":10,"write_p50":20}}}}\n' + '{"p50_wall_ms":{"openpyxl":{"cell_values":{"read_p50":8,"write_p50":25}}}}\n' + ) + out = tmp_path / "dash.html" + render_html_dashboard(fidelity, perf_json=None, output_path=out, scatter_dir=None) + html = out.read_text() + assert "Delta Since Last Run" in html + assert "unsupported" in html