feat(results): separate unsupported vs failed, add run deltas, and surface failure tags

docs-site/benchmark/scoring.mdx

@@ -58,3 +58,10 @@ Libraries are grouped by overall fidelity + capability:
 | C | below 60% fidelity or limited scope |
 
 See [live dashboard](https://excelbench.vercel.app) for current tier assignments.
+
+## Interpreting failed vs unsupported
+- **Failed** means the adapter attempted the operation but produced mismatched output.
+- **Unsupported** means the adapter explicitly cannot perform that operation; these are visually distinct in HTML dashboards and should be triaged as coverage gaps first.
+
+## Delta Since Last Run
+Dashboard and report outputs now include a **Delta Since Last Run** summary sourced from `history.jsonl` in each results directory.

docs-site/cli/report.mdx

@@ -64,3 +64,8 @@ uv run excelbench scatter
 ```
 
 Generates fidelity-vs-throughput scatter plots as PNG and SVG, broken down by tier.
+
+## Delta Since Last Run
+`excelbench report` and dashboard layers read `results/*/history.jsonl` and emit a compact run-over-run delta summary:
+- fidelity score changes (improvements vs regressions)
+- perf median read/write throughput percentage changes when perf history exists

src/excelbench/results/dashboard.py

@@ -35,7 +35,12 @@ def render_dashboard(
             perf_data = json.load(f)
         perf_data = filter_report_data(perf_data)
 
-    lines = _build_dashboard(fidelity_data, perf_data)
+    lines = _build_dashboard(
+        fidelity_data,
+        perf_data,
+        fidelity_history_path=fidelity_json.parent / "history.jsonl",
+        perf_history_path=perf_json.parent / "history.jsonl" if perf_json else None,
+    )
 
     output_path.parent.mkdir(parents=True, exist_ok=True)
     with open(output_path, "w") as f:
@@ -45,6 +50,9 @@ def render_dashboard(
 def _build_dashboard(
     fidelity: dict[str, Any],
     perf: dict[str, Any] | None,
+    *,
+    fidelity_history_path: Path | None = None,
+    perf_history_path: Path | None = None,
 ) -> list[str]:
     """Build combined dashboard markdown lines."""
     fidelity = filter_report_data(fidelity)
@@ -61,6 +69,7 @@ def _build_dashboard(
     lines.append("> Combined fidelity and performance view. Fidelity shows correctness;")
     lines.append("> throughput shows speed. Use this to find the right library for your needs.")
     lines.append("")
+    lines.extend(_render_delta_since_last_run(fidelity_history_path, perf_history_path))
 
     # ── Compute per-library fidelity stats ──
     lib_stats = _compute_fidelity_stats(fidelity)
@@ -128,6 +137,30 @@ def _build_dashboard(
     return lines
 
 
+def _render_delta_since_last_run(
+    fidelity_history_path: Path | None,
+    perf_history_path: Path | None,
+) -> list[str]:
+    fidelity_delta = _compute_history_delta_summary(fidelity_history_path)
+    perf_delta = _compute_perf_delta_summary(perf_history_path)
+    if not fidelity_delta and not perf_delta:
+        return []
+    lines = ["## Delta Since Last Run", ""]
+    if fidelity_delta:
+        lines.append(
+            "- Fidelity score changes: "
+            f"**{fidelity_delta['changes']}** (improvements: **{fidelity_delta['improvements']}**, "
+            f"regressions: **{fidelity_delta['regressions']}**)"
+        )
+    if perf_delta:
+        if "read_ops_per_sec" in perf_delta:
+            lines.append(f"- Median read throughput: **{perf_delta['read_ops_per_sec']:+d}%**")
+        if "write_ops_per_sec" in perf_delta:
+            lines.append(f"- Median write throughput: **{perf_delta['write_ops_per_sec']:+d}%**")
+    lines.append("")
+    return lines
+
+
 def _compute_fidelity_stats(data: dict[str, Any]) -> dict[str, dict[str, Any]]:
     """Compute per-library fidelity statistics from results JSON."""
     libs_info = data.get("libraries", {})
@@ -186,6 +219,99 @@ def _compute_fidelity_stats(data: dict[str, Any]) -> dict[str, dict[str, Any]]:
     return out
 
 
+def _load_recent_history(history_path: Path | None) -> tuple[dict[str, Any], dict[str, Any]] | None:
+    if history_path is None or not history_path.exists():
+        return None
+    entries: list[dict[str, Any]] = []
+    for line in history_path.read_text().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            parsed = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict):
+            entries.append(parsed)
+    if len(entries) < 2:
+        return None
+    return entries[-2], entries[-1]
+
+
+def _compute_history_delta_summary(history_path: Path | None) -> dict[str, int] | None:
+    pair = _load_recent_history(history_path)
+    if pair is None:
+        return None
+    previous, current = pair
+    changes = regressions = improvements = 0
+    for item in _compute_fidelity_deltas(previous, current):
+        changes += 1
+        if item["delta"] < 0:
+            regressions += 1
+        elif item["delta"] > 0:
+            improvements += 1
+    return {"changes": changes, "improvements": improvements, "regressions": regressions}
+
+
+def _compute_perf_delta_summary(history_path: Path | None) -> dict[str, int] | None:
+    pair = _load_recent_history(history_path)
+    if pair is None:
+        return None
+    previous, current = pair
+    out: dict[str, int] = {}
+    for key, op_key in (("read_ops_per_sec", "read_p50"), ("write_ops_per_sec", "write_p50")):
+        deltas = _compute_perf_p50_deltas(previous, current, op_key)
+        if deltas:
+            out[key] = round(sorted(deltas)[len(deltas) // 2])
+    return out or None
+
+
+def _compute_perf_p50_deltas(
+    previous: dict[str, Any], current: dict[str, Any], op_key: str
+) -> list[float]:
+    deltas: list[float] = []
+    prev_wall = previous.get("p50_wall_ms", {})
+    curr_wall = current.get("p50_wall_ms", {})
+    if not isinstance(prev_wall, dict) or not isinstance(curr_wall, dict):
+        return deltas
+    for library in set(prev_wall) | set(curr_wall):
+        prev_lib = prev_wall.get(library, {})
+        curr_lib = curr_wall.get(library, {})
+        if not isinstance(prev_lib, dict) or not isinstance(curr_lib, dict):
+            continue
+        for feature in set(prev_lib) | set(curr_lib):
+            prev_val = (prev_lib.get(feature) or {}).get(op_key)
+            curr_val = (curr_lib.get(feature) or {}).get(op_key)
+            if (
+                isinstance(prev_val, (int, float))
+                and isinstance(curr_val, (int, float))
+                and prev_val != 0
+            ):
+                deltas.append((prev_val - curr_val) / prev_val * 100)
+    return deltas
+
+
+def _compute_fidelity_deltas(
+    previous: dict[str, Any], current: dict[str, Any]
+) -> list[dict[str, int]]:
+    deltas: list[dict[str, int]] = []
+    prev_scores: dict[str, Any] = previous.get("scores", {})
+    curr_scores: dict[str, Any] = current.get("scores", {})
+    for library in sorted(set(prev_scores) | set(curr_scores)):
+        prev_lib = prev_scores.get(library, {})
+        curr_lib = curr_scores.get(library, {})
+        for feature in sorted(set(prev_lib) | set(curr_lib)):
+            prev_feature = prev_lib.get(feature, {})
+            curr_feature = curr_lib.get(feature, {})
+            for mode in ("read", "write"):
+                prev_value = prev_feature.get(mode)
+                curr_value = curr_feature.get(mode)
+                if prev_value is None or curr_value is None or prev_value == curr_value:
+                    continue
+                deltas.append({"delta": int(curr_value) - int(prev_value)})
+    return deltas
+
+
 def _compute_throughput(data: dict[str, Any]) -> dict[str, dict[str, float | None]]:
     """Extract representative throughput (cells/s) per library from perf results.
 

src/excelbench/results/failure_explainer.py

@@ -18,13 +18,15 @@ class FailureExplanation:
     summary: str
     probable_cause: str
     next_step: str
+    tag: str
 
     def to_json_dict(self) -> JSONDict:
         return {
             "code": self.code,
             "summary": self.summary,
             "probable_cause": self.probable_cause,
             "next_step": self.next_step,
+            "tag": self.tag,
         }
 
 
@@ -51,6 +53,7 @@ def explain_diagnostic(
             summary=diagnostic.probable_cause or diagnostic.adapter_message,
             probable_cause=diagnostic.probable_cause or diagnostic.adapter_message,
             next_step=diagnostic.suggested_next_step,
+            tag=diagnostic.root_cause_code or "uncategorized",
         )
     message = f"{diagnostic.adapter_message} {diagnostic.probable_cause or ''}".lower()
     payload_text = f"{expected or {}} {actual or {}}".lower()
@@ -107,13 +110,15 @@ def _classify_payload(expected: JSONDict, actual: JSONDict) -> FailureExplanatio
             "adapter raised instead of returning comparable workbook data",
             "adapter/runtime error interrupted the assertion path",
             "open the diagnostic adapter message and reproduce the adapter call directly",
+            "exception",
         )
     if "formula" in expected_keys or "formula" in actual_keys or "cached" in text:
         return FailureExplanation(
             "formula_cache_or_formula_drift",
             "formula text or cached formula result drifted",
             "formula preservation and cached-value handling differ between libraries",
             "inspect formula XML and cached value handling for the target cell",
+            "formula",
         )
     if {"bg_color", "font_color", "number_format", "format"} & expected_keys or "style" in text:
         return FailureExplanation(
@@ -122,6 +127,7 @@ def _classify_payload(expected: JSONDict, actual: JSONDict) -> FailureExplanatio
             "style normalization, default style handling, or writer formatting support "
             "is incomplete",
             "diff styles.xml and the adapter's cell-format read/write path",
+            "style",
         )
     return _classify_text("", text)
 
@@ -136,6 +142,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "adapter does not support this feature surface",
                 "the library or adapter has no implementation for this operation",
                 "check adapter capability gating before treating this as semantic drift",
+                "unsupported",
             ),
         ),
         (
@@ -145,6 +152,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "cell style metadata changed",
                 "formatting was dropped, defaulted, or normalized differently",
                 "inspect styles.xml and the adapter's style mapping",
+                "style",
             ),
         ),
         (
@@ -154,6 +162,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "table metadata changed",
                 "table XML, totals-row state, or auto-filter metadata was not preserved",
                 "inspect xl/tables/table*.xml and worksheet table relationships",
+                "table",
             ),
         ),
         (
@@ -163,6 +172,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "image or drawing relationship changed",
                 "media part may exist but worksheet drawing rels or anchors do not match",
                 "inspect drawing XML, drawing rels, and xl/media package parts",
+                "drawing",
             ),
         ),
         (
@@ -172,6 +182,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "named range metadata changed",
                 "workbook-level vs sheet-level defined-name scope was lost or rewritten",
                 "inspect workbook.xml definedNames and localSheetId values",
+                "named_range",
             ),
         ),
         (
@@ -181,6 +192,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "merged-cell metadata changed",
                 "merged range XML or non-anchor cell handling differs",
                 "inspect mergeCells in the worksheet XML and subordinate cell behavior",
+                "merge",
             ),
         ),
         (
@@ -190,6 +202,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "data validation metadata changed",
                 "validation type, formula, or target range was dropped or rewritten",
                 "inspect dataValidations in the worksheet XML",
+                "validation",
             ),
         ),
         (
@@ -199,6 +212,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "hyperlink target or display metadata changed",
                 "hyperlink rel target, tooltip, or internal location was not preserved",
                 "inspect worksheet hyperlinks and worksheet rels",
+                "hyperlink",
             ),
         ),
         (
@@ -208,6 +222,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "comment metadata changed",
                 "legacy comment text, author, or VML relationship was not preserved",
                 "inspect comments XML plus VML drawing relationships",
+                "comment",
             ),
         ),
         (
@@ -217,6 +232,7 @@ def _classify_text(message: str, payload_text: str) -> FailureExplanation | None
                 "freeze pane settings changed",
                 "pane split/top-left metadata was dropped or normalized incorrectly",
                 "inspect sheetViews/pane in the worksheet XML",
+                "freeze_pane",
             ),
         ),
     ]