Add git_numstat backend (RepoState): session-retrospective change digest

src/perfdigest/adapters/git_numstat/__init__.py

@@ -0,0 +1 @@
+"""git numstat backend (RepoState) — changed files, domain repo_change."""

src/perfdigest/adapters/git_numstat/backend.py

@@ -0,0 +1,84 @@
+"""git numstat backend registration (RepoState — session-retrospective).
+
+RepoState is NOT GitHub/API monitoring: it is a backward look at what a
+working session did to a repo, read from a SAVED artifact. It is also the
+provenance axis — a numstat snapshot pins the perf/build/CI reports captured
+next to it to a concrete code state. Facts only, never verdicts: the backend
+reports which files changed and by how many lines; whether that footprint is
+"ready" is the model's conclusion.
+"""
+
+from __future__ import annotations
+
+from perfdigest.adapters import registry
+from perfdigest.adapters.git_numstat import mapping, numstat_reader
+from perfdigest.core.backend import Backend, CapabilityReport
+from perfdigest.platform.detect import PlatformInfo, detect
+
+_PLATFORMS = frozenset({"linux", "darwin", "win32"})  # git runs everywhere
+
+GIT_NUMSTAT_USAGE = (
+    "git numstat digest (RepoState — the session-retrospective change digest). "
+    "Artifact-first; the raw diff is the token sink, numstat is its digest:\n"
+    "  git diff --numstat -M <base>.. > session.numstat   (the session's range, "
+    "format git-numstat)\n"
+    "  git diff --numstat -M > session.numstat            (uncommitted work only)\n"
+    "Units are changed files named by path — a renamed file is named by its "
+    "NEW path (where it lives now); the old path is in expand. Use this to "
+    "REORIENT after compaction or a session hand-off instead of re-reading raw "
+    "diffs: summarize_report lists the footprint, lines_added ranks it (units "
+    "carry no duration — a change digest has no time dimension — so ordering "
+    "is file order and size ranking comes from reading the metrics). BINARY "
+    "files have no line counts: lines_added/deleted are honestly "
+    "'not_available_in_this_export', never 0.0 — while a pure no-edit rename "
+    "is a genuine measured 0. Pair two snapshots with compare_metrics "
+    "(report_a=earlier, report_b=later, kernel=<path>) to see how the "
+    "session's footprint evolved. Keep snapshots next to captured perf/build/"
+    "CI reports as cheap provenance anchors: the numstat pins those reports "
+    "to the code state that produced them."
+)
+
+
+def _probe() -> CapabilityReport:
+    info = detect()
+    exe = info.profilers_on_path.get("git_numstat")
+    if not exe:
+        return CapabilityReport(False, "git not found on PATH", None)
+    return CapabilityReport(
+        True,
+        "git present (capture runs inside the repo being digested)",
+        exe,
+        notes=(
+            "Add -M so renames digest as one moved file instead of a fake "
+            "add+delete pair; note git only pairs a rename when similarity "
+            "stays above its threshold — a heavily rewritten move still "
+            "splits, and that split is git's real answer, not a parse bug.",
+        ),
+    )
+
+
+def _capture_command(target: str, info: PlatformInfo) -> str:
+    # `target` is the base ref of the session (a tag, branch, or commit).
+    return (
+        f"git diff --numstat -M {target}.. > session.numstat"
+        "  # uncommitted-only: git diff --numstat -M > session.numstat"
+    )
+
+
+registry.register(
+    Backend(
+        name="git_numstat",
+        formats=frozenset({"git-numstat", "numstat"}),
+        suffixes=(".numstat",),
+        domain="repo_change",
+        platforms=_PLATFORMS,
+        standard_to_vendor=mapping.STANDARD_TO_VENDOR,
+        default_core_set=tuple(mapping.DEFAULT_CORE_SET),
+        usage_prompt=GIT_NUMSTAT_USAGE,
+        load_units=numstat_reader.load_units,
+        raw_metrics=numstat_reader.raw_metrics,
+        probe=_probe,
+        reader_available=lambda: True,  # pure Python
+        capture_command=_capture_command,
+    )
+)

src/perfdigest/adapters/git_numstat/mapping.py

@@ -0,0 +1,21 @@
+"""git numstat columns -> standard repo-change terms.
+
+A numstat export has exactly two numbers per file — the vocabulary is
+deliberately the smallest in the project. It carries FACTS about a session's
+footprint (which files, how many lines), never verdicts: whether a 400-line
+delta is "too big to release" is the model's conclusion, not this backend's.
+"""
+
+from __future__ import annotations
+
+# standard term -> the numstat column behind it (the `expand` hint)
+STANDARD_TO_VENDOR: dict[str, str] = {
+    "lines_added": "numstat column 1 ('-' for binary: not a line count, stays absent)",
+    "lines_deleted": "numstat column 2 ('-' for binary: not a line count, stays absent)",
+}
+
+# What get_metrics(metrics=None) returns for a repo-change unit.
+DEFAULT_CORE_SET: list[str] = [
+    "lines_added",
+    "lines_deleted",
+]

src/perfdigest/adapters/git_numstat/numstat_reader.py

@@ -0,0 +1,157 @@
+"""Read a saved ``git diff --numstat`` export -> NormalizedUnit (repo_change).
+Pure Python, no deps.
+
+RepoState semantics (owner directive): a token-efficient BACKWARD look at what
+a working session did to a repo, so a re-orienting agent (post-compaction, new
+session) does not re-read raw diffs. The raw diff is the token sink; numstat
+is its digest. This reader emits FACTS (paths, line counts) and never
+verdicts — "is this footprint releasable" is the model's conclusion.
+
+Artifact-first: we bind to a SAVED file produced by::
+
+    git diff --numstat -M <base>..<tip> > session.numstat   # a session's range
+    git diff --numstat -M > session.numstat                 # uncommitted only
+
+Line grammar, verified against REAL ``git diff --numstat -M`` output
+(git 2.53, throwaway-lab capture committed as
+``tests/fixtures/repo_lab_renames_sample.numstat``)::
+
+    <added>\\t<deleted>\\t<path spec>
+
+  * added/deleted are decimal counts, or ``-`` for a BINARY file. The dash is
+    the headline honesty case: a binary has no line counts, so both metrics
+    stay ``None`` -> 'not_available_in_this_export' — NEVER 0.0. A genuine
+    ``0`` (e.g. a pure rename with no edits) is preserved as measured 0.0.
+  * The path spec has three real shapes:
+      - plain path              (spaces verbatim: ``docs/read me.txt``)
+      - ``old => new``          (whole-path rename: ``NOTES => TASKS.md``)
+      - ``pre{old => new}post`` (partial rename: ``src/{a.py => b.py}``,
+        ``{src => docs}/util.py``, ``docs/{ => api}/config.py`` — either brace
+        side may be empty; rebuilding a path over an empty side can produce
+        ``//``, which git's own display collapses, so we do too)
+    A renamed unit is NAMED BY ITS NEW PATH (where the file lives now — the
+    reorientation answer); the old path stays reachable via ``expand``.
+  * Non-ASCII paths arrive C-quoted by git (``"docs/\\303\\266l\\303\\247..."``,
+    core.quotePath default) and are kept VERBATIM as printed — the name is
+    honest to the artifact; we do not unquote.
+
+``duration_us`` is ``None`` for every unit: a change digest has no time
+dimension at all, so ``summarize_report`` falls back to file order and ranking
+by size is done by reading ``lines_added`` from the metrics — a fact the agent
+sorts by, not a hotness verdict we bake in.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from perfdigest.core.metrics import DOMAIN_REPO_CHANGE, NormalizedUnit
+
+_LINE = re.compile(r"^(?P<added>\d+|-)\t(?P<deleted>\d+|-)\t(?P<path>.+)$")
+# One brace group per line is all real git emits (the common prefix/suffix
+# around the single differing middle segment).
+_BRACED = re.compile(r"^(?P<pre>.*)\{(?P<old>.*) => (?P<new>.*)\}(?P<post>.*)$")
+
+
+def _count(field: str) -> float | None:
+    """A numstat count column: int, or None for the binary-file dash."""
+    return None if field == "-" else float(field)
+
+
+def _split_path(path_spec: str) -> tuple[str, str | None]:
+    """-> (new_path, old_path or None if not a rename)."""
+    m = _BRACED.match(path_spec)
+    if m:
+        new = f"{m['pre']}{m['new']}{m['post']}".replace("//", "/")
+        old = f"{m['pre']}{m['old']}{m['post']}".replace("//", "/")
+        return new, old
+    if " => " in path_spec:
+        old, _, new = path_spec.partition(" => ")
+        return new, old
+    return path_spec, None
+
+
+def _parse(report_path: str) -> list[dict[str, Any]]:
+    with open(report_path, "r", encoding="utf-8", errors="ignore") as fh:
+        text = fh.read()
+
+    records: list[dict[str, Any]] = []
+    malformed = 0
+    for raw_line in text.splitlines():
+        if not raw_line.strip():
+            continue
+        m = _LINE.match(raw_line)
+        if not m:
+            malformed += 1
+            continue
+        new_path, old_path = _split_path(m["path"])
+        records.append(
+            {
+                "path": new_path,
+                "old_path": old_path,
+                "path_as_printed": m["path"],
+                "lines_added": _count(m["added"]),
+                "lines_deleted": _count(m["deleted"]),
+            }
+        )
+
+    if not records:
+        raise ValueError(
+            f"{report_path} does not look like a `git diff --numstat` export: no "
+            "line matched the expected '<added>\\t<deleted>\\t<path>' shape "
+            "(tab-separated counts, '-' for binary files). An empty diff also "
+            "produces an empty file — if the session made no changes there is "
+            "nothing to digest. Produce this artifact with: "
+            "git diff --numstat -M <base>.. > session.numstat"
+        )
+    if malformed:
+        # Mixed content (a numstat pasted into some other log) digests the
+        # matching lines but must not do so silently.
+        raise ValueError(
+            f"{report_path}: {malformed} line(s) do not match the numstat "
+            f"'<added>\\t<deleted>\\t<path>' shape next to {len(records)} that "
+            "do — this file is not a clean `git diff --numstat` export. "
+            "Re-capture with: git diff --numstat -M <base>.. > session.numstat"
+        )
+    return records
+
+
+def load_units(report_path: str) -> list[NormalizedUnit]:
+    units: list[NormalizedUnit] = []
+    for index, rec in enumerate(_parse(report_path)):
+        units.append(
+            NormalizedUnit(
+                name=rec["path"],
+                index=index,
+                duration_us=None,  # a change digest has no time dimension
+                raw_ref=report_path,
+                metrics={
+                    "lines_added": rec["lines_added"],
+                    "lines_deleted": rec["lines_deleted"],
+                },
+                domain=DOMAIN_REPO_CHANGE,
+            )
+        )
+    return units
+
+
+def raw_metrics(report_path: str, kernel_index: int, name_filter: str) -> dict[str, Any]:
+    """The parsed line behind one unit — old path and rename/binary facts."""
+    records = _parse(report_path)
+    if kernel_index >= len(records):
+        raise IndexError(f"unit index {kernel_index} not present in {report_path}")
+    rec = records[kernel_index]
+    out: dict[str, Any] = {
+        "path": rec["path"],
+        "path_as_printed": rec["path_as_printed"],
+        "renamed": rec["old_path"] is not None,
+        "old_path": rec["old_path"],
+        "is_binary": rec["lines_added"] is None and rec["lines_deleted"] is None,
+        "lines_added": rec["lines_added"],
+        "lines_deleted": rec["lines_deleted"],
+    }
+    wanted = None if name_filter.lower() == "all" else name_filter.lower()
+    if wanted is not None:
+        out = {k: v for k, v in out.items() if wanted in k.lower()}
+    return out

src/perfdigest/core/metrics.py

@@ -30,6 +30,9 @@
     * ``build_diag``      — one crate's build diagnostics from a saved
       ``cargo build --message-format=json`` stream; error/warning counts,
       NO timing of any kind (duration stays honestly ``None``)
+    * ``repo_change``     — one changed file from a saved ``git diff --numstat``
+      (RepoState: the session-retrospective change digest); line counts only,
+      no time dimension at all
 
 THE absence rule (the single most dangerous failure mode if violated):
 
@@ -55,6 +58,7 @@
 DOMAIN_CI_STEP = "ci_step"
 DOMAIN_BENCHMARK = "benchmark"
 DOMAIN_BUILD_DIAG = "build_diag"
+DOMAIN_REPO_CHANGE = "repo_change"
 
 
 @dataclass(frozen=True)

src/perfdigest/platform/detect.py

@@ -28,6 +28,7 @@
     "gha_log": "gh",  # CIDigest: capture needs the GitHub CLI, digest needs nothing
     "criterion": "cargo",  # `cargo bench` writes target/criterion/**/new/estimates.json
     "cargo_diag": "cargo",  # BuildDigest: --message-format=json is a cargo flag
+    "git_numstat": "git",  # RepoState: `git diff --numstat -M` is the capture
 }
 
 

src/perfdigest/server/app.py

@@ -17,6 +17,7 @@ def _register_backends() -> None:
     from perfdigest.adapters.clang_time_trace import backend as _clang_tt  # noqa: F401
     from perfdigest.adapters.criterion import backend as _criterion  # noqa: F401
     from perfdigest.adapters.gha_log import backend as _gha_log  # noqa: F401
+    from perfdigest.adapters.git_numstat import backend as _git_numstat  # noqa: F401
     from perfdigest.adapters.linux_perf import backend as _perf  # noqa: F401
     from perfdigest.adapters.metal import backend as _metal  # noqa: F401
     from perfdigest.adapters.ninja_log import backend as _ninja_log  # noqa: F401

tests/fixtures/repo_lab_renames_sample.numstat

@@ -0,0 +1,11 @@
+1	0	NOTES => TASKS.md
+-	-	data/assets.dat
+-	-	data/logo.bin
+0	0	docs/{ => api}/config.py
+1	0	docs/read me.txt
+0	0	{src => docs}/util.py
+1	0	"docs/\303\266l\303\247\303\274m.txt"
+4	1	src/engine.py
+0	3	src/legacy.py
+2	0	src/new_module.py
+1	0	src/{parser.py => parser_v2.py}

tests/fixtures/repo_session_early_sample.numstat

@@ -0,0 +1,27 @@
+20	4	src/perfdigest/adapters/chrome_trace/trace_reader.py
+0	0	src/perfdigest/adapters/clang_time_trace/__init__.py
+88	0	src/perfdigest/adapters/clang_time_trace/backend.py
+30	0	src/perfdigest/adapters/clang_time_trace/mapping.py
+104	0	src/perfdigest/adapters/clang_time_trace/time_trace_reader.py
+1	0	src/perfdigest/adapters/gha_log/__init__.py
+96	0	src/perfdigest/adapters/gha_log/backend.py
+204	0	src/perfdigest/adapters/gha_log/gha_log_reader.py
+26	0	src/perfdigest/adapters/gha_log/mapping.py
+1	0	src/perfdigest/adapters/ninja_log/__init__.py
+65	0	src/perfdigest/adapters/ninja_log/backend.py
+26	0	src/perfdigest/adapters/ninja_log/mapping.py
+143	0	src/perfdigest/adapters/ninja_log/ninja_log_reader.py
+13	2	src/perfdigest/core/metrics.py
+3	0	src/perfdigest/platform/detect.py
+3	0	src/perfdigest/server/app.py
+333	0	tests/fixtures/ci_digest_macos_sample.gha.log
+7	0	tests/fixtures/ninja_log_sample.ninja_log
+1	0	tests/fixtures/ninja_log_sample/a.c
+1	0	tests/fixtures/ninja_log_sample/b.c
+12	0	tests/fixtures/ninja_log_sample/build.ninja
+3	0	tests/fixtures/ninja_log_sample/c.c
+83	0	tests/fixtures/template_heavy.cpp
+1	0	tests/fixtures/template_heavy.ftime-trace.json
+202	0	tests/test_clang_time_trace.py
+321	0	tests/test_gha_log.py
+153	0	tests/test_ninja_log.py

tests/fixtures/repo_session_sample.numstat

@@ -0,0 +1,46 @@
+20	4	src/perfdigest/adapters/chrome_trace/trace_reader.py
+0	0	src/perfdigest/adapters/clang_time_trace/__init__.py
+88	0	src/perfdigest/adapters/clang_time_trace/backend.py
+30	0	src/perfdigest/adapters/clang_time_trace/mapping.py
+104	0	src/perfdigest/adapters/clang_time_trace/time_trace_reader.py
+1	0	src/perfdigest/adapters/criterion/__init__.py
+76	0	src/perfdigest/adapters/criterion/backend.py
+131	0	src/perfdigest/adapters/criterion/criterion_reader.py
+29	0	src/perfdigest/adapters/criterion/mapping.py
+1	0	src/perfdigest/adapters/gha_log/__init__.py
+110	0	src/perfdigest/adapters/gha_log/backend.py
+204	0	src/perfdigest/adapters/gha_log/gha_log_reader.py
+26	0	src/perfdigest/adapters/gha_log/mapping.py
+1	0	src/perfdigest/adapters/ninja_log/__init__.py
+65	0	src/perfdigest/adapters/ninja_log/backend.py
+26	0	src/perfdigest/adapters/ninja_log/mapping.py
+143	0	src/perfdigest/adapters/ninja_log/ninja_log_reader.py
+4	0	src/perfdigest/core/backend.py
+17	2	src/perfdigest/core/metrics.py
+4	0	src/perfdigest/platform/detect.py
+9	0	src/perfdigest/report_store/cache.py
+16	1	src/perfdigest/report_store/discovery.py
+4	0	src/perfdigest/server/app.py
+3	3	src/perfdigest/server/tools.py
+333	0	tests/fixtures/ci_digest_macos_sample.gha.log
+431	0	tests/fixtures/ci_perf_green_runA_sample.gha.log
+431	0	tests/fixtures/ci_perf_green_runB_sample.gha.log
+268	0	tests/fixtures/ci_test_failed_sample.gha.log
+244	0	tests/fixtures/ci_test_green_sample.gha.log
+11	0	tests/fixtures/criterion_sample/Cargo.toml
+28	0	tests/fixtures/criterion_sample/benches/fib_bench.rs
+1	0	tests/fixtures/criterion_sample/criterion/fib/fib_20/new/estimates.json
+1	0	tests/fixtures/criterion_sample/criterion/fib_plain/new/estimates.json
+3	0	tests/fixtures/criterion_sample/src/lib.rs
+7	0	tests/fixtures/ninja_log_sample.ninja_log
+1	0	tests/fixtures/ninja_log_sample/a.c
+1	0	tests/fixtures/ninja_log_sample/b.c
+12	0	tests/fixtures/ninja_log_sample/build.ninja
+3	0	tests/fixtures/ninja_log_sample/c.c
+83	0	tests/fixtures/template_heavy.cpp
+1	0	tests/fixtures/template_heavy.ftime-trace.json
+219	0	tests/test_ci_prev_green.py
+202	0	tests/test_clang_time_trace.py
+174	0	tests/test_criterion.py
+322	0	tests/test_gha_log.py
+153	0	tests/test_ninja_log.py