From ba7bfac44c371a05cdfb954dfbcc5f55646a56e2 Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:24:13 -0400 Subject: [PATCH 01/10] scripts: Restore structural matching benchmark The project has a structural matcher benchmark that models several line layouts and reports matcher resource use from a source checkout. The benchmark depends on removed buffer and mapping interfaces, so it cannot run. The project therefore lacks repeatable measurements for a computationally expensive workflow. This commit begins restoring that performance coverage by rebuilding the runner around public line-buffer and matching APIs. It provides seeded quick and full workloads while keeping setup outside the measured phases. Subsequent commits will validate the runner, add ownership attribution coverage, expose the suite in CI and documentation, and provide report comparison. --- scripts/benchmark_matching.py | 759 +++++++++++++++++++++++++++------- 1 file changed, 618 insertions(+), 141 deletions(-) diff --git a/scripts/benchmark_matching.py b/scripts/benchmark_matching.py index 557780402..d111422c0 100644 --- a/scripts/benchmark_matching.py +++ b/scripts/benchmark_matching.py @@ -1,198 +1,675 @@ #!/usr/bin/env python3 -"""Benchmark structural matching storage and resource usage.""" +"""Benchmark public structural matching workflows.""" from __future__ import annotations import argparse -from collections.abc import Iterable -from contextlib import contextmanager +from collections.abc import Callable, Sequence +from dataclasses import dataclass +import gc +import hashlib import json -import os +import math from pathlib import Path -import resource +import platform +import random +import statistics +import subprocess import sys import time import tracemalloc -from typing import Any +from typing import Any, TypeVar + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT / "src")) + +from git_stage_batch import __version__ +from git_stage_batch.batch.attribution_units import ( + AttributionUnit, + FileComparison, + build_file_comparison_from_lines, + enumerate_units_from_file_comparison, +) +from git_stage_batch.batch.line_mapping import LineMapping +from git_stage_batch.batch.match import match_lines +from git_stage_batch.core.buffer import LineBuffer + + +SCHEMA_VERSION = 1 +DEFAULT_SEED = 20260711 +FIXTURE_CHUNK_SIZE = 64 * 1024 +_StateT = TypeVar("_StateT") + + +@dataclass(frozen=True) +class MatchingFixture: + """Prepared text inputs whose construction is outside measured phases.""" + + source_chunks: tuple[bytes, ...] + target_chunks: tuple[bytes, ...] + source_line_count: int + target_line_count: int + source_byte_count: int + target_byte_count: int + + +@dataclass(frozen=True) +class CaseDefinition: + """One deterministic benchmark case and its mode-specific dimensions.""" + + name: str + description: str + kind: str + quick_size: int + full_size: int | None + + + +@dataclass +class _BufferLoadingState: + """Resources created by one measured buffer-loading operation.""" + + fixture: MatchingFixture + source: LineBuffer | None = None + target: LineBuffer | None = None + + +@dataclass +class _MappingState: + """Prepared buffers and the mapping retained until measurement ends.""" + + source: LineBuffer + target: LineBuffer + mapping: LineMapping | None = None + + +@dataclass +class _UnitEnumerationState: + """Prepared comparison and units retained until measurement ends.""" + + source: LineBuffer + target: LineBuffer + comparison: FileComparison + units: dict[str, AttributionUnit] | None = None + + + +CASES = ( + CaseDefinition( + "small-interactive", + "A short file with nearby insertions, replacements, and deletions.", + "text", + 80, + 400, + ), + CaseDefinition( + "repeated-lines", + "Ambiguous repeated content separated by stable anchors.", + "text", + 300, + 5_000, + ), + CaseDefinition( + "unicode", + "UTF-8 text containing multibyte scripts and emoji.", + "text", + 150, + 5_000, + ), + CaseDefinition( + "low-similarity", + "Pathological source and target inputs with almost no shared lines.", + "text", + 250, + 4_000, + ), + CaseDefinition( + "reversed-unique", + "Unique lines in reverse order to stress candidate ordering.", + "text", + 250, + 4_000, + ), + CaseDefinition( + "binary-exclusion", + "NUL-containing input excluded from the text-matching pipeline.", + "binary", + 32, + 1_024, + ), + CaseDefinition( + "large-file", + "A large mostly stable file with sparse edits.", + "text", + 0, + 50_000, + ), +) +CASE_BY_NAME = {case.name: case for case in CASES} + + +def _case_random(seed: int, case_name: str) -> random.Random: + digest = hashlib.sha256(f"{seed}:{case_name}".encode()).digest() + return random.Random(int.from_bytes(digest[:8], "big")) + + +def _unique_lines(prefix: str, count: int) -> list[bytes]: + return [f"{prefix} {index:08d}\n".encode() for index in range(count)] + + +def _bounded_chunks( + lines: Sequence[bytes], + chunk_size: int = FIXTURE_CHUNK_SIZE, +) -> tuple[bytes, ...]: + payload = b"".join(lines) + return tuple( + payload[offset : offset + chunk_size] + for offset in range(0, len(payload), chunk_size) + ) -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from git_stage_batch.batch import match as match_module -from git_stage_batch.batch.match import LineMapping -from git_stage_batch.batch.match_storage import MatcherWorkspace -from git_stage_batch.editor import EditorBuffer +def _matching_fixture( + source_lines: Sequence[bytes], + target_lines: Sequence[bytes], +) -> MatchingFixture: + return MatchingFixture( + source_chunks=_bounded_chunks(source_lines), + target_chunks=_bounded_chunks(target_lines), + source_line_count=len(source_lines), + target_line_count=len(target_lines), + source_byte_count=sum(map(len, source_lines)), + target_byte_count=sum(map(len, target_lines)), + ) -class _MeasuredWorkspace(MatcherWorkspace): - """Matcher workspace that reports allocation counters on close.""" +def _chunks_sha256(chunks: Sequence[bytes]) -> str: + digest = hashlib.sha256() + for chunk in chunks: + digest.update(chunk) + return digest.hexdigest() - recorder: "_WorkspaceRecorder | None" = None - def close(self) -> None: - recorder = self.recorder - if recorder is not None: - recorder.observe_workspace(self) - super().close() +def build_matching_fixture(case_name: str, size: int, seed: int) -> MatchingFixture: + """Build one seeded matching fixture without running measured code.""" + if size < 0: + raise ValueError("fixture size must be non-negative") + randomizer = _case_random(seed, case_name) + if case_name in {"small-interactive", "large-file"}: + source = _unique_lines("stable", size) + target = list(source) + edit_count = max(1, size // (20 if case_name == "small-interactive" else 1_000)) + positions = sorted(randomizer.sample(range(size), min(edit_count, size))) + for number, position in enumerate(positions): + target[position] = f"replacement {number:08d}\n".encode() + insertion = min(size, size // 3) + target[insertion:insertion] = [b"inserted alpha\n", b"inserted beta\n"] + if size > 4: + del target[(size * 2) // 3] + return _matching_fixture(source, target) + + if case_name == "repeated-lines": + source = [] + for index in range(size): + source.append( + f"anchor {index:08d}\n".encode() if index % 23 == 0 else b"same\n" + ) + target = list(source) + if target: + target.insert(len(target) // 2, b"same\n") + target.pop(min(len(target) - 1, len(target) // 3)) + return _matching_fixture(source, target) + + if case_name == "unicode": + samples = ("naive cafe", "café", "東京", "مرحبا", "🧪", "Straße") + source = [ + f"{samples[index % len(samples)]} {index:08d}\n".encode("utf-8") + for index in range(size) + ] + target = list(source) + if target: + target[len(target) // 2] = "更新 🚀\n".encode("utf-8") + return _matching_fixture(source, target) -class _WorkspaceRecorder: - """Collect matcher workspace metrics across recursive matching.""" + if case_name == "low-similarity": + source = _unique_lines("source", size) + target = _unique_lines("target", size) + if source and target: + target[len(target) // 2] = source[len(source) // 2] + return _matching_fixture(source, target) - def __init__(self) -> None: - self.high_water_bytes = 0 - self.total_allocated_bytes = 0 - self.workspace_count = 0 - self.candidate_count = 0 + if case_name == "reversed-unique": + source = _unique_lines("reversed", size) + return _matching_fixture(source, list(reversed(source))) - def observe_workspace(self, workspace: MatcherWorkspace) -> None: - self.workspace_count += 1 - self.high_water_bytes = max( - self.high_water_bytes, - workspace.high_water_bytes, - ) - self.total_allocated_bytes += workspace.total_allocated_bytes + if case_name == "binary-exclusion": + payload = tuple(b"binary\0payload\n" for _ in range(size)) + return _matching_fixture(payload, payload) + + raise ValueError(f"{case_name!r} does not define a matching fixture") + + +def _percentile(values: Sequence[float | int], percentile: float) -> float: + ordered = sorted(values) + if not ordered: + raise ValueError("cannot summarize an empty sample") + rank = max(0, math.ceil(percentile * len(ordered)) - 1) + return float(ordered[rank]) + + +def _summary(values: Sequence[float | int]) -> dict[str, float]: + return { + "minimum": float(min(values)), + "median": float(statistics.median(values)), + "p95": _percentile(values, 0.95), + "maximum": float(max(values)), + } -def _open_fd_count() -> int | None: - fd_path = "/proc/self/fd" - if not os.path.isdir(fd_path): - return None - return len(os.listdir(fd_path)) +def _measure_phase( + prepare: Callable[[], _StateT], + operation: Callable[[_StateT], dict[str, Any]], + cleanup: Callable[[_StateT], None], + *, + warmups: int, + repeats: int, +) -> dict[str, Any]: + """Measure untraced time and traced Python allocations in separate runs.""" + for _ in range(warmups): + state = prepare() + try: + operation(state) + finally: + cleanup(state) + + expected_result: dict[str, Any] | None = None + + def observe_result(result: dict[str, Any]) -> None: + nonlocal expected_result + if expected_result is None: + expected_result = result + elif result != expected_result: + raise RuntimeError("benchmark phase produced inconsistent results") + + timing_samples = [] + for _ in range(repeats): + state = prepare() + try: + gc.collect() + started = time.perf_counter() + result = operation(state) + elapsed = time.perf_counter() - started + finally: + cleanup(state) + observe_result(result) + timing_samples.append(elapsed) + + memory_samples = [] + for _ in range(min(repeats, 3)): + if tracemalloc.is_tracing(): + raise RuntimeError("benchmark cannot reuse an active tracemalloc session") + state = prepare() + started_tracing = False + try: + gc.collect() + tracemalloc.start() + started_tracing = True + result = operation(state) + _, peak_memory = tracemalloc.get_traced_memory() + finally: + if started_tracing: + tracemalloc.stop() + cleanup(state) + observe_result(result) + memory_samples.append(peak_memory) + return { + "seconds": _summary(timing_samples), + "tracemalloc_peak_bytes": _summary(memory_samples), + "samples": { + "seconds": timing_samples, + "tracemalloc_peak_bytes": memory_samples, + }, + "result": expected_result or {}, + } -def _rss_bytes() -> int | None: + +def _open_matching_buffers(fixture: MatchingFixture) -> _MappingState: + source = LineBuffer.from_chunks(fixture.source_chunks) + target: LineBuffer | None = None try: - return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024 + target = LineBuffer.from_chunks(fixture.target_chunks) + # Force lazy line indexing during preparation, outside algorithm timings. + len(source) + len(target) except Exception: - return None + source.close() + if target is not None: + target.close() + raise + return _MappingState(source=source, target=target) -def _line_span_bytes(buffer: EditorBuffer) -> int: - line_spans = getattr(buffer, "_line_spans", None) - records = getattr(line_spans, "_records", None) - byte_count = getattr(records, "byte_count", 0) - return byte_count if isinstance(byte_count, int) else 0 +def _close_mapping(state: _MappingState) -> None: + try: + if state.mapping is not None: + state.mapping.close() + finally: + try: + state.source.close() + finally: + state.target.close() -def _line_payloads(pattern: str, line_count: int) -> tuple[Iterable[bytes], Iterable[bytes]]: - if pattern == "identical": - source = [f"line {index}\n".encode() for index in range(line_count)] - return source, list(source) +def _measure_buffer_loading(state: _BufferLoadingState) -> dict[str, Any]: + state.source = LineBuffer.from_chunks(state.fixture.source_chunks) + state.target = LineBuffer.from_chunks(state.fixture.target_chunks) + return { + "source_bytes": state.source.byte_count, + "source_lines": len(state.source), + "target_bytes": state.target.byte_count, + "target_lines": len(state.target), + "uses_mapped_storage": state.source.uses_mapped_storage + or state.target.uses_mapped_storage, + } - if pattern == "unique-inserted-prefix": - source = [f"line {index}\n".encode() for index in range(line_count)] - target = [f"extra {index}\n".encode() for index in range(line_count // 10)] - target.extend(source) - return source, target - if pattern == "repeated": - source = [b"same\n" for _ in range(line_count)] - target = list(source) - return source, target +def _close_buffer_loading(state: _BufferLoadingState) -> None: + try: + if state.source is not None: + state.source.close() + finally: + if state.target is not None: + state.target.close() - if pattern == "reversed-unique": - source = [f"line {index}\n".encode() for index in range(line_count)] - return source, list(reversed(source)) - raise ValueError(f"unknown pattern: {pattern}") +def _measure_mapping(state: _MappingState) -> dict[str, Any]: + state.mapping = match_lines(state.source, state.target) + return { + "mapped_lines": sum(1 for _ in state.mapping.mapped_line_pairs()) + } -@contextmanager -def _instrument_matcher(recorder: _WorkspaceRecorder): - original_workspace = match_module.MatcherWorkspace - original_lis = match_module._longest_increasing_subsequence_records +def _prepare_unit_enumeration( + fixture: MatchingFixture, +) -> _UnitEnumerationState: + mapping_state = _open_matching_buffers(fixture) + try: + comparison = build_file_comparison_from_lines( + "benchmark.txt", + baseline_lines=mapping_state.source, + working_tree_lines=mapping_state.target, + ) + except Exception: + _close_mapping(mapping_state) + raise + return _UnitEnumerationState( + source=mapping_state.source, + target=mapping_state.target, + comparison=comparison, + ) + - def measured_lis(pairs, target_start, target_end, workspace): - recorder.candidate_count += len(pairs) - return original_lis(pairs, target_start, target_end, workspace) +def _measure_unit_enumeration( + state: _UnitEnumerationState, +) -> dict[str, Any]: + state.units = {} + enumerate_units_from_file_comparison(state.comparison, state.units) + kinds: dict[str, int] = {} + for unit in state.units.values(): + kinds[unit.kind.value] = kinds.get(unit.kind.value, 0) + 1 + return {"units": len(state.units), "unit_kinds": kinds} - _MeasuredWorkspace.recorder = recorder - match_module.MatcherWorkspace = _MeasuredWorkspace - match_module._longest_increasing_subsequence_records = measured_lis + +def _close_unit_enumeration(state: _UnitEnumerationState) -> None: + state.units = None try: - yield + state.comparison.close() finally: - match_module._longest_increasing_subsequence_records = original_lis - match_module.MatcherWorkspace = original_workspace - _MeasuredWorkspace.recorder = None + try: + state.source.close() + finally: + state.target.close() + + +def run_matching_case( + case: CaseDefinition, + size: int, + seed: int, + *, + warmups: int, + repeats: int, +) -> dict[str, Any]: + """Run one text case with setup excluded from all measured phases.""" + fixture = build_matching_fixture(case.name, size, seed) + dimensions = { + "requested_lines": size, + "source_lines": fixture.source_line_count, + "target_lines": fixture.target_line_count, + "source_bytes": fixture.source_byte_count, + "target_bytes": fixture.target_byte_count, + "source_chunks": len(fixture.source_chunks), + "target_chunks": len(fixture.target_chunks), + "chunk_size_bytes": FIXTURE_CHUNK_SIZE, + "source_sha256": _chunks_sha256(fixture.source_chunks), + "target_sha256": _chunks_sha256(fixture.target_chunks), + } + if case.kind == "binary": + contains_nul = any(b"\0" in chunk for chunk in fixture.source_chunks) + return { + "name": case.name, + "description": case.description, + "category": "exclusion", + "status": "excluded", + "exclusion_reason": "NUL-containing input is not text-matching work", + "dimensions": {**dimensions, "contains_nul": contains_nul}, + "setup": {"measured": False, "seed": seed}, + "phases": {}, + } + + phases = { + "buffer_loading": _measure_phase( + lambda: _BufferLoadingState(fixture), + _measure_buffer_loading, + _close_buffer_loading, + warmups=warmups, + repeats=repeats, + ), + "mapping": _measure_phase( + lambda: _open_matching_buffers(fixture), + _measure_mapping, + _close_mapping, + warmups=warmups, + repeats=repeats, + ), + "unit_attribution": _measure_phase( + lambda: _prepare_unit_enumeration(fixture), + _measure_unit_enumeration, + _close_unit_enumeration, + warmups=warmups, + repeats=repeats, + ), + } + return { + "name": case.name, + "description": case.description, + "category": "matching", + "status": "measured", + "dimensions": dimensions, + "setup": { + "measured": False, + "seed": seed, + "description": "Payload generation and phase prerequisites are excluded.", + }, + "phases": phases, + } -def _mapping_source_to_target_bytes(mapping: LineMapping) -> int: - source_bytes = getattr(mapping.source_to_target, "byte_count", 0) - target_bytes = getattr(mapping.target_to_source, "byte_count", 0) - return ( - (source_bytes if isinstance(source_bytes, int) else 0) - + (target_bytes if isinstance(target_bytes, int) else 0) +def _command_output(arguments: Sequence[str]) -> str: + try: + return subprocess.run( + arguments, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +def _metadata(seed: int, warmups: int, repeats: int) -> dict[str, Any]: + project_version = __version__ + if project_version == "unknown": + try: + project_version = (PROJECT_ROOT / "VERSION").read_text( + encoding="utf-8" + ).strip() + except OSError: + pass + working_tree_status = _command_output( + [ + "git", + "-C", + str(PROJECT_ROOT), + "status", + "--porcelain", + "--untracked-files=no", + ] ) + return { + "project_version": project_version, + "python_version": platform.python_version(), + "python_implementation": platform.python_implementation(), + "git_version": _command_output(["git", "--version"]), + "platform": platform.platform(), + "revision": _command_output( + ["git", "-C", str(PROJECT_ROOT), "rev-parse", "HEAD"] + ), + "working_tree_dirty": ( + None if working_tree_status == "unknown" else bool(working_tree_status) + ), + "seed": seed, + "warmups": warmups, + "repeats": repeats, + "memory_repeats": min(repeats, 3), + "clock": "time.perf_counter", + "timing_metric": "perf_counter without tracemalloc", + "memory_metric": "tracemalloc peak Python allocations", + } -def run_benchmark(pattern: str, line_count: int) -> dict[str, Any]: - source_payloads, target_payloads = _line_payloads(pattern, line_count) - recorder = _WorkspaceRecorder() - fd_before = _open_fd_count() - rss_before = _rss_bytes() - tracemalloc.start() - start_time = time.perf_counter() - - with ( - EditorBuffer.from_chunks(source_payloads) as source, - EditorBuffer.from_chunks(target_payloads) as target, - ): - with _instrument_matcher(recorder): - with match_module.match_lines(source, target) as mapping: - mapped_line_count = sum(1 for _ in mapping.mapped_line_pairs()) - mapping_bytes = _mapping_source_to_target_bytes(mapping) - - source_line_count = len(source) - target_line_count = len(target) - line_span_bytes = _line_span_bytes(source) + _line_span_bytes(target) - - elapsed = time.perf_counter() - start_time - _, peak_heap = tracemalloc.get_traced_memory() - tracemalloc.stop() - rss_after = _rss_bytes() - fd_after = _open_fd_count() +def selected_cases( + mode: str, + names: Sequence[str] | None = None, +) -> list[CaseDefinition]: + """Select named cases, or every case available in the requested mode.""" + if names: + cases = [CASE_BY_NAME[name] for name in dict.fromkeys(names)] + else: + cases = list(CASES) + if mode == "quick": + unavailable = [case.name for case in cases if case.quick_size <= 0] + if names and unavailable: + raise ValueError( + f"{', '.join(unavailable)} requires --mode full" + ) + return [case for case in cases if case.quick_size > 0] + return [case for case in cases if case.full_size is not None] + + +def run_suite( + mode: str = "quick", + *, + case_names: Sequence[str] | None = None, + seed: int = DEFAULT_SEED, + warmups: int = 1, + repeats: int = 3, +) -> dict[str, Any]: + """Run selected deterministic cases and return the stable JSON schema.""" + if mode not in {"quick", "full"}: + raise ValueError("mode must be 'quick' or 'full'") + if warmups < 0: + raise ValueError("warmups must be non-negative") + if repeats <= 0: + raise ValueError("repeats must be positive") + + results = [] + for case in selected_cases(mode, case_names): + size = case.quick_size if mode == "quick" else case.full_size + if size is None or size <= 0: + continue + result = run_matching_case( + case, + size, + seed, + warmups=warmups, + repeats=repeats, + ) + results.append(result) return { - "pattern": pattern, - "requested_source_lines": line_count, - "source_lines": source_line_count, - "target_lines": target_line_count, - "mapped_line_count": mapped_line_count, - "candidate_count": recorder.candidate_count, - "tracemalloc_peak_bytes": peak_heap, - "rss_before_bytes": rss_before, - "rss_after_bytes": rss_after, - "elapsed_seconds": elapsed, - "fd_before": fd_before, - "fd_after": fd_after, - "line_span_bytes": line_span_bytes, - "line_mapping_bytes": mapping_bytes, - "matcher_workspace_high_water_bytes": recorder.high_water_bytes, - "matcher_workspace_total_allocated_bytes": recorder.total_allocated_bytes, - "matcher_workspace_count": recorder.workspace_count, + "schema_version": SCHEMA_VERSION, + "suite": "matching", + "mode": mode, + "metadata": _metadata(seed, warmups, repeats), + "cases": results, } -def main() -> None: - parser = argparse.ArgumentParser() +def _write_report(report: dict[str, Any], output: Path | None) -> None: + rendered = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n" + if output is not None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + sys.stdout.write(rendered) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=("quick", "full"), default="quick") parser.add_argument( - "--pattern", - choices=[ - "identical", - "unique-inserted-prefix", - "repeated", - "reversed-unique", - ], - default="unique-inserted-prefix", + "--case", + action="append", + choices=tuple(CASE_BY_NAME), + dest="case_names", + help="run one case (repeat to select more than one)", ) - parser.add_argument("--lines", type=int, default=10000) - args = parser.parse_args() + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--warmups", type=int) + parser.add_argument("--repeats", type=int) + parser.add_argument("--output", type=Path) + return parser + - if args.lines < 0: - raise SystemExit("--lines must be non-negative") +def main() -> None: + parser = _parser() + args = parser.parse_args() - print(json.dumps(run_benchmark(args.pattern, args.lines), indent=2)) + warmups = ( + args.warmups + if args.warmups is not None + else (1 if args.mode == "quick" else 2) + ) + repeats = ( + args.repeats + if args.repeats is not None + else (3 if args.mode == "quick" else 7) + ) + try: + report = run_suite( + args.mode, + case_names=args.case_names, + seed=args.seed, + warmups=warmups, + repeats=repeats, + ) + except ValueError as error: + parser.error(str(error)) + try: + _write_report(report, args.output) + except OSError as error: + parser.error(str(error)) if __name__ == "__main__": From ae73fff30bf1ed2a82e762a42c7b1a6692f0bb90 Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:36:20 -0400 Subject: [PATCH 02/10] tests: Cover structural matching benchmark The structural matching benchmark now runs deterministic quick and full workloads through the public matching interfaces. The project does not yet verify the benchmark schema, fixture shapes, resource boundaries, or adversarial matcher behavior. API drift could therefore make the runner unusable again without a focused failure. This commit addresses that validation gap with smoke coverage for the smallest workload, repeated and reordered inputs, binary exclusion, bounded fixture storage, and separate timing and allocation passes. The next implementation commit will extend the benchmark with ownership attribution measurements. --- tests/test_benchmark_matching.py | 211 +++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 tests/test_benchmark_matching.py diff --git a/tests/test_benchmark_matching.py b/tests/test_benchmark_matching.py new file mode 100644 index 000000000..0634fcdeb --- /dev/null +++ b/tests/test_benchmark_matching.py @@ -0,0 +1,211 @@ +"""Tests for the maintained structural matching benchmark.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys + +import pytest + + +SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "benchmark_matching.py" +SPEC = importlib.util.spec_from_file_location("benchmark_matching", SCRIPT_PATH) +assert SPEC is not None +assert SPEC.loader is not None +benchmark_matching = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = benchmark_matching +SPEC.loader.exec_module(benchmark_matching) + + +def test_seeded_fixtures_are_deterministic_and_cover_edge_cases(): + """Generated fixtures should be repeatable and retain adversarial shapes.""" + repeated = benchmark_matching.build_matching_fixture( + "repeated-lines", 100, 42 + ) + repeated_again = benchmark_matching.build_matching_fixture( + "repeated-lines", 100, 42 + ) + low_similarity = benchmark_matching.build_matching_fixture( + "low-similarity", 100, 42 + ) + binary = benchmark_matching.build_matching_fixture("binary-exclusion", 2, 42) + large = benchmark_matching.build_matching_fixture("large-file", 5_000, 42) + seeded_edit = benchmark_matching.build_matching_fixture( + "small-interactive", 80, 42 + ) + differently_seeded_edit = benchmark_matching.build_matching_fixture( + "small-interactive", 80, 43 + ) + repeated_source_lines = b"".join(repeated.source_chunks).splitlines( + keepends=True + ) + low_source_lines = b"".join(low_similarity.source_chunks).splitlines( + keepends=True + ) + low_target_lines = b"".join(low_similarity.target_chunks).splitlines( + keepends=True + ) + + assert repeated == repeated_again + assert repeated_source_lines.count(b"same\n") > 90 + assert set(low_source_lines) & set(low_target_lines) == {low_source_lines[50]} + assert b"\0" in b"".join(binary.source_chunks) + assert large.source_line_count == 5_000 + assert len(large.source_chunks) > 1 + assert max(map(len, large.source_chunks)) <= ( + benchmark_matching.FIXTURE_CHUNK_SIZE + ) + assert seeded_edit.target_chunks != differently_seeded_edit.target_chunks + + +def test_smallest_case_smokes_public_apis_and_stable_schema(): + """The benchmark should remain importable and runnable after API changes.""" + report = benchmark_matching.run_suite( + "quick", + case_names=["small-interactive"], + seed=42, + warmups=0, + repeats=1, + ) + + assert report["schema_version"] == 1 + assert report["suite"] == "matching" + assert report["metadata"]["seed"] == 42 + assert isinstance(report["metadata"]["working_tree_dirty"], bool) + case = report["cases"][0] + assert case["name"] == "small-interactive" + assert case["setup"]["measured"] is False + assert set(case["phases"]) == { + "buffer_loading", + "mapping", + "unit_attribution", + } + for phase in case["phases"].values(): + assert set(phase) == { + "seconds", + "tracemalloc_peak_bytes", + "samples", + "result", + } + assert len(phase["samples"]["seconds"]) == 1 + assert len(phase["samples"]["tracemalloc_peak_bytes"]) == 1 + + +@pytest.mark.parametrize( + ("case_name", "expected_mapped_lines"), + ( + ("repeated-lines", 99), + ("low-similarity", 1), + ("reversed-unique", 1), + ), +) +def test_adversarial_matching_cases_run(case_name, expected_mapped_lines): + """Ambiguous and low-similarity inputs should traverse the real matcher.""" + case = benchmark_matching.CASE_BY_NAME[case_name] + result = benchmark_matching.run_matching_case( + case, + 100, + 42, + warmups=0, + repeats=1, + ) + + assert result["phases"]["mapping"]["result"]["mapped_lines"] == ( + expected_mapped_lines + ) + + + +def test_binary_case_is_explicitly_excluded_from_text_matching(): + """Binary fixtures should document exclusion and run no matching phases.""" + report = benchmark_matching.run_suite( + "quick", + case_names=["binary-exclusion"], + warmups=0, + repeats=1, + ) + + case = report["cases"][0] + assert case["status"] == "excluded" + assert case["dimensions"]["contains_nul"] is True + assert case["phases"] == {} + + +def test_phase_preparation_and_cleanup_are_outside_the_timer(monkeypatch): + """Setup stays untimed and allocation tracing uses a separate run.""" + events = [] + ticks = iter((10.0, 10.25)) + tracing = False + + def prepare(): + events.append("prepare") + return "state" + + def operation(state): + assert state == "state" + events.append("operation") + return {"answer": 42} + + def cleanup(state): + assert state == "state" + events.append("cleanup") + + def perf_counter(): + events.append("clock") + return next(ticks) + + def start_tracing(): + nonlocal tracing + assert not tracing + tracing = True + events.append("trace-start") + + def read_tracing(): + assert tracing + events.append("trace-read") + return 0, 123 + + def stop_tracing(): + nonlocal tracing + assert tracing + tracing = False + events.append("trace-stop") + + monkeypatch.setattr(benchmark_matching.time, "perf_counter", perf_counter) + monkeypatch.setattr(benchmark_matching.tracemalloc, "start", start_tracing) + monkeypatch.setattr( + benchmark_matching.tracemalloc, + "get_traced_memory", + read_tracing, + ) + monkeypatch.setattr(benchmark_matching.tracemalloc, "stop", stop_tracing) + monkeypatch.setattr( + benchmark_matching.tracemalloc, + "is_tracing", + lambda: tracing, + ) + phase = benchmark_matching._measure_phase( + prepare, + operation, + cleanup, + warmups=0, + repeats=1, + ) + + assert events == [ + "prepare", + "clock", + "operation", + "clock", + "cleanup", + "prepare", + "trace-start", + "operation", + "trace-read", + "trace-stop", + "cleanup", + ] + assert phase["seconds"]["median"] == pytest.approx(0.25) + assert phase["samples"]["seconds"] == [pytest.approx(0.25)] + assert phase["samples"]["tracemalloc_peak_bytes"] == [123] From e97b33423b8f753ff50af71cd116720670693525 Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:36:51 -0400 Subject: [PATCH 03/10] scripts: Benchmark ownership attribution The benchmark suite measures structural matching across deterministic text layouts and records phase-level time and Python allocation data. Ownership attribution adds Git object resolution, streamed blob loading, shared source mappings, and per-batch claim evaluation to that path. Matching-only results cannot locate regressions in those repository-backed phases. This commit extends the suite with a fixed-size repository fixture whose quick and full modes vary the number of batch claims. Distinct state refs resolve to one source blob so the report exposes deduplicated mapping work. The next commit will validate the attribution workload before CI and documentation expose the complete runner. --- scripts/benchmark_matching.py | 371 +++++++++++++++++++++++++++++++++- 1 file changed, 361 insertions(+), 10 deletions(-) diff --git a/scripts/benchmark_matching.py b/scripts/benchmark_matching.py index d111422c0..a31f5007a 100644 --- a/scripts/benchmark_matching.py +++ b/scripts/benchmark_matching.py @@ -1,21 +1,24 @@ #!/usr/bin/env python3 -"""Benchmark public structural matching workflows.""" +"""Benchmark public matching and ownership-attribution workflows.""" from __future__ import annotations import argparse from collections.abc import Callable, Sequence -from dataclasses import dataclass +from contextlib import contextmanager +from dataclasses import asdict, dataclass import gc import hashlib import json import math +import os from pathlib import Path import platform import random import statistics import subprocess import sys +import tempfile import time import tracemalloc from typing import Any, TypeVar @@ -25,6 +28,11 @@ sys.path.insert(0, str(PROJECT_ROOT / "src")) from git_stage_batch import __version__ +from git_stage_batch.batch.attribution import ( + AttributionMetrics, + FileAttribution, + build_file_attribution, +) from git_stage_batch.batch.attribution_units import ( AttributionUnit, FileComparison, @@ -33,7 +41,14 @@ ) from git_stage_batch.batch.line_mapping import LineMapping from git_stage_batch.batch.match import match_lines +from git_stage_batch.batch.state_refs import get_batch_state_ref_name from git_stage_batch.core.buffer import LineBuffer +from git_stage_batch.utils.git_object_io import resolve_git_objects +from git_stage_batch.utils.repository_buffers import ( + load_git_object_as_buffer_or_empty, + load_working_tree_file_as_buffer, + stream_git_blob_buffers, +) SCHEMA_VERSION = 1 @@ -65,6 +80,20 @@ class CaseDefinition: full_size: int | None +@dataclass(frozen=True) +class AttributionFixture: + """Prepared repository and metadata for the attribution benchmark.""" + + file_path: str + metadata: dict[str, dict[str, Any]] + object_names: tuple[str, ...] + source_object_ids: tuple[str, ...] + baseline_line_count: int + working_line_count: int + baseline_sha256: str + working_sha256: str + batch_count: int + @dataclass class _BufferLoadingState: @@ -94,6 +123,13 @@ class _UnitEnumerationState: units: dict[str, AttributionUnit] | None = None +@dataclass +class _ClaimAttributionState: + """Prepared claim fixture and result retained until measurement ends.""" + + fixture: AttributionFixture + attribution: FileAttribution | None = None + CASES = ( CaseDefinition( @@ -138,6 +174,13 @@ class _UnitEnumerationState: 32, 1_024, ), + CaseDefinition( + "many-batches", + "Many ownership claims sharing one source blob and line mapping.", + "attribution", + 50, + 1_000, + ), CaseDefinition( "large-file", "A large mostly stable file with sparse edits.", @@ -506,6 +549,305 @@ def run_matching_case( } +def _git(repo: Path, *arguments: str, input_text: str | None = None) -> str: + return subprocess.run( + ["git", *arguments], + cwd=repo, + input=input_text, + text=True, + check=True, + capture_output=True, + ).stdout.strip() + + +def _build_attribution_fixture(repo: Path, batch_count: int) -> AttributionFixture: + if batch_count <= 0: + raise ValueError("batch count must be positive") + file_path = "benchmark.txt" + line_count = 300 + baseline = _unique_lines("baseline", line_count) + working = list(baseline) + claim_line = max(1, line_count // 2) + working[claim_line - 1] = b"owned replacement\n" + working.append(b"owned addition\n") + baseline_payload = b"".join(baseline) + working_payload = b"".join(working) + + _git(repo, "init", "--quiet") + _git(repo, "config", "user.name", "Benchmark") + _git(repo, "config", "user.email", "benchmark@example.invalid") + path = repo / file_path + path.write_bytes(baseline_payload) + _git(repo, "add", "--", file_path) + _git(repo, "commit", "--quiet", "-m", "benchmark baseline") + + path.write_bytes(working_payload) + _git(repo, "add", "--", file_path) + source_tree = _git(repo, "write-tree") + source_commit = _git( + repo, + "commit-tree", + source_tree, + "-p", + "HEAD", + "-m", + "benchmark source", + ) + _git(repo, "reset", "--mixed", "HEAD") + + batch_names = tuple(f"benchmark-{index:04d}" for index in range(batch_count)) + metadata = { + name: { + "files": { + file_path: { + "batch_source_commit": source_commit, + "source_path": f"sources/{file_path}", + "presence_claims": [ + {"source_lines": [str(claim_line), str(line_count + 1)]} + ], + "deletions": [], + } + } + } + for name in batch_names + } + fallback = f"{source_commit}:{file_path}" + source_object_id = _git(repo, "rev-parse", fallback) + sources_tree = _git( + repo, + "mktree", + input_text=f"100644 blob {source_object_id}\t{file_path}\n", + ) + state_tree = _git( + repo, + "mktree", + input_text=f"040000 tree {sources_tree}\tsources\n", + ) + state_commits = { + name: _git( + repo, + "commit-tree", + state_tree, + "-p", + "HEAD", + "-m", + f"benchmark state {name}", + ) + for name in batch_names + } + _git( + repo, + "update-ref", + "--stdin", + input_text="".join( + f"update {get_batch_state_ref_name(name)} {state_commits[name]}\n" + for name in batch_names + ), + ) + object_names = tuple( + [ + *( + f"{get_batch_state_ref_name(name)}:sources/{file_path}" + for name in batch_names + ), + fallback, + ] + ) + return AttributionFixture( + file_path=file_path, + metadata=metadata, + object_names=object_names, + source_object_ids=(source_object_id,), + baseline_line_count=len(baseline), + working_line_count=len(working), + baseline_sha256=hashlib.sha256(baseline_payload).hexdigest(), + working_sha256=hashlib.sha256(working_payload).hexdigest(), + batch_count=batch_count, + ) + + +@contextmanager +def _working_directory(path: Path): + previous = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(previous) + + +def _measure_object_resolution(fixture: AttributionFixture) -> dict[str, Any]: + resolved = resolve_git_objects(fixture.object_names) + return { + "requests": len(fixture.object_names), + "resolved": len(resolved), + "unique_object_ids": len({info.object_id for info in resolved.values()}), + } + + +def _measure_blob_loading(fixture: AttributionFixture) -> dict[str, Any]: + object_count = 0 + byte_count = 0 + line_count = 0 + for blob in stream_git_blob_buffers(fixture.source_object_ids): + object_count += 1 + byte_count += blob.size + line_count += len(blob.buffer) + return {"objects": object_count, "bytes": byte_count, "lines": line_count} + + +def _open_repository_buffers( + fixture: AttributionFixture, +) -> _MappingState: + source = load_git_object_as_buffer_or_empty( + next(iter(fixture.metadata.values()))["files"][fixture.file_path][ + "batch_source_commit" + ] + + f":{fixture.file_path}" + ) + working: LineBuffer | None = None + try: + working = load_working_tree_file_as_buffer(fixture.file_path) + len(source) + len(working) + except Exception: + source.close() + if working is not None: + working.close() + raise + return _MappingState(source=source, target=working) + + +def _measure_claim_attribution(state: _ClaimAttributionState) -> dict[str, Any]: + metrics = AttributionMetrics() + state.attribution = build_file_attribution( + state.fixture.file_path, + batch_metadata_by_name=state.fixture.metadata, + metrics=metrics, + ) + owned_units = sum( + bool(unit.owning_batches) for unit in state.attribution.units + ) + owner_links = sum( + len(unit.owning_batches) for unit in state.attribution.units + ) + return { + "units": len(state.attribution.units), + "owned_units": owned_units, + "owner_links": owner_links, + "metrics": asdict(metrics), + } + + +def _close_claim_attribution(state: _ClaimAttributionState) -> None: + state.attribution = None + + +def run_attribution_case( + case: CaseDefinition, + batch_count: int, + seed: int, + *, + warmups: int, + repeats: int, +) -> dict[str, Any]: + """Run the public attribution workflow in an isolated prepared repository.""" + del seed # This fixture is fully enumerated; keep the shared schema explicit. + with tempfile.TemporaryDirectory(prefix="git-stage-batch-matching-") as directory: + repo = Path(directory) + fixture = _build_attribution_fixture(repo, batch_count) + with _working_directory(repo): + phases = { + "git_object_resolution": _measure_phase( + lambda: fixture, + _measure_object_resolution, + lambda _state: None, + warmups=warmups, + repeats=repeats, + ), + "blob_loading": _measure_phase( + lambda: fixture, + _measure_blob_loading, + lambda _state: None, + warmups=warmups, + repeats=repeats, + ), + "mapping": _measure_phase( + lambda: _open_repository_buffers(fixture), + _measure_mapping, + _close_mapping, + warmups=warmups, + repeats=repeats, + ), + "unit_attribution": _measure_phase( + lambda: _prepare_repository_unit_enumeration(fixture), + _measure_unit_enumeration, + _close_unit_enumeration, + warmups=warmups, + repeats=repeats, + ), + "claim_attribution": _measure_phase( + lambda: _ClaimAttributionState(fixture), + _measure_claim_attribution, + _close_claim_attribution, + warmups=warmups, + repeats=repeats, + ), + } + + return { + "name": case.name, + "description": case.description, + "category": "attribution", + "status": "measured", + "dimensions": { + "batches": fixture.batch_count, + "baseline_lines": fixture.baseline_line_count, + "source_lines": fixture.working_line_count, + "target_lines": fixture.working_line_count, + "baseline_sha256": fixture.baseline_sha256, + "source_sha256": fixture.working_sha256, + "target_sha256": fixture.working_sha256, + "object_resolution_requests": len(fixture.object_names), + }, + "setup": { + "measured": False, + "description": ( + "Repository, commits, metadata, and prerequisites are excluded." + ), + }, + "phases": phases, + } + + +def _prepare_repository_unit_enumeration( + fixture: AttributionFixture, +) -> _UnitEnumerationState: + source = load_git_object_as_buffer_or_empty(f"HEAD:{fixture.file_path}") + try: + working = load_working_tree_file_as_buffer(fixture.file_path) + except Exception: + source.close() + raise + try: + len(source) + len(working) + comparison = build_file_comparison_from_lines( + fixture.file_path, + baseline_lines=source, + working_tree_lines=working, + ) + except Exception: + source.close() + working.close() + raise + return _UnitEnumerationState( + source=source, + target=working, + comparison=comparison, + ) + + def _command_output(arguments: Sequence[str]) -> str: try: return subprocess.run( @@ -599,18 +941,27 @@ def run_suite( size = case.quick_size if mode == "quick" else case.full_size if size is None or size <= 0: continue - result = run_matching_case( - case, - size, - seed, - warmups=warmups, - repeats=repeats, - ) + if case.kind == "attribution": + result = run_attribution_case( + case, + size, + seed, + warmups=warmups, + repeats=repeats, + ) + else: + result = run_matching_case( + case, + size, + seed, + warmups=warmups, + repeats=repeats, + ) results.append(result) return { "schema_version": SCHEMA_VERSION, - "suite": "matching", + "suite": "matching-and-attribution", "mode": mode, "metadata": _metadata(seed, warmups, repeats), "cases": results, From e891407efca0469105f32b5f9abef0edcb45134b Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:37:14 -0400 Subject: [PATCH 04/10] tests: Cover ownership attribution benchmark The benchmark suite now includes a repository-backed workload that attributes shared source content across many batch claims. The new workload depends on state-ref resolution, source deduplication, unit enumeration, and ownership counts. Without focused assertions, a fixture mistake could produce plausible timings without exercising those paths. This commit addresses that risk by checking the measured attribution phases, the number of resolved expressions, the single blob request, the shared mapping count, and the resulting ownership links. The following CI and documentation commits will make the validated quick and full suites available to contributors. --- tests/test_benchmark_matching.py | 36 ++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/test_benchmark_matching.py b/tests/test_benchmark_matching.py index 0634fcdeb..208cf24f1 100644 --- a/tests/test_benchmark_matching.py +++ b/tests/test_benchmark_matching.py @@ -1,4 +1,4 @@ -"""Tests for the maintained structural matching benchmark.""" +"""Tests for the maintained matching and attribution benchmark.""" from __future__ import annotations @@ -70,7 +70,7 @@ def test_smallest_case_smokes_public_apis_and_stable_schema(): ) assert report["schema_version"] == 1 - assert report["suite"] == "matching" + assert report["suite"] == "matching-and-attribution" assert report["metadata"]["seed"] == 42 assert isinstance(report["metadata"]["working_tree_dirty"], bool) case = report["cases"][0] @@ -116,6 +116,38 @@ def test_adversarial_matching_cases_run(case_name, expected_mapped_lines): ) +def test_attribution_case_exercises_many_claims_and_deduplicates_work(): + """The end-to-end case should cover Git IO, matching, and claim ownership.""" + report = benchmark_matching.run_suite( + "quick", + case_names=["many-batches"], + warmups=0, + repeats=1, + ) + + case = report["cases"][0] + phases = case["phases"] + claim_result = phases["claim_attribution"]["result"] + metrics = claim_result["metrics"] + assert case["setup"]["measured"] is False + assert set(phases) == { + "git_object_resolution", + "blob_loading", + "mapping", + "unit_attribution", + "claim_attribution", + } + assert metrics["claimed_batches"] == 50 + assert metrics["object_requests"] == 1 + assert metrics["mapping_computations"] == 1 + assert claim_result["owner_links"] == 100 + assert phases["unit_attribution"]["result"]["units"] == 2 + assert phases["git_object_resolution"]["result"] == { + "requests": 51, + "resolved": 51, + "unique_object_ids": 1, + } + def test_binary_case_is_explicitly_excluded_from_text_matching(): """Binary fixtures should document exclusion and run no matching phases.""" From 4b1aca6acb3e4dae769197a1d722dad1ff43e416 Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:37:32 -0400 Subject: [PATCH 05/10] ci: Smoke test matching benchmark The matching and ownership attribution benchmarks have focused local tests for their fixtures, schemas, and public API use. The regular CI job exercises the installed project only on Python 3.10 and does not retain benchmark output. Source-checkout drift on the newest supported interpreter can therefore go unnoticed. This commit addresses that gap with a quick-suite smoke job on Python 3.10 and 3.13. Each matrix entry uploads its JSON report without enforcing hardware-dependent timing thresholds. The next commits will explain the benchmark workflow and link that guide from the project website. --- .github/workflows/ci.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9af89a2ea..231b5c9ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,35 @@ on: branches: [main] jobs: + benchmark-smoke: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.13"] + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Smoke test matching benchmark + run: | + python scripts/benchmark_matching.py \ + --mode quick \ + --warmups 0 \ + --repeats 1 \ + --output matching-benchmark-smoke.json + + - name: Upload matching benchmark report + uses: actions/upload-artifact@v4 + with: + name: matching-benchmark-python-${{ matrix.python-version }} + path: matching-benchmark-smoke.json + test: runs-on: ubuntu-latest strategy: From b64d91c03363b543160bed56d0d6cbb2ac5449ae Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:37:55 -0400 Subject: [PATCH 06/10] docs: Explain matching benchmark workflow The project provides quick and full matching benchmarks, attribution phase measurements, JSON reports, and pull-request smoke coverage. Users cannot discover the available workloads, sampling controls, phase boundaries, or memory interpretation from the command itself. Results are difficult to reproduce without those details. This commit addresses that documentation gap with a benchmark guide covering invocation, deterministic fixture storage, measured phases, report metadata, and baseline refresh expectations. The next commit will expose the guide in website navigation. Later commits will add report comparison and document that workflow. --- docs/benchmarking.md | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 docs/benchmarking.md diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 000000000..c5b7d3487 --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,62 @@ +# Matching benchmarks + +The matching benchmark exercises the public line-matching and ownership +attribution APIs. It uses deterministic fixtures and emits JSON so results can +be retained and compared across revisions. + +Run the pull-request-sized suite from a source checkout: + +```console +python scripts/benchmark_matching.py --mode quick --output matching.json +``` + +The full suite increases the repeated-line, low-similarity, reversed-order, +Unicode, and many-batch workloads and adds a 50,000-line sparse-edit case: + +```console +python scripts/benchmark_matching.py --mode full --output matching-full.json +``` + +Use `--case NAME` to isolate a regression. The option can be repeated. Use +`--seed`, `--warmups`, and `--repeats` to control reproducibility and sampling. +The defaults are one warm-up and three samples in quick mode, and two warm-ups +and seven samples in full mode. The full suite can take several minutes on a +typical workstation. + +## Reading results + +Each measured phase reports every sample plus minimum, median, 95th percentile, +and maximum elapsed time and peak Python allocation. Elapsed-time samples run +without allocation tracing. Peak-allocation samples run separately, with at +most three samples per phase, so `tracemalloc` does not distort the timing +results. Fixture generation, repository creation, and separately managed phase +prerequisites and cleanup happen outside the timer. End-to-end public APIs still +include any cleanup that is intrinsic to the operation. + +Synthetic text is coalesced into bounded 64 KiB byte chunks before measurement, +so large cases do not retain a Python object for every line alongside the +line-addressable buffers. + +The phases are: + +- `buffer_loading`: create line-addressable buffers and build their line indexes. +- `git_object_resolution`: resolve the expressions needed by batch claims. +- `blob_loading`: stream source blobs into bounded line buffers. +- `mapping`: construct and traverse the structural line mapping. +- `unit_attribution`: enumerate changed units from an already-built comparison. +- `claim_attribution`: run file attribution end to end, including Git I/O, + matching, and ownership claims. + +The many-batch case keeps its file dimensions fixed between modes so it isolates +the cost of increasing the number of claims. + +The report records the seed, sample counts, project revision, project version, +Python, Git, platform, input dimensions, and content hashes. +`tracemalloc_peak_bytes` measures Python allocator activity; it is not total +process resident memory. Tracing adds overhead, so compare runs made with the +same settings and on comparable hardware. Reports also identify a dirty tracked +working tree so results made from uncommitted code are not mistaken for their +recorded revision. + +The `binary-exclusion` case records that NUL-containing input is intentionally +excluded from text matching. It has no measured phases. From 3ca224df8c6f128175b0e8e10b4f3fef01a0eb99 Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:38:11 -0400 Subject: [PATCH 07/10] site: List matching benchmark guide The documentation site has an advanced-features section for batch operations and storage behavior, while the matching benchmark guide exists outside its navigation. Users browsing the published site cannot reach the benchmark instructions through the documented feature hierarchy. This commit addresses that discoverability gap by listing the matching benchmark guide alongside the other advanced project references. The remaining commits will add machine-readable report comparison, validate it, and finish the guide with comparison policy. --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 821e1ad72..14ecf3520 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -71,6 +71,7 @@ nav: - Advanced Features: - Batch Operations: batches.md - Storage and Recovery: storage.md + - Matching Benchmarks: benchmarking.md plugins: - search From a9a428d3f61d023d2e15889c5b117393c34ba039 Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:38:31 -0400 Subject: [PATCH 08/10] scripts: Compare matching benchmark reports The benchmark suite emits versioned JSON containing phase medians, raw samples, workload hashes, environment details, and source provenance. Users must inspect separate reports by hand and can accidentally compare different workloads or measurement settings. That makes regression decisions difficult to reproduce. This commit adds report comparison with compatibility checks for suite settings and case dimensions. It calculates median time and allocation changes, preserves environment warnings, and can return failure for changes above a chosen threshold. The final two commits will validate the comparison rules and document the baseline workflow, completing the benchmark repair series. --- scripts/benchmark_matching.py | 198 ++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/scripts/benchmark_matching.py b/scripts/benchmark_matching.py index a31f5007a..d73201a47 100644 --- a/scripts/benchmark_matching.py +++ b/scripts/benchmark_matching.py @@ -968,6 +968,167 @@ def run_suite( } +def compare_reports( + before: dict[str, Any], + after: dict[str, Any], + *, + threshold_percent: float = 20.0, +) -> dict[str, Any]: + """Compare median phase time and memory from two compatible reports.""" + if before.get("schema_version") != SCHEMA_VERSION: + raise ValueError("baseline has an unsupported schema version") + if after.get("schema_version") != SCHEMA_VERSION: + raise ValueError("candidate has an unsupported schema version") + if not math.isfinite(threshold_percent) or threshold_percent < 0: + raise ValueError("regression threshold must be finite and non-negative") + + for field in ("suite", "mode"): + before_value = before.get(field) + after_value = after.get(field) + if before_value is None or after_value is None: + raise ValueError(f"reports are missing required {field!r} metadata") + if before_value != after_value: + raise ValueError( + f"reports use different {field} values: " + f"{before_value!r} != {after_value!r}" + ) + + before_metadata = before.get("metadata") + after_metadata = after.get("metadata") + if not isinstance(before_metadata, dict) or not isinstance(after_metadata, dict): + raise ValueError("reports are missing benchmark metadata") + for field in ( + "seed", + "warmups", + "repeats", + "memory_repeats", + "clock", + "timing_metric", + "memory_metric", + ): + if field not in before_metadata or field not in after_metadata: + raise ValueError(f"reports are missing required {field!r} metadata") + if before_metadata[field] != after_metadata[field]: + raise ValueError( + f"reports use different {field} values: " + f"{before_metadata[field]!r} != {after_metadata[field]!r}" + ) + + before_cases = {case["name"]: case for case in before.get("cases", [])} + after_cases = {case["name"]: case for case in after.get("cases", [])} + common_case_names = before_cases.keys() & after_cases.keys() + if not common_case_names: + raise ValueError("reports do not contain any matching benchmark cases") + + environment_warnings = [] + for field in ( + "python_version", + "python_implementation", + "git_version", + "platform", + "working_tree_dirty", + ): + if field not in before_metadata or field not in after_metadata: + raise ValueError(f"reports are missing required {field!r} metadata") + before_value = before_metadata.get(field) + after_value = after_metadata.get(field) + if before_value != after_value or ( + field == "working_tree_dirty" and (before_value or after_value) + ): + environment_warnings.append( + { + "field": field, + "before": before_value, + "after": after_value, + } + ) + + comparisons = [] + unmatched_phases = [] + for case_name in sorted(common_case_names): + before_dimensions = before_cases[case_name].get("dimensions") + after_dimensions = after_cases[case_name].get("dimensions") + if before_dimensions != after_dimensions: + raise ValueError( + f"case {case_name!r} uses different input dimensions" + ) + before_phases = before_cases[case_name].get("phases", {}) + after_phases = after_cases[case_name].get("phases", {}) + before_only = sorted(before_phases.keys() - after_phases.keys()) + after_only = sorted(after_phases.keys() - before_phases.keys()) + if before_only or after_only: + unmatched_phases.append( + { + "case": case_name, + "baseline_only": before_only, + "candidate_only": after_only, + } + ) + for phase_name in sorted(before_phases.keys() & after_phases.keys()): + for metric in ("seconds", "tracemalloc_peak_bytes"): + if ( + metric not in before_phases[phase_name] + or metric not in after_phases[phase_name] + ): + raise ValueError( + f"phase {case_name}/{phase_name} is missing {metric}" + ) + before_median = before_phases[phase_name][metric]["median"] + after_median = after_phases[phase_name][metric]["median"] + for label, value in ( + ("baseline", before_median), + ("candidate", after_median), + ): + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + ): + raise ValueError( + f"{label} {case_name}/{phase_name} {metric} " + "median must be finite and non-negative" + ) + change_percent = ( + None + if before_median == 0 + else (after_median - before_median) / before_median * 100 + ) + regression = ( + after_median > 0 + if change_percent is None + else change_percent > threshold_percent + ) + comparisons.append( + { + "case": case_name, + "phase": phase_name, + "metric": metric, + "before_median": before_median, + "after_median": after_median, + "change_percent": change_percent, + "regression": regression, + } + ) + + if not comparisons: + raise ValueError("reports do not contain any matching phase measurements") + + return { + "schema_version": SCHEMA_VERSION, + "comparison": "median phase time and memory", + "regression_threshold_percent": threshold_percent, + "regressions": sum(item["regression"] for item in comparisons), + "environment_warnings": environment_warnings, + "unmatched_cases": { + "baseline_only": sorted(before_cases.keys() - after_cases.keys()), + "candidate_only": sorted(after_cases.keys() - before_cases.keys()), + }, + "unmatched_phases": unmatched_phases, + "measurements": comparisons, + } + + def _write_report(report: dict[str, Any], output: Path | None) -> None: rendered = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n" if output is not None: @@ -990,12 +1151,49 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--warmups", type=int) parser.add_argument("--repeats", type=int) parser.add_argument("--output", type=Path) + parser.add_argument( + "--compare", + nargs=2, + type=Path, + metavar=("BASELINE", "CANDIDATE"), + help="compare two existing JSON reports instead of running cases", + ) + parser.add_argument("--regression-threshold-percent", type=float, default=20.0) + parser.add_argument("--fail-on-regression", action="store_true") return parser def main() -> None: parser = _parser() args = parser.parse_args() + if args.compare: + try: + before = json.loads(args.compare[0].read_text(encoding="utf-8")) + after = json.loads(args.compare[1].read_text(encoding="utf-8")) + report = compare_reports( + before, + after, + threshold_percent=args.regression_threshold_percent, + ) + except ( + AttributeError, + IndexError, + KeyError, + OSError, + TypeError, + ValueError, + ) as error: + parser.error(str(error)) + try: + _write_report(report, args.output) + except OSError as error: + parser.error(str(error)) + if args.fail_on_regression and report["regressions"]: + raise SystemExit(1) + return + + if args.fail_on_regression: + parser.error("--fail-on-regression requires --compare") warmups = ( args.warmups From bd2167e7bc69525bb0d9b67600f132edbe78f457 Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:38:58 -0400 Subject: [PATCH 09/10] tests: Cover matching report comparison The benchmark command now compares compatible report medians and records regression flags, environment warnings, and unmatched coverage. The comparison path accepts external JSON, so malformed metrics or incompatible fixture dimensions could otherwise produce misleading results instead of a clear rejection. This commit addresses that risk with checks for time and allocation regressions, interpreter warnings, dimension mismatches, and non-finite threshold values. The final commit will document report comparison and conclude the maintained benchmark workflow. --- tests/test_benchmark_matching.py | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/test_benchmark_matching.py b/tests/test_benchmark_matching.py index 208cf24f1..a23368446 100644 --- a/tests/test_benchmark_matching.py +++ b/tests/test_benchmark_matching.py @@ -241,3 +241,81 @@ def stop_tracing(): assert phase["seconds"]["median"] == pytest.approx(0.25) assert phase["samples"]["seconds"] == [pytest.approx(0.25)] assert phase["samples"]["tracemalloc_peak_bytes"] == [123] + + +def test_report_comparison_flags_time_and_memory_regressions(): + """Comparison should align phase measurements rather than sample ordering.""" + def report(seconds, memory, *, dimensions=None, python_version="3.10.0"): + return { + "schema_version": 1, + "suite": "matching-and-attribution", + "mode": "quick", + "metadata": { + "seed": 42, + "warmups": 1, + "repeats": 3, + "memory_repeats": 3, + "clock": "time.perf_counter", + "timing_metric": "perf_counter without tracemalloc", + "memory_metric": "tracemalloc peak Python allocations", + "python_version": python_version, + "python_implementation": "CPython", + "git_version": "git version 2.0", + "platform": "test-platform", + "working_tree_dirty": False, + }, + "cases": [ + { + "name": "case", + "dimensions": dimensions or {"source_lines": 10}, + "phases": { + "mapping": { + "seconds": {"median": seconds}, + "tracemalloc_peak_bytes": {"median": memory}, + } + }, + } + ], + } + + comparison = benchmark_matching.compare_reports( + report(1.0, 1_000), + report(1.25, 1_250), + threshold_percent=20.0, + ) + + assert comparison["regressions"] == 2 + assert {item["metric"] for item in comparison["measurements"]} == { + "seconds", + "tracemalloc_peak_bytes", + } + assert all( + item["change_percent"] == pytest.approx(25.0) + for item in comparison["measurements"] + ) + assert comparison["environment_warnings"] == [] + + environment_mismatch = benchmark_matching.compare_reports( + report(1.0, 1_000), + report(1.0, 1_000, python_version="3.13.0"), + ) + assert environment_mismatch["environment_warnings"] == [ + { + "field": "python_version", + "before": "3.10.0", + "after": "3.13.0", + } + ] + + with pytest.raises(ValueError, match="different input dimensions"): + benchmark_matching.compare_reports( + report(1.0, 1_000), + report(1.0, 1_000, dimensions={"source_lines": 20}), + ) + + with pytest.raises(ValueError, match="finite and non-negative"): + benchmark_matching.compare_reports( + report(1.0, 1_000), + report(1.0, 1_000), + threshold_percent=float("nan"), + ) From f998a9eddf6a8d6cc42cfede5941777de7717527 Mon Sep 17 00:00:00 2001 From: Ray Strode Date: Sat, 11 Jul 2026 12:39:33 -0400 Subject: [PATCH 10/10] docs: Document benchmark report comparison The benchmark guide explains workload selection, phase boundaries, sampling behavior, and report metadata, while the command can now compare saved results. Users cannot discover the compatibility requirements, regression threshold behavior, environment warnings, or baseline refresh policy for that comparison workflow. This commit addresses that gap with a comparison example and guidance for interpreting medians, rejecting mismatched inputs, opting into failure status, and refreshing an accepted baseline. The benchmark workflow now runs against supported public APIs, covers matching and attribution costs, guards API drift in CI, and supports reproducible trend review. --- docs/benchmarking.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/benchmarking.md b/docs/benchmarking.md index c5b7d3487..ba00140ff 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -60,3 +60,32 @@ recorded revision. The `binary-exclusion` case records that NUL-containing input is intentionally excluded from text matching. It has no measured phases. + +## Comparing revisions + +Create reports from clean checkouts with the same Python, Git, mode, seed, +warm-up count, and repeat count. Then compare their median phase times and +peak Python allocations: + +```console +python scripts/benchmark_matching.py \ + --compare before.json after.json \ + --regression-threshold-percent 20 \ + --output comparison.json +``` + +Comparison is trend-based: matching case, phase, and measurement names are +aligned, and a measurement is marked as a regression when its median exceeds +the selected percentage. Comparison rejects reports with different modes, +seeds, sample settings, measurement methods, or case dimensions. Environment +differences such as Python, Git, or platform versions are retained as warnings +in the comparison report. +Add `--fail-on-regression` when a stable, dedicated runner should return a +failure status. Pull-request CI deliberately runs only a functional smoke test, +without hardware-dependent performance thresholds. + +Refresh a stored baseline only after investigating every flagged phase and +confirming that the change is expected. Record the revision and benchmark +settings with the baseline; the JSON already contains both. Prefer several +full-suite runs on a quiet dedicated machine over treating one workstation run +as authoritative.