diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9af89a2e..231b5c9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,35 @@ on: branches: [main] jobs: + benchmark-smoke: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.13"] + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Smoke test matching benchmark + run: | + python scripts/benchmark_matching.py \ + --mode quick \ + --warmups 0 \ + --repeats 1 \ + --output matching-benchmark-smoke.json + + - name: Upload matching benchmark report + uses: actions/upload-artifact@v4 + with: + name: matching-benchmark-python-${{ matrix.python-version }} + path: matching-benchmark-smoke.json + test: runs-on: ubuntu-latest strategy: diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 00000000..ba00140f --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,91 @@ +# Matching benchmarks + +The matching benchmark exercises the public line-matching and ownership +attribution APIs. It uses deterministic fixtures and emits JSON so results can +be retained and compared across revisions. + +Run the pull-request-sized suite from a source checkout: + +```console +python scripts/benchmark_matching.py --mode quick --output matching.json +``` + +The full suite increases the repeated-line, low-similarity, reversed-order, +Unicode, and many-batch workloads and adds a 50,000-line sparse-edit case: + +```console +python scripts/benchmark_matching.py --mode full --output matching-full.json +``` + +Use `--case NAME` to isolate a regression. The option can be repeated. Use +`--seed`, `--warmups`, and `--repeats` to control reproducibility and sampling. +The defaults are one warm-up and three samples in quick mode, and two warm-ups +and seven samples in full mode. The full suite can take several minutes on a +typical workstation. + +## Reading results + +Each measured phase reports every sample plus minimum, median, 95th percentile, +and maximum elapsed time and peak Python allocation. Elapsed-time samples run +without allocation tracing. Peak-allocation samples run separately, with at +most three samples per phase, so `tracemalloc` does not distort the timing +results. Fixture generation, repository creation, and separately managed phase +prerequisites and cleanup happen outside the timer. End-to-end public APIs still +include any cleanup that is intrinsic to the operation. + +Synthetic text is coalesced into bounded 64 KiB byte chunks before measurement, +so large cases do not retain a Python object for every line alongside the +line-addressable buffers. + +The phases are: + +- `buffer_loading`: create line-addressable buffers and build their line indexes. +- `git_object_resolution`: resolve the expressions needed by batch claims. +- `blob_loading`: stream source blobs into bounded line buffers. +- `mapping`: construct and traverse the structural line mapping. +- `unit_attribution`: enumerate changed units from an already-built comparison. +- `claim_attribution`: run file attribution end to end, including Git I/O, + matching, and ownership claims. + +The many-batch case keeps its file dimensions fixed between modes so it isolates +the cost of increasing the number of claims. + +The report records the seed, sample counts, project revision, project version, +Python, Git, platform, input dimensions, and content hashes. +`tracemalloc_peak_bytes` measures Python allocator activity; it is not total +process resident memory. Tracing adds overhead, so compare runs made with the +same settings and on comparable hardware. Reports also identify a dirty tracked +working tree so results made from uncommitted code are not mistaken for their +recorded revision. + +The `binary-exclusion` case records that NUL-containing input is intentionally +excluded from text matching. It has no measured phases. + +## Comparing revisions + +Create reports from clean checkouts with the same Python, Git, mode, seed, +warm-up count, and repeat count. Then compare their median phase times and +peak Python allocations: + +```console +python scripts/benchmark_matching.py \ + --compare before.json after.json \ + --regression-threshold-percent 20 \ + --output comparison.json +``` + +Comparison is trend-based: matching case, phase, and measurement names are +aligned, and a measurement is marked as a regression when its median exceeds +the selected percentage. Comparison rejects reports with different modes, +seeds, sample settings, measurement methods, or case dimensions. Environment +differences such as Python, Git, or platform versions are retained as warnings +in the comparison report. +Add `--fail-on-regression` when a stable, dedicated runner should return a +failure status. Pull-request CI deliberately runs only a functional smoke test, +without hardware-dependent performance thresholds. + +Refresh a stored baseline only after investigating every flagged phase and +confirming that the change is expected. Record the revision and benchmark +settings with the baseline; the JSON already contains both. Prefer several +full-suite runs on a quiet dedicated machine over treating one workstation run +as authoritative. diff --git a/mkdocs.yml b/mkdocs.yml index 821e1ad7..14ecf352 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -71,6 +71,7 @@ nav: - Advanced Features: - Batch Operations: batches.md - Storage and Recovery: storage.md + - Matching Benchmarks: benchmarking.md plugins: - search diff --git a/scripts/benchmark_matching.py b/scripts/benchmark_matching.py index 55778040..d73201a4 100644 --- a/scripts/benchmark_matching.py +++ b/scripts/benchmark_matching.py @@ -1,198 +1,1224 @@ #!/usr/bin/env python3 -"""Benchmark structural matching storage and resource usage.""" +"""Benchmark public matching and ownership-attribution workflows.""" from __future__ import annotations import argparse -from collections.abc import Iterable +from collections.abc import Callable, Sequence from contextlib import contextmanager +from dataclasses import asdict, dataclass +import gc +import hashlib import json +import math import os from pathlib import Path -import resource +import platform +import random +import statistics +import subprocess import sys +import tempfile import time import tracemalloc -from typing import Any +from typing import Any, TypeVar -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from git_stage_batch.batch import match as match_module -from git_stage_batch.batch.match import LineMapping -from git_stage_batch.batch.match_storage import MatcherWorkspace -from git_stage_batch.editor import EditorBuffer +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT / "src")) +from git_stage_batch import __version__ +from git_stage_batch.batch.attribution import ( + AttributionMetrics, + FileAttribution, + build_file_attribution, +) +from git_stage_batch.batch.attribution_units import ( + AttributionUnit, + FileComparison, + build_file_comparison_from_lines, + enumerate_units_from_file_comparison, +) +from git_stage_batch.batch.line_mapping import LineMapping +from git_stage_batch.batch.match import match_lines +from git_stage_batch.batch.state_refs import get_batch_state_ref_name +from git_stage_batch.core.buffer import LineBuffer +from git_stage_batch.utils.git_object_io import resolve_git_objects +from git_stage_batch.utils.repository_buffers import ( + load_git_object_as_buffer_or_empty, + load_working_tree_file_as_buffer, + stream_git_blob_buffers, +) -class _MeasuredWorkspace(MatcherWorkspace): - """Matcher workspace that reports allocation counters on close.""" - recorder: "_WorkspaceRecorder | None" = None +SCHEMA_VERSION = 1 +DEFAULT_SEED = 20260711 +FIXTURE_CHUNK_SIZE = 64 * 1024 +_StateT = TypeVar("_StateT") - def close(self) -> None: - recorder = self.recorder - if recorder is not None: - recorder.observe_workspace(self) - super().close() +@dataclass(frozen=True) +class MatchingFixture: + """Prepared text inputs whose construction is outside measured phases.""" -class _WorkspaceRecorder: - """Collect matcher workspace metrics across recursive matching.""" + source_chunks: tuple[bytes, ...] + target_chunks: tuple[bytes, ...] + source_line_count: int + target_line_count: int + source_byte_count: int + target_byte_count: int - def __init__(self) -> None: - self.high_water_bytes = 0 - self.total_allocated_bytes = 0 - self.workspace_count = 0 - self.candidate_count = 0 - def observe_workspace(self, workspace: MatcherWorkspace) -> None: - self.workspace_count += 1 - self.high_water_bytes = max( - self.high_water_bytes, - workspace.high_water_bytes, - ) - self.total_allocated_bytes += workspace.total_allocated_bytes +@dataclass(frozen=True) +class CaseDefinition: + """One deterministic benchmark case and its mode-specific dimensions.""" + + name: str + description: str + kind: str + quick_size: int + full_size: int | None + + +@dataclass(frozen=True) +class AttributionFixture: + """Prepared repository and metadata for the attribution benchmark.""" + + file_path: str + metadata: dict[str, dict[str, Any]] + object_names: tuple[str, ...] + source_object_ids: tuple[str, ...] + baseline_line_count: int + working_line_count: int + baseline_sha256: str + working_sha256: str + batch_count: int + + +@dataclass +class _BufferLoadingState: + """Resources created by one measured buffer-loading operation.""" + + fixture: MatchingFixture + source: LineBuffer | None = None + target: LineBuffer | None = None + + +@dataclass +class _MappingState: + """Prepared buffers and the mapping retained until measurement ends.""" + + source: LineBuffer + target: LineBuffer + mapping: LineMapping | None = None + + +@dataclass +class _UnitEnumerationState: + """Prepared comparison and units retained until measurement ends.""" + + source: LineBuffer + target: LineBuffer + comparison: FileComparison + units: dict[str, AttributionUnit] | None = None + + +@dataclass +class _ClaimAttributionState: + """Prepared claim fixture and result retained until measurement ends.""" + + fixture: AttributionFixture + attribution: FileAttribution | None = None + + +CASES = ( + CaseDefinition( + "small-interactive", + "A short file with nearby insertions, replacements, and deletions.", + "text", + 80, + 400, + ), + CaseDefinition( + "repeated-lines", + "Ambiguous repeated content separated by stable anchors.", + "text", + 300, + 5_000, + ), + CaseDefinition( + "unicode", + "UTF-8 text containing multibyte scripts and emoji.", + "text", + 150, + 5_000, + ), + CaseDefinition( + "low-similarity", + "Pathological source and target inputs with almost no shared lines.", + "text", + 250, + 4_000, + ), + CaseDefinition( + "reversed-unique", + "Unique lines in reverse order to stress candidate ordering.", + "text", + 250, + 4_000, + ), + CaseDefinition( + "binary-exclusion", + "NUL-containing input excluded from the text-matching pipeline.", + "binary", + 32, + 1_024, + ), + CaseDefinition( + "many-batches", + "Many ownership claims sharing one source blob and line mapping.", + "attribution", + 50, + 1_000, + ), + CaseDefinition( + "large-file", + "A large mostly stable file with sparse edits.", + "text", + 0, + 50_000, + ), +) +CASE_BY_NAME = {case.name: case for case in CASES} + + +def _case_random(seed: int, case_name: str) -> random.Random: + digest = hashlib.sha256(f"{seed}:{case_name}".encode()).digest() + return random.Random(int.from_bytes(digest[:8], "big")) + + +def _unique_lines(prefix: str, count: int) -> list[bytes]: + return [f"{prefix} {index:08d}\n".encode() for index in range(count)] + + +def _bounded_chunks( + lines: Sequence[bytes], + chunk_size: int = FIXTURE_CHUNK_SIZE, +) -> tuple[bytes, ...]: + payload = b"".join(lines) + return tuple( + payload[offset : offset + chunk_size] + for offset in range(0, len(payload), chunk_size) + ) + + +def _matching_fixture( + source_lines: Sequence[bytes], + target_lines: Sequence[bytes], +) -> MatchingFixture: + return MatchingFixture( + source_chunks=_bounded_chunks(source_lines), + target_chunks=_bounded_chunks(target_lines), + source_line_count=len(source_lines), + target_line_count=len(target_lines), + source_byte_count=sum(map(len, source_lines)), + target_byte_count=sum(map(len, target_lines)), + ) + + +def _chunks_sha256(chunks: Sequence[bytes]) -> str: + digest = hashlib.sha256() + for chunk in chunks: + digest.update(chunk) + return digest.hexdigest() + + +def build_matching_fixture(case_name: str, size: int, seed: int) -> MatchingFixture: + """Build one seeded matching fixture without running measured code.""" + if size < 0: + raise ValueError("fixture size must be non-negative") + randomizer = _case_random(seed, case_name) + + if case_name in {"small-interactive", "large-file"}: + source = _unique_lines("stable", size) + target = list(source) + edit_count = max(1, size // (20 if case_name == "small-interactive" else 1_000)) + positions = sorted(randomizer.sample(range(size), min(edit_count, size))) + for number, position in enumerate(positions): + target[position] = f"replacement {number:08d}\n".encode() + insertion = min(size, size // 3) + target[insertion:insertion] = [b"inserted alpha\n", b"inserted beta\n"] + if size > 4: + del target[(size * 2) // 3] + return _matching_fixture(source, target) + + if case_name == "repeated-lines": + source = [] + for index in range(size): + source.append( + f"anchor {index:08d}\n".encode() if index % 23 == 0 else b"same\n" + ) + target = list(source) + if target: + target.insert(len(target) // 2, b"same\n") + target.pop(min(len(target) - 1, len(target) // 3)) + return _matching_fixture(source, target) + + if case_name == "unicode": + samples = ("naive cafe", "café", "東京", "مرحبا", "🧪", "Straße") + source = [ + f"{samples[index % len(samples)]} {index:08d}\n".encode("utf-8") + for index in range(size) + ] + target = list(source) + if target: + target[len(target) // 2] = "更新 🚀\n".encode("utf-8") + return _matching_fixture(source, target) + + if case_name == "low-similarity": + source = _unique_lines("source", size) + target = _unique_lines("target", size) + if source and target: + target[len(target) // 2] = source[len(source) // 2] + return _matching_fixture(source, target) + + if case_name == "reversed-unique": + source = _unique_lines("reversed", size) + return _matching_fixture(source, list(reversed(source))) + if case_name == "binary-exclusion": + payload = tuple(b"binary\0payload\n" for _ in range(size)) + return _matching_fixture(payload, payload) -def _open_fd_count() -> int | None: - fd_path = "/proc/self/fd" - if not os.path.isdir(fd_path): - return None - return len(os.listdir(fd_path)) + raise ValueError(f"{case_name!r} does not define a matching fixture") -def _rss_bytes() -> int | None: +def _percentile(values: Sequence[float | int], percentile: float) -> float: + ordered = sorted(values) + if not ordered: + raise ValueError("cannot summarize an empty sample") + rank = max(0, math.ceil(percentile * len(ordered)) - 1) + return float(ordered[rank]) + + +def _summary(values: Sequence[float | int]) -> dict[str, float]: + return { + "minimum": float(min(values)), + "median": float(statistics.median(values)), + "p95": _percentile(values, 0.95), + "maximum": float(max(values)), + } + + +def _measure_phase( + prepare: Callable[[], _StateT], + operation: Callable[[_StateT], dict[str, Any]], + cleanup: Callable[[_StateT], None], + *, + warmups: int, + repeats: int, +) -> dict[str, Any]: + """Measure untraced time and traced Python allocations in separate runs.""" + for _ in range(warmups): + state = prepare() + try: + operation(state) + finally: + cleanup(state) + + expected_result: dict[str, Any] | None = None + + def observe_result(result: dict[str, Any]) -> None: + nonlocal expected_result + if expected_result is None: + expected_result = result + elif result != expected_result: + raise RuntimeError("benchmark phase produced inconsistent results") + + timing_samples = [] + for _ in range(repeats): + state = prepare() + try: + gc.collect() + started = time.perf_counter() + result = operation(state) + elapsed = time.perf_counter() - started + finally: + cleanup(state) + observe_result(result) + timing_samples.append(elapsed) + + memory_samples = [] + for _ in range(min(repeats, 3)): + if tracemalloc.is_tracing(): + raise RuntimeError("benchmark cannot reuse an active tracemalloc session") + state = prepare() + started_tracing = False + try: + gc.collect() + tracemalloc.start() + started_tracing = True + result = operation(state) + _, peak_memory = tracemalloc.get_traced_memory() + finally: + if started_tracing: + tracemalloc.stop() + cleanup(state) + observe_result(result) + memory_samples.append(peak_memory) + + return { + "seconds": _summary(timing_samples), + "tracemalloc_peak_bytes": _summary(memory_samples), + "samples": { + "seconds": timing_samples, + "tracemalloc_peak_bytes": memory_samples, + }, + "result": expected_result or {}, + } + + +def _open_matching_buffers(fixture: MatchingFixture) -> _MappingState: + source = LineBuffer.from_chunks(fixture.source_chunks) + target: LineBuffer | None = None try: - return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024 + target = LineBuffer.from_chunks(fixture.target_chunks) + # Force lazy line indexing during preparation, outside algorithm timings. + len(source) + len(target) except Exception: - return None + source.close() + if target is not None: + target.close() + raise + return _MappingState(source=source, target=target) -def _line_span_bytes(buffer: EditorBuffer) -> int: - line_spans = getattr(buffer, "_line_spans", None) - records = getattr(line_spans, "_records", None) - byte_count = getattr(records, "byte_count", 0) - return byte_count if isinstance(byte_count, int) else 0 +def _close_mapping(state: _MappingState) -> None: + try: + if state.mapping is not None: + state.mapping.close() + finally: + try: + state.source.close() + finally: + state.target.close() -def _line_payloads(pattern: str, line_count: int) -> tuple[Iterable[bytes], Iterable[bytes]]: - if pattern == "identical": - source = [f"line {index}\n".encode() for index in range(line_count)] - return source, list(source) +def _measure_buffer_loading(state: _BufferLoadingState) -> dict[str, Any]: + state.source = LineBuffer.from_chunks(state.fixture.source_chunks) + state.target = LineBuffer.from_chunks(state.fixture.target_chunks) + return { + "source_bytes": state.source.byte_count, + "source_lines": len(state.source), + "target_bytes": state.target.byte_count, + "target_lines": len(state.target), + "uses_mapped_storage": state.source.uses_mapped_storage + or state.target.uses_mapped_storage, + } - if pattern == "unique-inserted-prefix": - source = [f"line {index}\n".encode() for index in range(line_count)] - target = [f"extra {index}\n".encode() for index in range(line_count // 10)] - target.extend(source) - return source, target - if pattern == "repeated": - source = [b"same\n" for _ in range(line_count)] - target = list(source) - return source, target +def _close_buffer_loading(state: _BufferLoadingState) -> None: + try: + if state.source is not None: + state.source.close() + finally: + if state.target is not None: + state.target.close() - if pattern == "reversed-unique": - source = [f"line {index}\n".encode() for index in range(line_count)] - return source, list(reversed(source)) - raise ValueError(f"unknown pattern: {pattern}") +def _measure_mapping(state: _MappingState) -> dict[str, Any]: + state.mapping = match_lines(state.source, state.target) + return { + "mapped_lines": sum(1 for _ in state.mapping.mapped_line_pairs()) + } -@contextmanager -def _instrument_matcher(recorder: _WorkspaceRecorder): - original_workspace = match_module.MatcherWorkspace - original_lis = match_module._longest_increasing_subsequence_records +def _prepare_unit_enumeration( + fixture: MatchingFixture, +) -> _UnitEnumerationState: + mapping_state = _open_matching_buffers(fixture) + try: + comparison = build_file_comparison_from_lines( + "benchmark.txt", + baseline_lines=mapping_state.source, + working_tree_lines=mapping_state.target, + ) + except Exception: + _close_mapping(mapping_state) + raise + return _UnitEnumerationState( + source=mapping_state.source, + target=mapping_state.target, + comparison=comparison, + ) + + +def _measure_unit_enumeration( + state: _UnitEnumerationState, +) -> dict[str, Any]: + state.units = {} + enumerate_units_from_file_comparison(state.comparison, state.units) + kinds: dict[str, int] = {} + for unit in state.units.values(): + kinds[unit.kind.value] = kinds.get(unit.kind.value, 0) + 1 + return {"units": len(state.units), "unit_kinds": kinds} + + +def _close_unit_enumeration(state: _UnitEnumerationState) -> None: + state.units = None + try: + state.comparison.close() + finally: + try: + state.source.close() + finally: + state.target.close() + + +def run_matching_case( + case: CaseDefinition, + size: int, + seed: int, + *, + warmups: int, + repeats: int, +) -> dict[str, Any]: + """Run one text case with setup excluded from all measured phases.""" + fixture = build_matching_fixture(case.name, size, seed) + dimensions = { + "requested_lines": size, + "source_lines": fixture.source_line_count, + "target_lines": fixture.target_line_count, + "source_bytes": fixture.source_byte_count, + "target_bytes": fixture.target_byte_count, + "source_chunks": len(fixture.source_chunks), + "target_chunks": len(fixture.target_chunks), + "chunk_size_bytes": FIXTURE_CHUNK_SIZE, + "source_sha256": _chunks_sha256(fixture.source_chunks), + "target_sha256": _chunks_sha256(fixture.target_chunks), + } + if case.kind == "binary": + contains_nul = any(b"\0" in chunk for chunk in fixture.source_chunks) + return { + "name": case.name, + "description": case.description, + "category": "exclusion", + "status": "excluded", + "exclusion_reason": "NUL-containing input is not text-matching work", + "dimensions": {**dimensions, "contains_nul": contains_nul}, + "setup": {"measured": False, "seed": seed}, + "phases": {}, + } + + phases = { + "buffer_loading": _measure_phase( + lambda: _BufferLoadingState(fixture), + _measure_buffer_loading, + _close_buffer_loading, + warmups=warmups, + repeats=repeats, + ), + "mapping": _measure_phase( + lambda: _open_matching_buffers(fixture), + _measure_mapping, + _close_mapping, + warmups=warmups, + repeats=repeats, + ), + "unit_attribution": _measure_phase( + lambda: _prepare_unit_enumeration(fixture), + _measure_unit_enumeration, + _close_unit_enumeration, + warmups=warmups, + repeats=repeats, + ), + } + return { + "name": case.name, + "description": case.description, + "category": "matching", + "status": "measured", + "dimensions": dimensions, + "setup": { + "measured": False, + "seed": seed, + "description": "Payload generation and phase prerequisites are excluded.", + }, + "phases": phases, + } + + +def _git(repo: Path, *arguments: str, input_text: str | None = None) -> str: + return subprocess.run( + ["git", *arguments], + cwd=repo, + input=input_text, + text=True, + check=True, + capture_output=True, + ).stdout.strip() - def measured_lis(pairs, target_start, target_end, workspace): - recorder.candidate_count += len(pairs) - return original_lis(pairs, target_start, target_end, workspace) - _MeasuredWorkspace.recorder = recorder - match_module.MatcherWorkspace = _MeasuredWorkspace - match_module._longest_increasing_subsequence_records = measured_lis +def _build_attribution_fixture(repo: Path, batch_count: int) -> AttributionFixture: + if batch_count <= 0: + raise ValueError("batch count must be positive") + file_path = "benchmark.txt" + line_count = 300 + baseline = _unique_lines("baseline", line_count) + working = list(baseline) + claim_line = max(1, line_count // 2) + working[claim_line - 1] = b"owned replacement\n" + working.append(b"owned addition\n") + baseline_payload = b"".join(baseline) + working_payload = b"".join(working) + + _git(repo, "init", "--quiet") + _git(repo, "config", "user.name", "Benchmark") + _git(repo, "config", "user.email", "benchmark@example.invalid") + path = repo / file_path + path.write_bytes(baseline_payload) + _git(repo, "add", "--", file_path) + _git(repo, "commit", "--quiet", "-m", "benchmark baseline") + + path.write_bytes(working_payload) + _git(repo, "add", "--", file_path) + source_tree = _git(repo, "write-tree") + source_commit = _git( + repo, + "commit-tree", + source_tree, + "-p", + "HEAD", + "-m", + "benchmark source", + ) + _git(repo, "reset", "--mixed", "HEAD") + + batch_names = tuple(f"benchmark-{index:04d}" for index in range(batch_count)) + metadata = { + name: { + "files": { + file_path: { + "batch_source_commit": source_commit, + "source_path": f"sources/{file_path}", + "presence_claims": [ + {"source_lines": [str(claim_line), str(line_count + 1)]} + ], + "deletions": [], + } + } + } + for name in batch_names + } + fallback = f"{source_commit}:{file_path}" + source_object_id = _git(repo, "rev-parse", fallback) + sources_tree = _git( + repo, + "mktree", + input_text=f"100644 blob {source_object_id}\t{file_path}\n", + ) + state_tree = _git( + repo, + "mktree", + input_text=f"040000 tree {sources_tree}\tsources\n", + ) + state_commits = { + name: _git( + repo, + "commit-tree", + state_tree, + "-p", + "HEAD", + "-m", + f"benchmark state {name}", + ) + for name in batch_names + } + _git( + repo, + "update-ref", + "--stdin", + input_text="".join( + f"update {get_batch_state_ref_name(name)} {state_commits[name]}\n" + for name in batch_names + ), + ) + object_names = tuple( + [ + *( + f"{get_batch_state_ref_name(name)}:sources/{file_path}" + for name in batch_names + ), + fallback, + ] + ) + return AttributionFixture( + file_path=file_path, + metadata=metadata, + object_names=object_names, + source_object_ids=(source_object_id,), + baseline_line_count=len(baseline), + working_line_count=len(working), + baseline_sha256=hashlib.sha256(baseline_payload).hexdigest(), + working_sha256=hashlib.sha256(working_payload).hexdigest(), + batch_count=batch_count, + ) + + +@contextmanager +def _working_directory(path: Path): + previous = Path.cwd() + os.chdir(path) try: yield finally: - match_module._longest_increasing_subsequence_records = original_lis - match_module.MatcherWorkspace = original_workspace - _MeasuredWorkspace.recorder = None + os.chdir(previous) + + +def _measure_object_resolution(fixture: AttributionFixture) -> dict[str, Any]: + resolved = resolve_git_objects(fixture.object_names) + return { + "requests": len(fixture.object_names), + "resolved": len(resolved), + "unique_object_ids": len({info.object_id for info in resolved.values()}), + } + + +def _measure_blob_loading(fixture: AttributionFixture) -> dict[str, Any]: + object_count = 0 + byte_count = 0 + line_count = 0 + for blob in stream_git_blob_buffers(fixture.source_object_ids): + object_count += 1 + byte_count += blob.size + line_count += len(blob.buffer) + return {"objects": object_count, "bytes": byte_count, "lines": line_count} + + +def _open_repository_buffers( + fixture: AttributionFixture, +) -> _MappingState: + source = load_git_object_as_buffer_or_empty( + next(iter(fixture.metadata.values()))["files"][fixture.file_path][ + "batch_source_commit" + ] + + f":{fixture.file_path}" + ) + working: LineBuffer | None = None + try: + working = load_working_tree_file_as_buffer(fixture.file_path) + len(source) + len(working) + except Exception: + source.close() + if working is not None: + working.close() + raise + return _MappingState(source=source, target=working) + + +def _measure_claim_attribution(state: _ClaimAttributionState) -> dict[str, Any]: + metrics = AttributionMetrics() + state.attribution = build_file_attribution( + state.fixture.file_path, + batch_metadata_by_name=state.fixture.metadata, + metrics=metrics, + ) + owned_units = sum( + bool(unit.owning_batches) for unit in state.attribution.units + ) + owner_links = sum( + len(unit.owning_batches) for unit in state.attribution.units + ) + return { + "units": len(state.attribution.units), + "owned_units": owned_units, + "owner_links": owner_links, + "metrics": asdict(metrics), + } + + +def _close_claim_attribution(state: _ClaimAttributionState) -> None: + state.attribution = None + + +def run_attribution_case( + case: CaseDefinition, + batch_count: int, + seed: int, + *, + warmups: int, + repeats: int, +) -> dict[str, Any]: + """Run the public attribution workflow in an isolated prepared repository.""" + del seed # This fixture is fully enumerated; keep the shared schema explicit. + with tempfile.TemporaryDirectory(prefix="git-stage-batch-matching-") as directory: + repo = Path(directory) + fixture = _build_attribution_fixture(repo, batch_count) + with _working_directory(repo): + phases = { + "git_object_resolution": _measure_phase( + lambda: fixture, + _measure_object_resolution, + lambda _state: None, + warmups=warmups, + repeats=repeats, + ), + "blob_loading": _measure_phase( + lambda: fixture, + _measure_blob_loading, + lambda _state: None, + warmups=warmups, + repeats=repeats, + ), + "mapping": _measure_phase( + lambda: _open_repository_buffers(fixture), + _measure_mapping, + _close_mapping, + warmups=warmups, + repeats=repeats, + ), + "unit_attribution": _measure_phase( + lambda: _prepare_repository_unit_enumeration(fixture), + _measure_unit_enumeration, + _close_unit_enumeration, + warmups=warmups, + repeats=repeats, + ), + "claim_attribution": _measure_phase( + lambda: _ClaimAttributionState(fixture), + _measure_claim_attribution, + _close_claim_attribution, + warmups=warmups, + repeats=repeats, + ), + } + + return { + "name": case.name, + "description": case.description, + "category": "attribution", + "status": "measured", + "dimensions": { + "batches": fixture.batch_count, + "baseline_lines": fixture.baseline_line_count, + "source_lines": fixture.working_line_count, + "target_lines": fixture.working_line_count, + "baseline_sha256": fixture.baseline_sha256, + "source_sha256": fixture.working_sha256, + "target_sha256": fixture.working_sha256, + "object_resolution_requests": len(fixture.object_names), + }, + "setup": { + "measured": False, + "description": ( + "Repository, commits, metadata, and prerequisites are excluded." + ), + }, + "phases": phases, + } -def _mapping_source_to_target_bytes(mapping: LineMapping) -> int: - source_bytes = getattr(mapping.source_to_target, "byte_count", 0) - target_bytes = getattr(mapping.target_to_source, "byte_count", 0) - return ( - (source_bytes if isinstance(source_bytes, int) else 0) - + (target_bytes if isinstance(target_bytes, int) else 0) +def _prepare_repository_unit_enumeration( + fixture: AttributionFixture, +) -> _UnitEnumerationState: + source = load_git_object_as_buffer_or_empty(f"HEAD:{fixture.file_path}") + try: + working = load_working_tree_file_as_buffer(fixture.file_path) + except Exception: + source.close() + raise + try: + len(source) + len(working) + comparison = build_file_comparison_from_lines( + fixture.file_path, + baseline_lines=source, + working_tree_lines=working, + ) + except Exception: + source.close() + working.close() + raise + return _UnitEnumerationState( + source=source, + target=working, + comparison=comparison, ) -def run_benchmark(pattern: str, line_count: int) -> dict[str, Any]: - source_payloads, target_payloads = _line_payloads(pattern, line_count) - recorder = _WorkspaceRecorder() - fd_before = _open_fd_count() - rss_before = _rss_bytes() - tracemalloc.start() - start_time = time.perf_counter() +def _command_output(arguments: Sequence[str]) -> str: + try: + return subprocess.run( + arguments, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +def _metadata(seed: int, warmups: int, repeats: int) -> dict[str, Any]: + project_version = __version__ + if project_version == "unknown": + try: + project_version = (PROJECT_ROOT / "VERSION").read_text( + encoding="utf-8" + ).strip() + except OSError: + pass + working_tree_status = _command_output( + [ + "git", + "-C", + str(PROJECT_ROOT), + "status", + "--porcelain", + "--untracked-files=no", + ] + ) + return { + "project_version": project_version, + "python_version": platform.python_version(), + "python_implementation": platform.python_implementation(), + "git_version": _command_output(["git", "--version"]), + "platform": platform.platform(), + "revision": _command_output( + ["git", "-C", str(PROJECT_ROOT), "rev-parse", "HEAD"] + ), + "working_tree_dirty": ( + None if working_tree_status == "unknown" else bool(working_tree_status) + ), + "seed": seed, + "warmups": warmups, + "repeats": repeats, + "memory_repeats": min(repeats, 3), + "clock": "time.perf_counter", + "timing_metric": "perf_counter without tracemalloc", + "memory_metric": "tracemalloc peak Python allocations", + } + + +def selected_cases( + mode: str, + names: Sequence[str] | None = None, +) -> list[CaseDefinition]: + """Select named cases, or every case available in the requested mode.""" + if names: + cases = [CASE_BY_NAME[name] for name in dict.fromkeys(names)] + else: + cases = list(CASES) + if mode == "quick": + unavailable = [case.name for case in cases if case.quick_size <= 0] + if names and unavailable: + raise ValueError( + f"{', '.join(unavailable)} requires --mode full" + ) + return [case for case in cases if case.quick_size > 0] + return [case for case in cases if case.full_size is not None] + + +def run_suite( + mode: str = "quick", + *, + case_names: Sequence[str] | None = None, + seed: int = DEFAULT_SEED, + warmups: int = 1, + repeats: int = 3, +) -> dict[str, Any]: + """Run selected deterministic cases and return the stable JSON schema.""" + if mode not in {"quick", "full"}: + raise ValueError("mode must be 'quick' or 'full'") + if warmups < 0: + raise ValueError("warmups must be non-negative") + if repeats <= 0: + raise ValueError("repeats must be positive") + + results = [] + for case in selected_cases(mode, case_names): + size = case.quick_size if mode == "quick" else case.full_size + if size is None or size <= 0: + continue + if case.kind == "attribution": + result = run_attribution_case( + case, + size, + seed, + warmups=warmups, + repeats=repeats, + ) + else: + result = run_matching_case( + case, + size, + seed, + warmups=warmups, + repeats=repeats, + ) + results.append(result) + + return { + "schema_version": SCHEMA_VERSION, + "suite": "matching-and-attribution", + "mode": mode, + "metadata": _metadata(seed, warmups, repeats), + "cases": results, + } - with ( - EditorBuffer.from_chunks(source_payloads) as source, - EditorBuffer.from_chunks(target_payloads) as target, + +def compare_reports( + before: dict[str, Any], + after: dict[str, Any], + *, + threshold_percent: float = 20.0, +) -> dict[str, Any]: + """Compare median phase time and memory from two compatible reports.""" + if before.get("schema_version") != SCHEMA_VERSION: + raise ValueError("baseline has an unsupported schema version") + if after.get("schema_version") != SCHEMA_VERSION: + raise ValueError("candidate has an unsupported schema version") + if not math.isfinite(threshold_percent) or threshold_percent < 0: + raise ValueError("regression threshold must be finite and non-negative") + + for field in ("suite", "mode"): + before_value = before.get(field) + after_value = after.get(field) + if before_value is None or after_value is None: + raise ValueError(f"reports are missing required {field!r} metadata") + if before_value != after_value: + raise ValueError( + f"reports use different {field} values: " + f"{before_value!r} != {after_value!r}" + ) + + before_metadata = before.get("metadata") + after_metadata = after.get("metadata") + if not isinstance(before_metadata, dict) or not isinstance(after_metadata, dict): + raise ValueError("reports are missing benchmark metadata") + for field in ( + "seed", + "warmups", + "repeats", + "memory_repeats", + "clock", + "timing_metric", + "memory_metric", + ): + if field not in before_metadata or field not in after_metadata: + raise ValueError(f"reports are missing required {field!r} metadata") + if before_metadata[field] != after_metadata[field]: + raise ValueError( + f"reports use different {field} values: " + f"{before_metadata[field]!r} != {after_metadata[field]!r}" + ) + + before_cases = {case["name"]: case for case in before.get("cases", [])} + after_cases = {case["name"]: case for case in after.get("cases", [])} + common_case_names = before_cases.keys() & after_cases.keys() + if not common_case_names: + raise ValueError("reports do not contain any matching benchmark cases") + + environment_warnings = [] + for field in ( + "python_version", + "python_implementation", + "git_version", + "platform", + "working_tree_dirty", ): - with _instrument_matcher(recorder): - with match_module.match_lines(source, target) as mapping: - mapped_line_count = sum(1 for _ in mapping.mapped_line_pairs()) - mapping_bytes = _mapping_source_to_target_bytes(mapping) + if field not in before_metadata or field not in after_metadata: + raise ValueError(f"reports are missing required {field!r} metadata") + before_value = before_metadata.get(field) + after_value = after_metadata.get(field) + if before_value != after_value or ( + field == "working_tree_dirty" and (before_value or after_value) + ): + environment_warnings.append( + { + "field": field, + "before": before_value, + "after": after_value, + } + ) - source_line_count = len(source) - target_line_count = len(target) - line_span_bytes = _line_span_bytes(source) + _line_span_bytes(target) + comparisons = [] + unmatched_phases = [] + for case_name in sorted(common_case_names): + before_dimensions = before_cases[case_name].get("dimensions") + after_dimensions = after_cases[case_name].get("dimensions") + if before_dimensions != after_dimensions: + raise ValueError( + f"case {case_name!r} uses different input dimensions" + ) + before_phases = before_cases[case_name].get("phases", {}) + after_phases = after_cases[case_name].get("phases", {}) + before_only = sorted(before_phases.keys() - after_phases.keys()) + after_only = sorted(after_phases.keys() - before_phases.keys()) + if before_only or after_only: + unmatched_phases.append( + { + "case": case_name, + "baseline_only": before_only, + "candidate_only": after_only, + } + ) + for phase_name in sorted(before_phases.keys() & after_phases.keys()): + for metric in ("seconds", "tracemalloc_peak_bytes"): + if ( + metric not in before_phases[phase_name] + or metric not in after_phases[phase_name] + ): + raise ValueError( + f"phase {case_name}/{phase_name} is missing {metric}" + ) + before_median = before_phases[phase_name][metric]["median"] + after_median = after_phases[phase_name][metric]["median"] + for label, value in ( + ("baseline", before_median), + ("candidate", after_median), + ): + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + ): + raise ValueError( + f"{label} {case_name}/{phase_name} {metric} " + "median must be finite and non-negative" + ) + change_percent = ( + None + if before_median == 0 + else (after_median - before_median) / before_median * 100 + ) + regression = ( + after_median > 0 + if change_percent is None + else change_percent > threshold_percent + ) + comparisons.append( + { + "case": case_name, + "phase": phase_name, + "metric": metric, + "before_median": before_median, + "after_median": after_median, + "change_percent": change_percent, + "regression": regression, + } + ) - elapsed = time.perf_counter() - start_time - _, peak_heap = tracemalloc.get_traced_memory() - tracemalloc.stop() - rss_after = _rss_bytes() - fd_after = _open_fd_count() + if not comparisons: + raise ValueError("reports do not contain any matching phase measurements") return { - "pattern": pattern, - "requested_source_lines": line_count, - "source_lines": source_line_count, - "target_lines": target_line_count, - "mapped_line_count": mapped_line_count, - "candidate_count": recorder.candidate_count, - "tracemalloc_peak_bytes": peak_heap, - "rss_before_bytes": rss_before, - "rss_after_bytes": rss_after, - "elapsed_seconds": elapsed, - "fd_before": fd_before, - "fd_after": fd_after, - "line_span_bytes": line_span_bytes, - "line_mapping_bytes": mapping_bytes, - "matcher_workspace_high_water_bytes": recorder.high_water_bytes, - "matcher_workspace_total_allocated_bytes": recorder.total_allocated_bytes, - "matcher_workspace_count": recorder.workspace_count, + "schema_version": SCHEMA_VERSION, + "comparison": "median phase time and memory", + "regression_threshold_percent": threshold_percent, + "regressions": sum(item["regression"] for item in comparisons), + "environment_warnings": environment_warnings, + "unmatched_cases": { + "baseline_only": sorted(before_cases.keys() - after_cases.keys()), + "candidate_only": sorted(after_cases.keys() - before_cases.keys()), + }, + "unmatched_phases": unmatched_phases, + "measurements": comparisons, } -def main() -> None: - parser = argparse.ArgumentParser() +def _write_report(report: dict[str, Any], output: Path | None) -> None: + rendered = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n" + if output is not None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + sys.stdout.write(rendered) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=("quick", "full"), default="quick") + parser.add_argument( + "--case", + action="append", + choices=tuple(CASE_BY_NAME), + dest="case_names", + help="run one case (repeat to select more than one)", + ) + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--warmups", type=int) + parser.add_argument("--repeats", type=int) + parser.add_argument("--output", type=Path) parser.add_argument( - "--pattern", - choices=[ - "identical", - "unique-inserted-prefix", - "repeated", - "reversed-unique", - ], - default="unique-inserted-prefix", - ) - parser.add_argument("--lines", type=int, default=10000) + "--compare", + nargs=2, + type=Path, + metavar=("BASELINE", "CANDIDATE"), + help="compare two existing JSON reports instead of running cases", + ) + parser.add_argument("--regression-threshold-percent", type=float, default=20.0) + parser.add_argument("--fail-on-regression", action="store_true") + return parser + + +def main() -> None: + parser = _parser() args = parser.parse_args() + if args.compare: + try: + before = json.loads(args.compare[0].read_text(encoding="utf-8")) + after = json.loads(args.compare[1].read_text(encoding="utf-8")) + report = compare_reports( + before, + after, + threshold_percent=args.regression_threshold_percent, + ) + except ( + AttributeError, + IndexError, + KeyError, + OSError, + TypeError, + ValueError, + ) as error: + parser.error(str(error)) + try: + _write_report(report, args.output) + except OSError as error: + parser.error(str(error)) + if args.fail_on_regression and report["regressions"]: + raise SystemExit(1) + return - if args.lines < 0: - raise SystemExit("--lines must be non-negative") + if args.fail_on_regression: + parser.error("--fail-on-regression requires --compare") - print(json.dumps(run_benchmark(args.pattern, args.lines), indent=2)) + warmups = ( + args.warmups + if args.warmups is not None + else (1 if args.mode == "quick" else 2) + ) + repeats = ( + args.repeats + if args.repeats is not None + else (3 if args.mode == "quick" else 7) + ) + try: + report = run_suite( + args.mode, + case_names=args.case_names, + seed=args.seed, + warmups=warmups, + repeats=repeats, + ) + except ValueError as error: + parser.error(str(error)) + try: + _write_report(report, args.output) + except OSError as error: + parser.error(str(error)) if __name__ == "__main__": diff --git a/tests/test_benchmark_matching.py b/tests/test_benchmark_matching.py new file mode 100644 index 00000000..a2336844 --- /dev/null +++ b/tests/test_benchmark_matching.py @@ -0,0 +1,321 @@ +"""Tests for the maintained matching and attribution benchmark.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys + +import pytest + + +SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "benchmark_matching.py" +SPEC = importlib.util.spec_from_file_location("benchmark_matching", SCRIPT_PATH) +assert SPEC is not None +assert SPEC.loader is not None +benchmark_matching = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = benchmark_matching +SPEC.loader.exec_module(benchmark_matching) + + +def test_seeded_fixtures_are_deterministic_and_cover_edge_cases(): + """Generated fixtures should be repeatable and retain adversarial shapes.""" + repeated = benchmark_matching.build_matching_fixture( + "repeated-lines", 100, 42 + ) + repeated_again = benchmark_matching.build_matching_fixture( + "repeated-lines", 100, 42 + ) + low_similarity = benchmark_matching.build_matching_fixture( + "low-similarity", 100, 42 + ) + binary = benchmark_matching.build_matching_fixture("binary-exclusion", 2, 42) + large = benchmark_matching.build_matching_fixture("large-file", 5_000, 42) + seeded_edit = benchmark_matching.build_matching_fixture( + "small-interactive", 80, 42 + ) + differently_seeded_edit = benchmark_matching.build_matching_fixture( + "small-interactive", 80, 43 + ) + repeated_source_lines = b"".join(repeated.source_chunks).splitlines( + keepends=True + ) + low_source_lines = b"".join(low_similarity.source_chunks).splitlines( + keepends=True + ) + low_target_lines = b"".join(low_similarity.target_chunks).splitlines( + keepends=True + ) + + assert repeated == repeated_again + assert repeated_source_lines.count(b"same\n") > 90 + assert set(low_source_lines) & set(low_target_lines) == {low_source_lines[50]} + assert b"\0" in b"".join(binary.source_chunks) + assert large.source_line_count == 5_000 + assert len(large.source_chunks) > 1 + assert max(map(len, large.source_chunks)) <= ( + benchmark_matching.FIXTURE_CHUNK_SIZE + ) + assert seeded_edit.target_chunks != differently_seeded_edit.target_chunks + + +def test_smallest_case_smokes_public_apis_and_stable_schema(): + """The benchmark should remain importable and runnable after API changes.""" + report = benchmark_matching.run_suite( + "quick", + case_names=["small-interactive"], + seed=42, + warmups=0, + repeats=1, + ) + + assert report["schema_version"] == 1 + assert report["suite"] == "matching-and-attribution" + assert report["metadata"]["seed"] == 42 + assert isinstance(report["metadata"]["working_tree_dirty"], bool) + case = report["cases"][0] + assert case["name"] == "small-interactive" + assert case["setup"]["measured"] is False + assert set(case["phases"]) == { + "buffer_loading", + "mapping", + "unit_attribution", + } + for phase in case["phases"].values(): + assert set(phase) == { + "seconds", + "tracemalloc_peak_bytes", + "samples", + "result", + } + assert len(phase["samples"]["seconds"]) == 1 + assert len(phase["samples"]["tracemalloc_peak_bytes"]) == 1 + + +@pytest.mark.parametrize( + ("case_name", "expected_mapped_lines"), + ( + ("repeated-lines", 99), + ("low-similarity", 1), + ("reversed-unique", 1), + ), +) +def test_adversarial_matching_cases_run(case_name, expected_mapped_lines): + """Ambiguous and low-similarity inputs should traverse the real matcher.""" + case = benchmark_matching.CASE_BY_NAME[case_name] + result = benchmark_matching.run_matching_case( + case, + 100, + 42, + warmups=0, + repeats=1, + ) + + assert result["phases"]["mapping"]["result"]["mapped_lines"] == ( + expected_mapped_lines + ) + + +def test_attribution_case_exercises_many_claims_and_deduplicates_work(): + """The end-to-end case should cover Git IO, matching, and claim ownership.""" + report = benchmark_matching.run_suite( + "quick", + case_names=["many-batches"], + warmups=0, + repeats=1, + ) + + case = report["cases"][0] + phases = case["phases"] + claim_result = phases["claim_attribution"]["result"] + metrics = claim_result["metrics"] + assert case["setup"]["measured"] is False + assert set(phases) == { + "git_object_resolution", + "blob_loading", + "mapping", + "unit_attribution", + "claim_attribution", + } + assert metrics["claimed_batches"] == 50 + assert metrics["object_requests"] == 1 + assert metrics["mapping_computations"] == 1 + assert claim_result["owner_links"] == 100 + assert phases["unit_attribution"]["result"]["units"] == 2 + assert phases["git_object_resolution"]["result"] == { + "requests": 51, + "resolved": 51, + "unique_object_ids": 1, + } + + +def test_binary_case_is_explicitly_excluded_from_text_matching(): + """Binary fixtures should document exclusion and run no matching phases.""" + report = benchmark_matching.run_suite( + "quick", + case_names=["binary-exclusion"], + warmups=0, + repeats=1, + ) + + case = report["cases"][0] + assert case["status"] == "excluded" + assert case["dimensions"]["contains_nul"] is True + assert case["phases"] == {} + + +def test_phase_preparation_and_cleanup_are_outside_the_timer(monkeypatch): + """Setup stays untimed and allocation tracing uses a separate run.""" + events = [] + ticks = iter((10.0, 10.25)) + tracing = False + + def prepare(): + events.append("prepare") + return "state" + + def operation(state): + assert state == "state" + events.append("operation") + return {"answer": 42} + + def cleanup(state): + assert state == "state" + events.append("cleanup") + + def perf_counter(): + events.append("clock") + return next(ticks) + + def start_tracing(): + nonlocal tracing + assert not tracing + tracing = True + events.append("trace-start") + + def read_tracing(): + assert tracing + events.append("trace-read") + return 0, 123 + + def stop_tracing(): + nonlocal tracing + assert tracing + tracing = False + events.append("trace-stop") + + monkeypatch.setattr(benchmark_matching.time, "perf_counter", perf_counter) + monkeypatch.setattr(benchmark_matching.tracemalloc, "start", start_tracing) + monkeypatch.setattr( + benchmark_matching.tracemalloc, + "get_traced_memory", + read_tracing, + ) + monkeypatch.setattr(benchmark_matching.tracemalloc, "stop", stop_tracing) + monkeypatch.setattr( + benchmark_matching.tracemalloc, + "is_tracing", + lambda: tracing, + ) + phase = benchmark_matching._measure_phase( + prepare, + operation, + cleanup, + warmups=0, + repeats=1, + ) + + assert events == [ + "prepare", + "clock", + "operation", + "clock", + "cleanup", + "prepare", + "trace-start", + "operation", + "trace-read", + "trace-stop", + "cleanup", + ] + assert phase["seconds"]["median"] == pytest.approx(0.25) + assert phase["samples"]["seconds"] == [pytest.approx(0.25)] + assert phase["samples"]["tracemalloc_peak_bytes"] == [123] + + +def test_report_comparison_flags_time_and_memory_regressions(): + """Comparison should align phase measurements rather than sample ordering.""" + def report(seconds, memory, *, dimensions=None, python_version="3.10.0"): + return { + "schema_version": 1, + "suite": "matching-and-attribution", + "mode": "quick", + "metadata": { + "seed": 42, + "warmups": 1, + "repeats": 3, + "memory_repeats": 3, + "clock": "time.perf_counter", + "timing_metric": "perf_counter without tracemalloc", + "memory_metric": "tracemalloc peak Python allocations", + "python_version": python_version, + "python_implementation": "CPython", + "git_version": "git version 2.0", + "platform": "test-platform", + "working_tree_dirty": False, + }, + "cases": [ + { + "name": "case", + "dimensions": dimensions or {"source_lines": 10}, + "phases": { + "mapping": { + "seconds": {"median": seconds}, + "tracemalloc_peak_bytes": {"median": memory}, + } + }, + } + ], + } + + comparison = benchmark_matching.compare_reports( + report(1.0, 1_000), + report(1.25, 1_250), + threshold_percent=20.0, + ) + + assert comparison["regressions"] == 2 + assert {item["metric"] for item in comparison["measurements"]} == { + "seconds", + "tracemalloc_peak_bytes", + } + assert all( + item["change_percent"] == pytest.approx(25.0) + for item in comparison["measurements"] + ) + assert comparison["environment_warnings"] == [] + + environment_mismatch = benchmark_matching.compare_reports( + report(1.0, 1_000), + report(1.0, 1_000, python_version="3.13.0"), + ) + assert environment_mismatch["environment_warnings"] == [ + { + "field": "python_version", + "before": "3.10.0", + "after": "3.13.0", + } + ] + + with pytest.raises(ValueError, match="different input dimensions"): + benchmark_matching.compare_reports( + report(1.0, 1_000), + report(1.0, 1_000, dimensions={"source_lines": 20}), + ) + + with pytest.raises(ValueError, match="finite and non-negative"): + benchmark_matching.compare_reports( + report(1.0, 1_000), + report(1.0, 1_000), + threshold_percent=float("nan"), + )