From c3c53d2d4bc8a5c22add5c802d8f58f993315803 Mon Sep 17 00:00:00 2001 From: Mac Anderson Date: Thu, 6 Aug 2026 22:09:13 -0700 Subject: [PATCH 1/7] wip(bench): SUT freshness guard + build_sut race diagnosis --- bench/evidence/run/build_sut.sh | 54 +- bench/evidence/run/env.sh | 30 + .../harbor_adapter/stella_harbor/freshness.py | 575 ++++++++++++++++++ 3 files changed, 656 insertions(+), 3 deletions(-) create mode 100644 bench/harbor_adapter/stella_harbor/freshness.py diff --git a/bench/evidence/run/build_sut.sh b/bench/evidence/run/build_sut.sh index b3e95661b..ac8b0fe0a 100755 --- a/bench/evidence/run/build_sut.sh +++ b/bench/evidence/run/build_sut.sh @@ -1,17 +1,59 @@ #!/bin/bash # Build the frozen SUT binary and prove its provenance. # +# build_sut.sh [] +# # The SUT is `origin/main` itself. The working tree may carry bench/ or docs # changes, which compile into nothing, so the check that matters is not "tree is # clean" but "every input to the Rust build is byte-identical to origin/main". +# +# Pass a commit to skip the fetch and build exactly that revision. A wrapper +# that fetches and checks out should pass what it checked out, because otherwise +# this script re-fetches and can resolve a *different* origin/main than the one +# the caller prepared against — a race whose symptom is a drift list of files +# nobody touched. When no commit is given the race is detected rather than +# guessed at: origin/main is recorded before and after the fetch, and a refusal +# says which of the two it is. set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/env.sh" cd "$TB_REPO" -git fetch origin main -q -SUT="$(git rev-parse origin/main)" -drift="$(git diff --name-only "$SUT" -- '*.rs' '*/Cargo.toml' Cargo.toml Cargo.lock rust-toolchain.toml)" +RUST_INPUTS=('*.rs' '*/Cargo.toml' Cargo.toml Cargo.lock rust-toolchain.toml) + +if [ "$#" -gt 0 ]; then + SUT="$(git rev-parse --verify "$1^{commit}")" + BEFORE="$SUT" +else + # `|| true`: a clone with no origin/main yet must reach the fetch, not die on + # the probe that exists only to describe where the fetch started. + BEFORE="$(git rev-parse --verify --quiet origin/main || true)" + git fetch origin main -q + SUT="$(git rev-parse origin/main)" +fi + +drift="$(git diff --name-only "$SUT" -- "${RUST_INPUTS[@]}")" if [ -n "$drift" ]; then + # Separate the two causes before blaming the operator. If the tree is + # byte-identical to where origin/main stood when this script started, the + # drift is entirely the commits the fetch just brought in — the tree is + # clean, and hunting it for local edits finds nothing because there are none. + raced="" + if [ -n "$BEFORE" ] && [ "$BEFORE" != "$SUT" ]; then + if [ -z "$(git diff --name-only "$BEFORE" -- "${RUST_INPUTS[@]}")" ]; then + raced="yes" + fi + fi + if [ -n "$raced" ]; then + moved="$(git rev-list --count "$BEFORE..$SUT")" + echo "FATAL: origin/main moved while this script ran — this is NOT local contamination." + echo " Your tree is byte-identical to $BEFORE, where origin/main stood" + echo " when the build started. The fetch advanced it $moved commit(s) to $SUT," + echo " and the files below are that difference, not edits of yours:" + echo "$drift" + echo " Update the tree and re-run, or build the revision you prepared against:" + echo " bench/evidence/run/build_sut.sh $BEFORE" + exit 1 + fi echo "FATAL: build inputs differ from $SUT:"; echo "$drift"; exit 1 fi echo "$SUT" > "$TB_ROOT/sut_commit.txt" @@ -31,6 +73,12 @@ test -x "$STELLA_BINARY" # this script and building through anything else are then distinguishable by # inspecting the output, which is the only evidence a later run actually has. assert_portable_binary +# And that the artifact carries the commit we asked for. `--max-behind 0` +# against $SUT itself is an equality assertion on the binary's own compile-time +# stamp, so a build that silently stamped something else — a stale target dir, +# an env var the caller already had set — is caught here rather than becoming +# the next thing that measures the wrong code. +assert_fresh_sut "$STELLA_BINARY" --reference "$SUT" --max-behind 0 file "$STELLA_BINARY" shasum -a 256 "$STELLA_BINARY" | awk '{print $1}' > "$TB_ROOT/binary_sha256.txt" echo "binary_sha256=$(cat "$TB_ROOT/binary_sha256.txt")" diff --git a/bench/evidence/run/env.sh b/bench/evidence/run/env.sh index f5e446373..b105efe6d 100755 --- a/bench/evidence/run/env.sh +++ b/bench/evidence/run/env.sh @@ -83,10 +83,40 @@ assert_portable_binary() { } } +# Assert the SUT binary is close enough to `origin/main` to be reported as it. +# +# `assert_portable_binary` answers "can this run"; the SHA-256 comparison +# answers "did it arrive intact"; `STELLA_SOURCE_COMMIT` answers "which commit +# is it". None of them answers "is that commit anywhere near the one a reader +# will assume was measured" — and on 2026-08-07 the path this file exports as +# STELLA_BINARY was 291 commits behind main, accompanied by its own +# sut_commit.txt, so it looked pinned and passed everything. Pinning and +# freshness are different properties; this checks the second. +# +# The limit deliberately lives in the checker and is not restated here: one +# copy cannot drift from the other. Pass --max-behind to measure older code on +# purpose, which puts the distance in the launch command rather than in a +# default nobody reads. +assert_fresh_sut() { + local binary="${1:-$STELLA_BINARY}" + [ "$#" -gt 0 ] && shift + local checker="$TB_REPO/bench/harbor_adapter/stella_harbor/freshness.py" + command -v python3 >/dev/null 2>&1 || { + echo "FATAL: python3 is required to verify SUT binary freshness"; return 1; } + test -f "$checker" || { echo "FATAL: freshness checker missing at $checker"; return 1; } + python3 "$checker" "$binary" --repo "$TB_REPO" ${1+"$@"} || { + echo "FATAL: $binary is not the code this run would be reported as measuring." + echo " Rebuild with bench/evidence/run/build_sut.sh. Repointing the" + echo " path or refreshing sut_commit.txt does not make the binary newer." + return 1 + } +} + preflight() { test -n "${OPENROUTER_API_KEY:-}" || { echo "FATAL: OPENROUTER_API_KEY unset"; return 1; } test -x "$STELLA_BINARY" || { echo "FATAL: no SUT binary at $STELLA_BINARY (run build_sut.sh)"; return 1; } assert_portable_binary || return 1 + assert_fresh_sut || return 1 test "${#STELLA_SOURCE_COMMIT}" = 40 || { echo "FATAL: STELLA_SOURCE_COMMIT is not a full SHA"; return 1; } test "$(command -v harbor)" = "$VENV/bin/harbor" || { echo "FATAL: wrong harbor on PATH"; return 1; } test "$(harbor --version)" = "0.6.1" || { echo "FATAL: harbor $(harbor --version) != 0.6.1 (audited constant)"; return 1; } diff --git a/bench/harbor_adapter/stella_harbor/freshness.py b/bench/harbor_adapter/stella_harbor/freshness.py new file mode 100644 index 000000000..494b0f9ca --- /dev/null +++ b/bench/harbor_adapter/stella_harbor/freshness.py @@ -0,0 +1,575 @@ +"""Refuse a SUT binary too far behind the code it will be reported as measuring. + +Why this exists +--------------- +Pinning and freshness are different properties, and until this module only the +first was checked. A run records the commit its binary was built from, so every +number is attributable to *some* revision — but nothing asked whether that +revision is anywhere near the one a reader will assume was measured. + +Measured 2026-08-07, the path every runbook and launch script exports as +``STELLA_BINARY``:: + + ~/.arenabench/sut/stella built Aug 4 + ~/.arenabench/sut/sut_commit.txt 47d587a37da0e1013010f4479295b33878760b7d + + $ git rev-list --count 47d587a3..origin/main + 291 + +That binary was accompanied by its own commit file, so it *looked* pinned and +passed every check that existed: it was pinned, to a commit from three days and +291 changes earlier. Every match run against it reported numbers for code that +had since been rewritten underneath them, and nothing said a word. + +What this module checks +----------------------- +Two questions the caller cannot answer from a path: + +**Which commit is this binary, really?** ``crates/stella-cli/build.rs`` stamps +``STELLA_BUILD_GIT_SHA`` into a single ``-dev.<40-hex>`` literal, and +``build_info.rs`` deliberately surrounds it with NUL bytes so LLVM's string +pooling cannot adjoin identifier characters to it — precisely so the identity +can be read out of the file without executing it. That is *intrinsic*: it +travels inside the artifact and no rename, copy, or symlink can separate the +two. ``sut_commit.txt`` beside the binary is *extrinsic* — a claim about the +file, which is the thing that went stale above. Both are read; the intrinsic +one decides, and a disagreement between them is itself a refusal, because it +means somebody's bookkeeping describes a binary that is not there. + +**How far is that commit from the reference?** ``git rev-list --left-right +--count ...`` in a checkout of this repository. A commit that +is not an ancestor of the reference at all — built from a topic branch, or from +a branch since force-pushed away — is reported as incomparable rather than as +zero, because unknown is not the same as current and must never render as one. + +Reading the ELF rather than running ``stella --version`` is what lets this run +on the host *before any container exists*, on a macOS development machine that +cannot exec a linux/amd64 binary at all. A guard that no-ops on the machine most +likely to be staging the wrong build is not a guard — the same reasoning that +made :mod:`stella_harbor.portability` parse ``.gnu.version_r`` in pure stdlib +instead of shelling out to ``readelf``. + +The reference and the distance +------------------------------ +``origin/main`` is the reference because it is what a published number is read +as measuring, and because ``bench/evidence/run/build_sut.sh`` already refuses to +build anything else. + +:data:`DEFAULT_MAX_BEHIND` is 25 commits. It is a stated distance, not a claim +that 25 commits change nothing: at the rate this repository's ``main`` actually +moves — 654 commits in the seven days to 2026-08-07, about 93 a day — 25 is +roughly six hours of drift. That tolerates a binary staged at the start of a +long Terminal-Bench run whose ``main`` moved underneath it, and refuses the +291-commit artifact above by a factor of twelve. An operator who genuinely means +to measure older code passes ``--max-behind`` explicitly, which makes the +distance appear in the launch command instead of in nobody's head. + +Run standalone:: + + python3 stella_harbor/freshness.py /path/to/stella --repo /path/to/checkout + +Exit status is ``0`` fresh, ``1`` stale, ``2`` could not be determined. +Preflight treats every nonzero status as fatal: a binary whose distance from the +reference cannot be established has not been shown to be near it. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +__all__ = [ + "DEFAULT_MAX_BEHIND", + "DEFAULT_REFERENCE", + "SIDECAR_FILENAME", + "Distance", + "ReferenceUnavailableError", + "SutFreshnessError", + "SutIdentity", + "SutIdentityUnknownError", + "check_freshness", + "embedded_source_commits", + "measure_distance", + "read_identity", + "read_sidecar_commit", +] + +#: How far behind the reference a SUT binary may be. See the module docstring +#: for where the number comes from; it is deliberately a stated distance rather +#: than a tolerance anybody has argued is harmless. +DEFAULT_MAX_BEHIND = 25 + +#: What a published number is read as measuring. `build_sut.sh` refuses to build +#: from anything else, so the reference and the build agree by construction. +DEFAULT_REFERENCE = "origin/main" + +#: The commit file `build_sut.sh` writes beside a staged binary. +SIDECAR_FILENAME = "sut_commit.txt" + +#: The compile-time `-dev.` marker `build.rs` stamps into the binary. The +#: trailing lookahead is what makes a match a whole token: the literal is +#: NUL-delimited on purpose (`crates/stella-cli/src/build_info.rs`), so a +#: 40-hex run followed by another hex digit is not this marker. +_VERSION_COMMIT_BYTES = re.compile(rb"-dev\.([0-9A-Fa-f]{40})(?=[^0-9A-Fa-f])") + +_FULL_SHA = re.compile(r"\A[0-9a-f]{40}\Z") + +#: Only a full hex SHA or a conservative ref name ever reaches `git`. In +#: particular a leading `-` is rejected, which git would otherwise read as an +#: option rather than as a ref. +_SAFE_REF = re.compile(r"\A[A-Za-z0-9][A-Za-z0-9._/-]{0,200}\Z") + +_CHUNK_BYTES = 1024 * 1024 + +#: Longest boundary a marker can straddle between two reads: the literal is +#: `-dev.` plus 40 hex plus the delimiter that ends it. +_OVERLAP_BYTES = 64 + + +class SutFreshnessError(RuntimeError): + """A binary's distance from the reference could not be established.""" + + +class SutIdentityUnknownError(SutFreshnessError): + """The binary does not say which commit it was built from, and nothing else does.""" + + +class ReferenceUnavailableError(SutFreshnessError): + """The reference commit could not be resolved in the given checkout.""" + + +@dataclass(frozen=True) +class SutIdentity: + """Which commit a staged binary is, and how that was established.""" + + #: The path as given, before symlinks are followed. + path: Path + #: The same path with every symlink resolved. The rig's mitigation for the + #: incident above is a symlink at the legacy location, so this is where the + #: sidecar actually lives — and reporting both is what makes a run's log + #: show that the runbook path and the artifact are the same file. + resolved_path: Path + #: The commit this binary is, or ``""`` when nothing recorded it. + commit: str = "" + #: ``"embedded"`` (read out of the binary) or ``"sidecar"``. + source: str = "" + #: Every distinct commit marker found in the binary's bytes. More than one + #: means the artifact is not a single stamped build and cannot be named. + embedded_commits: tuple[str, ...] = () + #: What `sut_commit.txt` beside the resolved binary claims, if anything. + sidecar_commit: str = "" + + @property + def known(self) -> bool: + return bool(self.commit) + + @property + def sidecar_disagrees(self) -> bool: + """The sidecar names a different commit than the binary's own bytes.""" + return bool( + self.sidecar_commit + and self.embedded_commits + and self.sidecar_commit not in self.embedded_commits + ) + + def describe(self) -> str: + """One line naming the properties that decide the verdict.""" + if not self.known: + return "records no commit" + parts = [f"{self.commit[:8]} (from the {self.source})"] + if self.sidecar_disagrees: + parts.append(f"sidecar claims {self.sidecar_commit[:8]}") + if len(self.embedded_commits) > 1: + parts.append(f"{len(self.embedded_commits)} embedded markers") + return ", ".join(parts) + + +@dataclass(frozen=True) +class Distance: + """How far a binary's commit is from the reference, in commits.""" + + commit: str + reference: str + reference_commit: str + behind: int = 0 + ahead: int = 0 + #: ``False`` when the two commits cannot be related in this checkout at all. + #: Unknown is not zero and must never be rendered as "current". + comparable: bool = True + + @property + def identical(self) -> bool: + return bool(self.commit) and self.commit == self.reference_commit + + def summary(self) -> str: + if self.identical: + return f"exactly {self.reference} ({self.reference_commit[:8]})" + if not self.comparable: + return ( + f"{self.commit[:8]} is neither an ancestor nor a descendant of " + f"{self.reference} ({self.reference_commit[:8]}) in this " + "checkout, so its distance cannot be measured" + ) + parts = [] + if self.behind: + parts.append(f"{self.behind} commit(s) behind") + if self.ahead: + parts.append(f"{self.ahead} commit(s) ahead of") + joined = " and ".join(parts) or "level with" + return ( + f"{self.commit[:8]} is {joined} {self.reference} " + f"({self.reference_commit[:8]})" + ) + + def to_json(self) -> dict[str, object]: + return { + "commit": self.commit, + "reference": self.reference, + "reference_commit": self.reference_commit, + "behind": self.behind, + "ahead": self.ahead, + "comparable": self.comparable, + "identical": self.identical, + "summary": self.summary(), + } + + +def embedded_source_commits(path: Path | str) -> tuple[str, ...]: + """Every compile-time commit marker in a binary's bytes, lowercased. + + Streamed with a fixed overlap rather than read whole: the SUT is ~32 MiB and + preflight has no reason to hold it in memory. The overlap is what keeps a + marker split across two reads from being missed — silently dropping the only + intrinsic evidence would fall back to the sidecar, which is the artifact this + module exists not to trust. + """ + binary = Path(path) + found: list[str] = [] + tail = b"" + with binary.open("rb") as handle: + for chunk in iter(lambda: handle.read(_CHUNK_BYTES), b""): + window = tail + chunk + found.extend( + match.decode("ascii").lower() + for match in _VERSION_COMMIT_BYTES.findall(window) + ) + tail = window[-_OVERLAP_BYTES:] + # A marker ending exactly at EOF has no delimiter after it for the lookahead + # to match, so it is accepted here instead of being lost to truncation. + at_eof = re.search(rb"-dev\.([0-9A-Fa-f]{40})\Z", tail) + if at_eof is not None: + found.append(at_eof.group(1).decode("ascii").lower()) + return tuple(dict.fromkeys(found)) + + +def read_sidecar_commit(directory: Path) -> str: + """The commit ``build_sut.sh`` recorded beside a staged binary, or ``""``.""" + try: + text = (directory / SIDECAR_FILENAME).read_text(encoding="utf-8").strip() + except OSError: + return "" + return text.lower() if _FULL_SHA.match(text.lower()) else "" + + +def read_identity(path: Path | str) -> SutIdentity: + """Establish which commit a staged binary is. + + The binary's own bytes decide. The sidecar is read regardless — not as a + fallback ranking but so a disagreement between the two can be *reported*, + since a commit file describing a binary that is not there is a bookkeeping + failure whichever of them is right. + """ + given = Path(path) + try: + resolved = given.resolve(strict=True) + except OSError as error: + raise SutFreshnessError(f"{given} cannot be resolved: {error}") from error + if not resolved.is_file(): + raise SutFreshnessError(f"{given} does not resolve to a regular file") + + try: + embedded = embedded_source_commits(resolved) + except OSError as error: + raise SutFreshnessError(f"cannot read {resolved}: {error}") from error + sidecar = read_sidecar_commit(resolved.parent) + + commit, source = "", "" + if len(embedded) == 1: + commit, source = embedded[0], "embedded build stamp" + elif not embedded and sidecar: + commit, source = sidecar, f"{SIDECAR_FILENAME} sidecar" + + return SutIdentity( + path=given, + resolved_path=resolved, + commit=commit, + source=source, + embedded_commits=embedded, + sidecar_commit=sidecar, + ) + + +def _git(repo: Path, *args: str, timeout: float = 30.0) -> str: + try: + result = subprocess.run( + ["git", *args], + cwd=str(repo), + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise ReferenceUnavailableError( + f"git {' '.join(args)} failed: {error}" + ) from error + if result.returncode != 0: + raise ReferenceUnavailableError( + f"git {' '.join(args)} exited {result.returncode}: {result.stderr.strip()}" + ) + return result.stdout + + +def _safe_ref(ref: str) -> str: + stripped = ref.strip() + if _FULL_SHA.match(stripped.lower()) or _SAFE_REF.match(stripped): + return stripped + raise ReferenceUnavailableError(f"refusing to resolve unsafe ref {ref!r}") + + +def measure_distance( + commit: str, + repo: Path | str, + reference: str = DEFAULT_REFERENCE, +) -> Distance: + """How far ``commit`` is behind ``reference`` in the checkout at ``repo``. + + Never fetches. Preflight runs on a rig that may be offline mid-session, and + a guard that reaches the network is a guard that fails for reasons unrelated + to what it measures — whoever wants a fresher reference fetches first, in + the wrapper that also builds. What this reports is therefore the distance + from the reference *as this checkout knows it*, which is stated in the + verdict so a stale checkout cannot masquerade as a fresh binary. + """ + checkout = Path(repo) + if not (checkout / ".git").exists(): + raise ReferenceUnavailableError(f"{checkout} is not a git checkout") + reference_commit = _git( + checkout, "rev-parse", "--verify", f"{_safe_ref(reference)}^{{commit}}" + ).strip() + + if commit == reference_commit: + return Distance( + commit=commit, reference=reference, reference_commit=reference_commit + ) + try: + counts = _git( + checkout, + "rev-list", + "--left-right", + "--count", + f"{commit}...{reference_commit}", + ).split() + ahead, behind = int(counts[0]), int(counts[1]) + except (ReferenceUnavailableError, IndexError, ValueError): + # The binary's commit is not in this checkout — built elsewhere, or on a + # branch since force-pushed away. Reported as incomparable, not as zero. + return Distance( + commit=commit, + reference=reference, + reference_commit=reference_commit, + comparable=False, + ) + return Distance( + commit=commit, + reference=reference, + reference_commit=reference_commit, + behind=behind, + ahead=ahead, + ) + + +def check_freshness( + identity: SutIdentity, + distance: Distance, + *, + max_behind: int = DEFAULT_MAX_BEHIND, +) -> list[str]: + """Every reason this binary must not be measured, most damning first. + + An empty list means the binary is close enough to the reference that a + number produced with it can be attributed to that reference. + """ + violations: list[str] = [] + + if len(identity.embedded_commits) > 1: + named = ", ".join(sha[:8] for sha in identity.embedded_commits) + violations.append( + f"carries {len(identity.embedded_commits)} different compile-time " + f"commit stamps ({named}), so it is not one stamped build and no " + "single commit can be attributed to it" + ) + + if identity.sidecar_disagrees: + violations.append( + f"{SIDECAR_FILENAME} beside it claims " + f"{identity.sidecar_commit[:8]}, but the binary's own compile-time " + f"stamp says {identity.commit[:8]} — the bookkeeping describes a " + "different artifact than the one that would run" + ) + + if not distance.comparable: + violations.append(distance.summary()) + elif distance.behind > max_behind: + violations.append( + f"{distance.summary()}, past the {max_behind}-commit limit — a " + "number measured with it would be reported as this branch's and is " + "not" + ) + elif distance.ahead: + violations.append( + f"{distance.summary()} — it carries {distance.ahead} commit(s) that " + f"{distance.reference} does not, so it is not the code a reader of " + "the result will assume was measured" + ) + + return violations + + +def _render_report( + identity: SutIdentity, distance: Distance, violations: list[str] +) -> str: + lines = [f"binary: {identity.path}"] + if identity.resolved_path != identity.path: + lines.append(f"resolves to: {identity.resolved_path}") + lines.append(f"identity: {identity.describe()}") + lines.append(f"distance: {distance.summary()}") + if violations: + lines.append("STALE:") + lines.extend(f" - {item}" for item in violations) + lines.append( + "Rebuild with `bench/evidence/run/build_sut.sh`, which resolves the " + "commit once and stamps it into the binary. Pass --max-behind " + "explicitly to measure older code on purpose — the distance then " + "appears in the launch command rather than in nobody's head." + ) + else: + lines.append("fresh: close enough to the reference to be reported as it") + return "\n".join(lines) + + +def _default_repo() -> str: + """The checkout to measure against, without asking the caller twice. + + ``TB_REPO`` is what ``bench/evidence/run/env.sh`` already exports, and this + file lives three directories below the repository root, so a standalone + invocation from anywhere still finds the right checkout. + """ + from_env = os.environ.get("TB_REPO") + if from_env: + return from_env + return str(Path(__file__).resolve().parents[3]) + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point. 0 fresh, 1 stale, 2 undeterminable.""" + parser = argparse.ArgumentParser( + prog="freshness", + description=( + "Assert a SUT binary is close enough to the reference branch that a " + "number measured with it can be attributed to that branch." + ), + ) + parser.add_argument("binary", help="path to the staged SUT binary") + parser.add_argument( + "--repo", + default=None, + help="checkout to measure against (default: $TB_REPO, else this repository)", + ) + parser.add_argument( + "--reference", + default=DEFAULT_REFERENCE, + help="ref the binary is reported as measuring (default: %(default)s)", + ) + parser.add_argument( + "--max-behind", + type=int, + default=DEFAULT_MAX_BEHIND, + help="commits behind the reference the binary may be (default: %(default)s)", + ) + parser.add_argument( + "--json", action="store_true", help="emit the identity and verdict as JSON" + ) + args = parser.parse_args(argv) + + if args.max_behind < 0: + print("FATAL: --max-behind cannot be negative", file=sys.stderr) + return 2 + + try: + identity = read_identity(args.binary) + except (SutFreshnessError, OSError) as error: + print(f"FATAL: cannot establish SUT identity: {error}", file=sys.stderr) + return 2 + + if not identity.known: + # Fail closed. A binary nobody stamped is exactly as unattributable as + # one stamped with the wrong commit, and this check exists because + # "it looked fine" is how the stale artifact kept being measured. + detail = ( + f"it carries {len(identity.embedded_commits)} compile-time stamps" + if identity.embedded_commits + else "it carries no compile-time stamp and has no " + f"{SIDECAR_FILENAME} beside it" + ) + print( + f"FATAL: {identity.resolved_path} does not say which commit it is: " + f"{detail}. Build it with bench/evidence/run/build_sut.sh, which " + "stamps STELLA_BUILD_GIT_SHA into the artifact.", + file=sys.stderr, + ) + return 2 + + try: + distance = measure_distance( + identity.commit, args.repo or _default_repo(), args.reference + ) + except SutFreshnessError as error: + print(f"FATAL: cannot measure distance: {error}", file=sys.stderr) + return 2 + + violations = check_freshness(identity, distance, max_behind=args.max_behind) + + if args.json: + print( + json.dumps( + { + "binary": str(identity.path), + "resolved_path": str(identity.resolved_path), + "commit": identity.commit, + "commit_source": identity.source, + "embedded_commits": list(identity.embedded_commits), + "sidecar_commit": identity.sidecar_commit, + "distance": distance.to_json(), + "max_behind": args.max_behind, + "violations": violations, + "fresh": not violations, + }, + indent=2, + ) + ) + else: + stream = sys.stderr if violations else sys.stdout + print(_render_report(identity, distance, violations), file=stream) + + return 1 if violations else 0 + + +if __name__ == "__main__": # pragma: no cover - exercised via subprocess in tests + raise SystemExit(main()) From f86365247db4236625ffdabc6717d6fe527320ea Mon Sep 17 00:00:00 2001 From: Mac Anderson Date: Thu, 6 Aug 2026 22:15:24 -0700 Subject: [PATCH 2/7] wip(bench,arenabench): freshness tests + unpinned staleness refusal --- arenabench/arenabench/sut.py | 148 ++++++- arenabench/tests/test_sut.py | 170 +++++++- bench/harbor_adapter/tests/test_freshness.py | 423 +++++++++++++++++++ 3 files changed, 736 insertions(+), 5 deletions(-) create mode 100644 bench/harbor_adapter/tests/test_freshness.py diff --git a/arenabench/arenabench/sut.py b/arenabench/arenabench/sut.py index 6cf8efb1b..fceff5c1d 100644 --- a/arenabench/arenabench/sut.py +++ b/arenabench/arenabench/sut.py @@ -52,13 +52,17 @@ from typing import Any __all__ = [ + "MAX_BEHIND_UNPINNED", + "STELLA_BINARY_ENV", "STELLA_REPO_ENV", "Branch", "Drift", "StagedSut", "SutUnavailableError", + "ambient_sut", "binary_for", "drift_between", + "embedded_commit", "list_branches", "resolve_ref", "staged_for", @@ -71,6 +75,30 @@ #: Points the arena at the Stella checkout it resolves refs and builds from. STELLA_REPO_ENV = "ARENABENCH_STELLA_REPO" +#: The binary a match runs when it is not pinned to a commit. +STELLA_BINARY_ENV = "STELLA_BINARY" + +#: How far behind the default branch an *unpinned* binary may be. +#: +#: An unpinned match is asking for "whatever is current", so a binary that is +#: measurably not current contradicts the request rather than opting out of it. +#: Measuring older code on purpose is still available and is spelled the +#: supported way — pin ``sut_ref`` to that commit, which records in the result +#: what was measured instead of leaving it to the state of one path on one +#: machine. The number matches the checker the Terminal-Bench runbook uses +#: (``bench/harbor_adapter/stella_harbor/freshness.py``): about six hours of +#: this repository's ``main``, which moved 654 commits in the seven days to +#: 2026-08-07. +MAX_BEHIND_UNPINNED = 25 + +#: The compile-time ``-dev.`` marker Stella's ``build.rs`` stamps into the +#: binary. ``crates/stella-cli/src/build_info.rs`` deliberately delimits it with +#: NUL bytes so LLVM's string pooling cannot adjoin identifier characters to it, +#: which is what makes the identity readable without executing the artifact — +#: necessary here, because the SUT is a linux/amd64 cross-build and the arena +#: routinely runs on a macOS host that cannot exec it. +_VERSION_COMMIT_BYTES = re.compile(rb"-dev\.([0-9A-Fa-f]{40})(?=[^0-9A-Fa-f])") + #: A full 40-hex commit id — the only ref shape ever passed to git unvalidated. _FULL_SHA = re.compile(r"\A[0-9a-f]{40}\Z") @@ -383,6 +411,76 @@ def legacy_staged() -> StagedSut | None: return _staged_at(sut_root()) +def embedded_commit(path: Path) -> str: + """The commit stamped into a Stella binary at compile time, or ``""``. + + This is the artifact's *intrinsic* identity: it travels inside the file, so + no rename, copy, or symlink can separate the two. ``sut_commit.txt`` beside + a binary is a *claim about* the file, and a claim is what went wrong — the + stale binary had one, correctly naming a commit from three days earlier, and + so passed for pinned. + + Read in chunks with an overlap, because a marker split across two reads + would otherwise be missed and silently downgrade the answer to the sidecar. + Any read error yields ``""``: this reports identity, and "cannot tell" is a + state its callers already handle as unknown rather than as current. + """ + found: list[str] = [] + tail = b"" + try: + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + window = tail + chunk + found.extend( + match.decode("ascii").lower() + for match in _VERSION_COMMIT_BYTES.findall(window) + ) + tail = window[-64:] + except OSError: + return "" + at_eof = re.search(rb"-dev\.([0-9A-Fa-f]{40})\Z", tail) + if at_eof is not None: + found.append(at_eof.group(1).decode("ascii").lower()) + unique = set(found) + # More than one stamp is not one build, and naming either would be a guess. + return unique.pop() if len(unique) == 1 else "" + + +def ambient_sut() -> StagedSut | None: + """The binary ``STELLA_BINARY`` names, with whatever identity it carries. + + This is what an unpinned match runs, and it is the path the operator runbook + hands to `arenabench serve` — so it is the one binary nothing was checking. + Symlinks are followed before the sidecar is read: the mitigation applied on + the rig for this defect is a symlink at the legacy location, and reading the + commit file next to the *link* rather than next to the *target* would report + the identity of whatever used to be there. + """ + raw = os.environ.get(STELLA_BINARY_ENV) + if not raw: + return None + try: + resolved = Path(raw).expanduser().resolve(strict=True) + except OSError: + return None + if not resolved.is_file(): + return None + # Read the sidecars directly rather than through `_staged_at`, which finds + # them only beside a file literally named `stella`. STELLA_BINARY may name + # anything, and this must report on the binary it was actually handed. + intrinsic = embedded_commit(resolved) + try: + built_at = resolved.stat().st_mtime + except OSError: + built_at = 0.0 + return StagedSut( + path=resolved, + commit=intrinsic or _read(resolved.parent / "sut_commit.txt"), + sha256=_read(resolved.parent / "binary_sha256.txt"), + built_at=built_at, + ) + + def binary_for(commit: str) -> StagedSut | None: """The binary a match pinned to ``commit`` should run, or ``None``. @@ -427,6 +525,48 @@ def file_sha256(path: Path) -> str: return digest.hexdigest() +def unpinned_problem() -> str | None: + """Why an *unpinned* Stella match must not launch, or ``None``. + + Clearing the pin says "run whatever is current", so the one thing that can + still be wrong is the binary not being current. That is not hypothetical: + the path the operator runbook hands to ``arenabench serve`` was 291 commits + behind ``main`` for three days, carrying a ``sut_commit.txt`` that named its + own ancient commit, and every unpinned match against it produced perfectly + scoreable trials attributed to code that had been rewritten underneath them. + + **Refuses only on positive evidence of staleness.** An ordinary + ``cargo build --release`` carries no compile-time stamp and cannot be dated, + and refusing it would block the local development loop this tool exists to + serve; those runs keep the existing "unverified" label and warning. The + Terminal-Bench evidence runbook takes the opposite posture and fails closed + on an unidentifiable binary, because it publishes numbers — the difference + is deliberate and follows from who reads the result, not from a disagreement + about what is safe. + """ + ambient = ambient_sut() + if ambient is None or not ambient.commit: + return None + try: + target = resolve_ref("main") + except SutUnavailableError: + # No checkout to compare against. Nothing is proven either way, and an + # arena that cannot see a repository still has to run matches. + return None + drift = drift_between(ambient.commit, target) + if not drift.comparable or drift.behind <= MAX_BEHIND_UNPINNED: + return None + return ( + f"the Stella seat is unpinned, so it runs {STELLA_BINARY_ENV}=" + f"{ambient.path} — and {drift.summary()}, past the " + f"{MAX_BEHIND_UNPINNED}-commit limit. An unpinned match asks for " + "whatever is current, and this binary is not. Rebuild it " + "(`bench/evidence/run/build_sut.sh`), or pin the SUT to the commit you " + "actually mean to measure — pinning is how the result records which " + "Stella it was." + ) + + def sut_problem_for(spec: Any) -> str | None: """Why this match must not launch, as far as the SUT is concerned. @@ -435,13 +575,13 @@ def sut_problem_for(spec: Any) -> str | None: Only matches carrying a Stella seat are checked — a Claude-Code-vs-Codex contest has no system under test of ours and must not be blocked by the - state of a binary it never runs. An empty ``sut_ref`` is the deliberate - opt-out and always passes; the run is labelled unverified instead. + state of a binary it never runs. An empty ``sut_ref`` opts out of *pinning*, + not out of every check: see :func:`unpinned_problem`. """ - if not getattr(spec, "sut_ref", ""): - return None if not any(c.agent == "stella" for c in spec.contestants): return None + if not getattr(spec, "sut_ref", ""): + return unpinned_problem() status = sut_status(spec.sut_ref) if status["ready"]: return None diff --git a/arenabench/tests/test_sut.py b/arenabench/tests/test_sut.py index 23a0865b0..5c69c8829 100644 --- a/arenabench/tests/test_sut.py +++ b/arenabench/tests/test_sut.py @@ -59,6 +59,9 @@ def repo(tmp_path: Path, monkeypatch) -> Path: monkeypatch.setenv(sut.STELLA_REPO_ENV, str(work)) monkeypatch.setenv("ARENABENCH_HOME", str(tmp_path / "home")) + # An unpinned seat runs whatever STELLA_BINARY names, so a developer whose + # shell exports it would otherwise be running a different test than CI. + monkeypatch.delenv(sut.STELLA_BINARY_ENV, raising=False) return work @@ -215,7 +218,8 @@ def test_a_match_with_no_stella_seat_is_never_blocked(self, repo: Path): """A Claude-Code-vs-Codex contest runs no SUT of ours.""" assert sut.sut_problem_for(self._spec(agent="claude-code")) is None - def test_the_empty_ref_is_an_explicit_opt_out(self, repo: Path): + def test_the_empty_ref_opts_out_of_pinning_with_nothing_staged(self, repo: Path): + """With no STELLA_BINARY there is nothing to be stale, so nothing to say.""" assert sut.sut_problem_for(self._spec(ref="")) is None def test_a_spec_written_before_this_field_defaults_to_main_not_unpinned(self): @@ -228,6 +232,170 @@ def test_a_spec_written_before_this_field_defaults_to_main_not_unpinned(self): assert MatchSpec.from_json({"name": "m", "dataset": "d"}).sut_ref == "main" +def _stamped(path: Path, commit: str | None) -> Path: + """A stand-in Stella binary carrying the compile-time stamp `build.rs` emits. + + NUL-delimited on both sides, exactly as ``build_info.rs`` lays it out — that + delimiting is what makes the commit readable out of the file, and a fixture + that omitted it would be testing a shape the real artifact does not have. + """ + path.parent.mkdir(parents=True, exist_ok=True) + body = b"\x7fELF" + b"\x00" * 64 + if commit is not None: + body += b"\x000.6.132-dev." + commit.encode("ascii") + b"\x00" + path.write_bytes(body) + return path + + +class TestAnUnpinnedMatchStillRefusesStaleCode: + """Clearing the pin says "run whatever is current", not "run anything". + + The witness for #2032. The operator runbook launches with + ``STELLA_BINARY=~/.arenabench/sut/stella``, and that path sat 291 commits + behind ``main`` for three days while producing perfectly scoreable trials. + The commit pinning added in #2016/#2020 did not catch it, because the stale + binary *was* pinned — to a commit from three days earlier. Pinning answers + which code; only a distance answers whether it is the code anyone meant. + """ + + #: Deeper than `MAX_BEHIND_UNPINNED`, so "behind" and "too far behind" are + #: distinguishable rather than coincidentally the same assertion. + DEPTH = sut.MAX_BEHIND_UNPINNED * 2 + + @pytest.fixture + def deep_repo(self, tmp_path: Path, monkeypatch) -> Path: + origin = tmp_path / "deep-origin" + origin.mkdir() + _git(origin, "init", "-q", "--initial-branch=main") + _git(origin, "config", "user.email", "t@example.com") + _git(origin, "config", "user.name", "t") + _git(origin, "commit", "-q", "--allow-empty", "-m", "base") + work = tmp_path / "deep-work" + _git(tmp_path, "clone", "-q", str(origin), str(work)) + for index in range(self.DEPTH): + _git(origin, "commit", "-q", "--allow-empty", "-m", f"c{index}") + _git(work, "fetch", "-q", "origin") + monkeypatch.setenv(sut.STELLA_REPO_ENV, str(work)) + monkeypatch.delenv(sut.STELLA_BINARY_ENV, raising=False) + return work + + def _unpinned_stella_spec(self) -> MatchSpec: + return MatchSpec.from_json( + { + "name": "m", + "dataset": "terminal-bench-2.1", + "tasks": ["fix-git"], + "sut_ref": "", + "contestants": [ + { + "name": "seat", + "agent": "stella", + "engine": {"api": "openrouter", "model": "m"}, + } + ], + } + ) + + def test_the_stale_runbook_binary_now_blocks_the_launch( + self, deep_repo: Path, tmp_path: Path, monkeypatch + ): + """The exact scenario in the issue: unpinned seat, ancient STELLA_BINARY.""" + old = _git(deep_repo, "rev-list", "--max-parents=0", "origin/main") + binary = _stamped(tmp_path / "sut" / "stella", old) + monkeypatch.setenv(sut.STELLA_BINARY_ENV, str(binary)) + + problem = sut.sut_problem_for(self._unpinned_stella_spec()) + assert problem is not None, ( + "an unpinned match asks for current code; this binary is not it" + ) + assert old[:8] in problem, "the refusal must name the commit" + assert f"{self.DEPTH} commit(s) behind" in problem, ( + "the refusal must name the distance, not merely that there is one" + ) + + def test_a_symlinked_runbook_path_is_followed_to_its_target( + self, deep_repo: Path, tmp_path: Path, monkeypatch + ): + """The rig's mitigation is a symlink; the guard must see through it. + + A guard that read the link's own directory would report whatever + `sut_commit.txt` was last left beside the legacy path — which is the + stale claim, not the binary that would actually run. + """ + head = _git(deep_repo, "rev-parse", "origin/main") + target = _stamped(tmp_path / "sut" / head / "stella", head) + old = _git(deep_repo, "rev-list", "--max-parents=0", "origin/main") + (tmp_path / "sut" / "sut_commit.txt").write_text(old) + link = tmp_path / "sut" / "stella" + link.symlink_to(target) + monkeypatch.setenv(sut.STELLA_BINARY_ENV, str(link)) + + assert sut.ambient_sut().commit == head + assert sut.sut_problem_for(self._unpinned_stella_spec()) is None + + def test_a_binary_within_the_limit_still_launches( + self, deep_repo: Path, tmp_path: Path, monkeypatch + ): + """`main` moves under every run; the limit exists so that is not fatal.""" + near = _git( + deep_repo, "rev-parse", f"origin/main~{sut.MAX_BEHIND_UNPINNED}" + ) + monkeypatch.setenv( + sut.STELLA_BINARY_ENV, str(_stamped(tmp_path / "stella", near)) + ) + assert sut.sut_problem_for(self._unpinned_stella_spec()) is None + + def test_an_unstamped_development_build_is_not_blocked( + self, deep_repo: Path, tmp_path: Path, monkeypatch + ): + """`cargo build --release` carries no stamp and cannot be dated. + + Refusing it would block the local loop this tool exists to serve. The + arena refuses only on positive evidence of staleness; the Terminal-Bench + evidence runbook fails closed instead, because it publishes numbers. + """ + monkeypatch.setenv( + sut.STELLA_BINARY_ENV, str(_stamped(tmp_path / "stella", None)) + ) + assert sut.sut_problem_for(self._unpinned_stella_spec()) is None + + def test_a_non_stella_contest_is_never_blocked( + self, deep_repo: Path, tmp_path: Path, monkeypatch + ): + old = _git(deep_repo, "rev-list", "--max-parents=0", "origin/main") + monkeypatch.setenv( + sut.STELLA_BINARY_ENV, str(_stamped(tmp_path / "stella", old)) + ) + spec = MatchSpec.from_json( + { + "name": "m", + "dataset": "terminal-bench-2.1", + "tasks": ["fix-git"], + "sut_ref": "", + "contestants": [ + { + "name": "seat", + "agent": "claude-code", + "engine": {"api": "anthropic", "model": "m"}, + } + ], + } + ) + assert sut.sut_problem_for(spec) is None + + def test_two_stamps_name_no_commit_and_do_not_block( + self, deep_repo: Path, tmp_path: Path, monkeypatch + ): + """Two stamps is not one build; guessing which is the point of failure.""" + path = tmp_path / "stella" + path.write_bytes( + b"\x000.6.1-dev." + b"a" * 40 + b"\x00\x000.6.1-dev." + b"b" * 40 + b"\x00" + ) + monkeypatch.setenv(sut.STELLA_BINARY_ENV, str(path)) + assert sut.embedded_commit(path) == "" + assert sut.sut_problem_for(self._unpinned_stella_spec()) is None + + class TestACommitPinSurvivesAMovingBranch: """Why a finished build pins its own commit rather than the branch. diff --git a/bench/harbor_adapter/tests/test_freshness.py b/bench/harbor_adapter/tests/test_freshness.py new file mode 100644 index 000000000..18f93e51b --- /dev/null +++ b/bench/harbor_adapter/tests/test_freshness.py @@ -0,0 +1,423 @@ +"""Regression tests for the SUT binary freshness guard. + +These would have caught the 2026-08-07 defect. ``~/.arenabench/sut/stella`` — +the path every runbook exports as ``STELLA_BINARY`` — was 291 commits and three +days behind ``origin/main``, and it carried its own ``sut_commit.txt`` naming +that ancient commit. So it *looked* pinned and passed every check that existed: +it was pinned, just not to anything near the code the numbers were reported as +measuring. Pinning and freshness are different properties and only the first +was ever checked. + +Every assertion here is on the *artifact and the graph*: a file is written with +chosen compile-time stamp bytes, a throwaway repository is given a chosen +history, and the guard's verdict is checked. Nothing asserts that a particular +command was typed, because the failure was that nobody re-typed the build +command for three days and everything downstream still agreed. + +``stella_harbor/freshness.py`` is loaded by path, the same way +``test_portability.py`` loads its subject: the module is stdlib-only on purpose +— ``env.sh`` invokes it by file path on a host that is only building the SUT — +and importing it through the package would pull in the adapter root, and with +it ``harbor``. +""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_CHECKER = _REPO_ROOT / "bench" / "harbor_adapter" / "stella_harbor" / "freshness.py" +_ENV_SH = _REPO_ROOT / "bench" / "evidence" / "run" / "env.sh" +_BUILD_SH = _REPO_ROOT / "bench" / "evidence" / "run" / "build_sut.sh" + +_spec = importlib.util.spec_from_file_location("stella_harbor_freshness", _CHECKER) +# Module-scope assert, deliberately: an import precondition, not a constant +# check — without it there is nothing to collect. +assert _spec is not None and _spec.loader is not None +_freshness = importlib.util.module_from_spec(_spec) +# Registered before execution: `@dataclass` resolves a class's own module out of +# `sys.modules` while the decorator runs, and a module that is not there yet +# fails the whole load. +sys.modules[_spec.name] = _freshness +_spec.loader.exec_module(_freshness) + +DEFAULT_MAX_BEHIND = _freshness.DEFAULT_MAX_BEHIND +SIDECAR_FILENAME = _freshness.SIDECAR_FILENAME +SutFreshnessError = _freshness.SutFreshnessError +check_freshness = _freshness.check_freshness +embedded_source_commits = _freshness.embedded_source_commits +measure_distance = _freshness.measure_distance +read_identity = _freshness.read_identity + +# The commit the stale binary was actually stamped with, and the distance it +# actually sat at. Used verbatim so the fixtures stay recognisable as the defect. +STALE_COMMIT = "47d587a37da0e1013010f4479295b33878760b7d" +STALE_DISTANCE = 291 + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], cwd=repo, capture_output=True, text=True, check=True + ).stdout.strip() + + +def _stamp(commit: str) -> bytes: + """The exact byte shape `build.rs` emits: NUL-delimited `-dev.`.""" + return b"\x000.6.132-dev." + commit.encode("ascii") + b"\x00" + + +def _binary(directory: Path, commit: str | None, *, name: str = "stella") -> Path: + """A stand-in SUT binary carrying (or lacking) a compile-time stamp.""" + directory.mkdir(parents=True, exist_ok=True) + path = directory / name + payload = b"\x7fELF" + b"\x00" * 128 + if commit is not None: + payload += _stamp(commit) + payload += b"trailing bytes" + path.write_bytes(payload) + return path + + +def _seed(root: Path, commits: int) -> Path: + """A clone whose `origin/main` is ``commits`` ahead of the root commit.""" + origin = root / "origin" + origin.mkdir(parents=True) + _git(origin, "init", "-q", "--initial-branch=main") + _git(origin, "config", "user.email", "t@example.com") + _git(origin, "config", "user.name", "t") + _git(origin, "commit", "-q", "--allow-empty", "-m", "base") + + work = root / "work" + _git(root, "clone", "-q", str(origin), str(work)) + _git(work, "config", "user.email", "t@example.com") + _git(work, "config", "user.name", "t") + for index in range(1, commits + 1): + _git(origin, "commit", "-q", "--allow-empty", "-m", f"c{index}") + _git(work, "fetch", "-q", "origin") + return work + + +@pytest.fixture(scope="module") +def repo(tmp_path_factory: pytest.TempPathFactory) -> Path: + """A checkout whose `origin/main` is far ahead of an older commit. + + Built to the shape of the defect rather than to a convenient minimum: the + guard's job is to notice a large distance, so the fixture produces the real + one and the assertions can name it. Module-scoped because 291 commits are + not free and nothing here mutates the history — tests stage their binaries + under their own ``tmp_path``. + """ + return _seed(tmp_path_factory.mktemp("stale"), STALE_DISTANCE) + + +@pytest.fixture +def base_commit(repo: Path) -> str: + """The commit `origin/main` is exactly ``STALE_DISTANCE`` ahead of.""" + return _git(repo, "rev-list", "--max-parents=0", "origin/main") + + +class TestEmbeddedIdentity: + """Reading the commit out of the artifact, not out of a file beside it.""" + + def test_reads_the_compile_time_stamp(self, tmp_path: Path) -> None: + assert embedded_source_commits(_binary(tmp_path, STALE_COMMIT)) == ( + STALE_COMMIT, + ) + + def test_uppercase_stamp_is_normalised(self, tmp_path: Path) -> None: + assert embedded_source_commits(_binary(tmp_path, STALE_COMMIT.upper())) == ( + STALE_COMMIT, + ) + + def test_repeated_stamp_is_one_commit(self, tmp_path: Path) -> None: + path = tmp_path / "stella" + path.write_bytes(_stamp(STALE_COMMIT) * 3) + assert embedded_source_commits(path) == (STALE_COMMIT,) + + def test_a_longer_hex_run_is_not_a_stamp(self, tmp_path: Path) -> None: + """The literal is NUL-delimited so a whole token is identifiable.""" + path = tmp_path / "stella" + path.write_bytes(b"-dev." + (STALE_COMMIT + "ab").encode("ascii") + b"\x00") + assert embedded_source_commits(path) == () + + def test_a_stamp_split_across_reads_is_still_found(self, tmp_path: Path) -> None: + """A missed marker would silently downgrade the answer to the sidecar. + + The reader streams in 1 MiB chunks, so a stamp straddling that boundary + is the one input a naive implementation loses — and losing it fails + *open*, back onto exactly the extrinsic claim this module distrusts. + """ + path = tmp_path / "stella" + stamp = _stamp(STALE_COMMIT) + # Land the stamp so it begins a few bytes before the first boundary. + path.write_bytes(b"\x00" * ((1 << 20) - 5) + stamp + b"\x00" * 32) + assert embedded_source_commits(path) == (STALE_COMMIT,) + + def test_a_stamp_at_end_of_file_is_found(self, tmp_path: Path) -> None: + path = tmp_path / "stella" + path.write_bytes(b"\x00" * 64 + b"-dev." + STALE_COMMIT.encode("ascii")) + assert embedded_source_commits(path) == (STALE_COMMIT,) + + +class TestIdentityResolution: + """Which of the two identities decides, and what a disagreement means.""" + + def test_the_binary_outranks_the_sidecar(self, tmp_path: Path) -> None: + fresh = "a" * 40 + path = _binary(tmp_path, fresh) + (tmp_path / SIDECAR_FILENAME).write_text(STALE_COMMIT) + identity = read_identity(path) + assert identity.commit == fresh + assert identity.source == "embedded build stamp" + assert identity.sidecar_disagrees + + def test_the_sidecar_answers_only_for_an_unstamped_binary( + self, tmp_path: Path + ) -> None: + path = _binary(tmp_path, None) + (tmp_path / SIDECAR_FILENAME).write_text(STALE_COMMIT) + identity = read_identity(path) + assert identity.commit == STALE_COMMIT + assert identity.source == f"{SIDECAR_FILENAME} sidecar" + assert not identity.sidecar_disagrees + + def test_two_stamps_name_no_commit(self, tmp_path: Path) -> None: + path = tmp_path / "stella" + path.write_bytes(_stamp(STALE_COMMIT) + _stamp("b" * 40)) + identity = read_identity(path) + assert not identity.known + assert len(identity.embedded_commits) == 2 + + def test_an_unidentifiable_binary_is_reported_as_such(self, tmp_path: Path) -> None: + assert not read_identity(_binary(tmp_path, None)).known + + def test_a_symlink_reads_the_sidecar_beside_its_target( + self, tmp_path: Path + ) -> None: + """The rig's mitigation for this defect is a symlink at the legacy path. + + Reading ``sut_commit.txt`` next to the *link* rather than next to the + *target* would report whatever used to be staged there — the very + confusion the symlink was introduced to end. + """ + target_dir = tmp_path / "sut" / ("c" * 40) + target = _binary(target_dir, None) + (target_dir / SIDECAR_FILENAME).write_text("c" * 40) + # A stale sidecar at the legacy location, exactly as found on the rig. + (tmp_path / "sut" / SIDECAR_FILENAME).write_text(STALE_COMMIT) + link = tmp_path / "sut" / "stella" + link.symlink_to(target) + + identity = read_identity(link) + assert identity.commit == "c" * 40 + assert identity.resolved_path == target.resolve() + assert identity.path == link + + +class TestDistance: + """Measuring how far the binary is from what it will be reported as.""" + + def test_counts_commits_behind_the_reference( + self, repo: Path, base_commit: str + ) -> None: + distance = measure_distance(base_commit, repo) + assert distance.behind == STALE_DISTANCE + assert distance.ahead == 0 + assert distance.comparable + + def test_the_reference_itself_is_identical(self, repo: Path) -> None: + head = _git(repo, "rev-parse", "origin/main") + assert measure_distance(head, repo).identical + + def test_an_unknown_commit_is_incomparable_not_zero(self, repo: Path) -> None: + """Unknown must never render as "current" — that is how a wrong number ships.""" + distance = measure_distance("f" * 40, repo) + assert not distance.comparable + assert distance.behind == 0 + assert "cannot be measured" in distance.summary() + + def test_a_divergent_commit_reports_both_directions(self, tmp_path: Path) -> None: + """Its own checkout: this is the one test here that writes history.""" + work = _seed(tmp_path, 10) + _git(work, "checkout", "-q", "-b", "side", "origin/main~10") + _git(work, "commit", "-q", "--allow-empty", "-m", "side") + distance = measure_distance(_git(work, "rev-parse", "HEAD"), work) + assert distance.ahead == 1 + assert distance.behind == 10 + + def test_a_non_checkout_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(SutFreshnessError): + measure_distance("a" * 40, tmp_path) + + def test_an_option_shaped_ref_never_reaches_git(self, repo: Path) -> None: + with pytest.raises(SutFreshnessError): + measure_distance("a" * 40, repo, "--upload-pack=touch /tmp/pwned") + + +class TestVerdict: + """What is refused, and what the refusal says.""" + + def test_the_stale_binary_is_refused(self, repo: Path, base_commit: str, tmp_path: Path) -> None: + identity = read_identity(_binary(tmp_path, base_commit)) + violations = check_freshness(identity, measure_distance(base_commit, repo)) + assert violations + assert f"{STALE_DISTANCE} commit(s) behind" in violations[0] + + def test_a_binary_within_the_limit_passes(self, repo: Path, tmp_path: Path) -> None: + near = _git(repo, "rev-parse", f"origin/main~{DEFAULT_MAX_BEHIND}") + identity = read_identity(_binary(tmp_path, near)) + assert check_freshness(identity, measure_distance(near, repo)) == [] + + def test_one_commit_past_the_limit_is_refused(self, repo: Path, tmp_path: Path) -> None: + past = _git(repo, "rev-parse", f"origin/main~{DEFAULT_MAX_BEHIND + 1}") + identity = read_identity(_binary(tmp_path, past)) + assert check_freshness(identity, measure_distance(past, repo)) + + def test_a_disagreeing_sidecar_is_refused_even_when_fresh(self, repo: Path, tmp_path: Path) -> None: + """Bookkeeping describing an absent artifact is a defect either way.""" + head = _git(repo, "rev-parse", "origin/main") + stage = tmp_path + identity_path = _binary(stage, head) + (stage / SIDECAR_FILENAME).write_text(STALE_COMMIT) + violations = check_freshness( + read_identity(identity_path), measure_distance(head, repo) + ) + assert any(SIDECAR_FILENAME in item for item in violations) + + def test_an_incomparable_commit_is_refused(self, repo: Path, tmp_path: Path) -> None: + identity = read_identity(_binary(tmp_path, "f" * 40)) + assert check_freshness(identity, measure_distance("f" * 40, repo)) + + +class TestCli: + """The surface `env.sh` actually invokes.""" + + def _run(self, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(_CHECKER), *args], + capture_output=True, + text=True, + check=False, + ) + + def test_the_documented_witness(self, repo: Path, base_commit: str, tmp_path: Path) -> None: + """Point the checker at the stale binary and it refuses, naming both facts. + + This is the witness the issue asks for: before this module the same + invocation produced scoreable trials, because nothing in the pipeline + had an opinion about distance. + """ + binary = _binary(tmp_path, base_commit) + result = self._run(str(binary), "--repo", str(repo)) + assert result.returncode == 1 + assert base_commit[:8] in result.stderr + assert f"{STALE_DISTANCE} commit(s) behind" in result.stderr + + def test_a_fresh_binary_is_accepted(self, repo: Path, tmp_path: Path) -> None: + head = _git(repo, "rev-parse", "origin/main") + result = self._run(str(_binary(tmp_path, head)), "--repo", str(repo)) + assert result.returncode == 0, result.stderr + assert "fresh" in result.stdout + + def test_an_unidentifiable_binary_fails_closed(self, repo: Path, tmp_path: Path) -> None: + """Status 2, not 0: not shown to be near the reference is not near it.""" + result = self._run(str(_binary(tmp_path, None)), "--repo", str(repo)) + assert result.returncode == 2 + assert "does not say which commit it is" in result.stderr + + def test_an_explicit_limit_is_honoured(self, repo: Path, base_commit: str, tmp_path: Path) -> None: + binary = _binary(tmp_path, base_commit) + result = self._run( + str(binary), "--repo", str(repo), "--max-behind", str(STALE_DISTANCE) + ) + assert result.returncode == 0, result.stderr + + def test_an_explicit_reference_pins_equality(self, repo: Path, tmp_path: Path) -> None: + """`--reference --max-behind 0` is how build_sut.sh checks its own output.""" + head = _git(repo, "rev-parse", "origin/main") + binary = _binary(tmp_path, head) + assert ( + self._run( + str(binary), + "--repo", + str(repo), + "--reference", + head, + "--max-behind", + "0", + ).returncode + == 0 + ) + stale = _binary(tmp_path / "other", _git(repo, "rev-parse", "origin/main~1")) + assert ( + self._run( + str(stale), + "--repo", + str(repo), + "--reference", + head, + "--max-behind", + "0", + ).returncode + == 1 + ) + + def test_json_carries_the_numbers(self, repo: Path, base_commit: str, tmp_path: Path) -> None: + import json + + binary = _binary(tmp_path, base_commit) + result = self._run(str(binary), "--repo", str(repo), "--json") + payload = json.loads(result.stdout) + assert payload["commit"] == base_commit + assert payload["distance"]["behind"] == STALE_DISTANCE + assert payload["fresh"] is False + + +class TestWiring: + """The guard is only a guard if the runbook runs it.""" + + def test_preflight_calls_it(self) -> None: + text = _ENV_SH.read_text() + assert "assert_fresh_sut" in text + preflight = text.split("preflight() {", 1)[1] + assert "assert_fresh_sut || return 1" in preflight + + def test_the_limit_is_not_restated_in_shell(self) -> None: + """One copy of the number, so the two cannot drift apart. + + Asserted as "preflight passes no limit" rather than "the digits do not + appear": `env.sh` is full of incidental digits (the dataset digest alone + contains most two-digit numbers), and a substring search over it would + be a test of the fixture's luck. + """ + preflight = _ENV_SH.read_text().split("preflight() {", 1)[1] + assert "--max-behind" not in preflight + + def test_build_sut_verifies_its_own_output(self) -> None: + assert "assert_fresh_sut" in _BUILD_SH.read_text() + + def test_build_sut_accepts_an_explicit_commit(self) -> None: + """The wrapper that fetches and checks out must be able to name the commit. + + Without it the script re-fetches and can resolve a *different* + `origin/main` than the caller prepared against, whose symptom is a drift + list of files nobody touched. + """ + text = _BUILD_SH.read_text() + assert 'if [ "$#" -gt 0 ]; then' in text + assert "moved while this script ran" in text + + def test_runs_without_importing_the_adapter_package(self) -> None: + """`env.sh` invokes this by path on a host that has no `harbor` installed.""" + result = subprocess.run( + [sys.executable, "-I", str(_CHECKER), "--help"], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr From 0be8405808f568f58d645d4466c6880de79c3913 Mon Sep 17 00:00:00 2001 From: Mac Anderson Date: Thu, 6 Aug 2026 22:19:06 -0700 Subject: [PATCH 3/7] fix(bench): register freshness.py in the frozen adapter tree --- bench/harbor_adapter/stella_harbor/secure_launcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/harbor_adapter/stella_harbor/secure_launcher.py b/bench/harbor_adapter/stella_harbor/secure_launcher.py index f784d9a04..21abd77c7 100644 --- a/bench/harbor_adapter/stella_harbor/secure_launcher.py +++ b/bench/harbor_adapter/stella_harbor/secure_launcher.py @@ -76,6 +76,7 @@ "bench/harbor_adapter/stella_harbor/credential_bundle.py", "bench/harbor_adapter/stella_harbor/exit_cause.py", "bench/harbor_adapter/stella_harbor/exit_status.py", + "bench/harbor_adapter/stella_harbor/freshness.py", "bench/harbor_adapter/stella_harbor/git_baseline.py", "bench/harbor_adapter/stella_harbor/host_attestation.py", "bench/harbor_adapter/stella_harbor/live_feed.py", From 68eb50ea77d6c62659f40efb1e25e1bd1ce9f941 Mon Sep 17 00:00:00 2001 From: Mac Anderson Date: Thu, 6 Aug 2026 22:21:07 -0700 Subject: [PATCH 4/7] docs(bench,arenabench): document the freshness guard and the build_sut race --- arenabench/README.md | 17 +++++++++++++++++ bench/RUNBOOK.md | 10 ++++++++++ bench/evidence/run/README.md | 37 +++++++++++++++++++++++++++++++++++- bench/evidence/run/env.sh | 2 ++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/arenabench/README.md b/arenabench/README.md index 0f3b2edee..86e1d169a 100644 --- a/arenabench/README.md +++ b/arenabench/README.md @@ -321,6 +321,23 @@ only. A native macOS build will be uploaded happily and then fail to exec inside the container, which Harbor records as an agent crash rather than a build mistake. +`STELLA_BINARY` is only the fallback for a match with **no SUT pin**. A pinned +match resolves its `sut_ref` to a commit and runs the binary built from that +commit, ignoring this variable entirely — which is how a result records which +Stella produced it. + +Clearing the pin opts out of *pinning*, not out of every check. An unpinned +match asks for whatever is current, so the arena reads the commit stamped into +`STELLA_BINARY` itself and refuses to launch when that commit is more than +`MAX_BEHIND_UNPINNED` commits behind `origin/main`. This is not hypothetical: +the path the launch scripts name sat 291 commits behind `main` for three days, +carrying a `sut_commit.txt` that correctly named its own ancient commit, and +every match against it produced perfectly scoreable numbers attributed to code +that had been rewritten underneath them. To measure older code deliberately, +pin `sut_ref` to it — pinning is what puts the answer in the result. An +ordinary `cargo build --release` carries no commit stamp, cannot be dated, and +is never blocked; those runs stay labelled unverified as before. + --- ## Recording diff --git a/bench/RUNBOOK.md b/bench/RUNBOOK.md index 9d4a23fe4..1308d0027 100644 --- a/bench/RUNBOOK.md +++ b/bench/RUNBOOK.md @@ -78,6 +78,12 @@ export STELLA_BINARY="$claim_repo/target/x86_64-unknown-linux-gnu/release/stella export STELLA_SOURCE_COMMIT="$claim_sha" file "$STELLA_BINARY" # ELF ... x86-64 ... GNU/Linux 2.0.0 sha256sum "$STELLA_BINARY" # record this — the run manifest freezes it + +# Which commit is this, really, and how far is it from what the number will be +# read as measuring? Answered from the binary's own compile-time bytes, so a +# path repointed by hand or a refreshed sut_commit.txt cannot make it look new. +python3 "$claim_repo/bench/harbor_adapter/stella_harbor/freshness.py" \ + "$STELLA_BINARY" --repo "$claim_repo" ``` > The binary SHA is host-specific (it embeds the builder's rustup/cargo paths) — @@ -220,6 +226,10 @@ test -d "$claim_jobs" && test ! -L "$claim_jobs" test "$(command -v harbor)" = "$claim_venv/bin/harbor" test "$(harbor --version)" = 0.6.1 test -x "$STELLA_BINARY"; test "${#STELLA_SOURCE_COMMIT}" = 40 +# Pinned is not the same as current: on 2026-08-07 the path the launch scripts +# name was 291 commits behind main and carried a sut_commit.txt saying so. +python3 "$claim_repo/bench/harbor_adapter/stella_harbor/freshness.py" \ + "$STELLA_BINARY" --repo "$claim_repo" umask 077 claim_adapter_root="$claim_repo/bench/harbor_adapter" claim_site_root="$claim_venv/lib/python3.13/site-packages" diff --git a/bench/evidence/run/README.md b/bench/evidence/run/README.md index e3717c120..c546cd93d 100644 --- a/bench/evidence/run/README.md +++ b/bench/evidence/run/README.md @@ -28,7 +28,13 @@ export TB_REPO="$(git rev-parse --show-toplevel)" # 1. Freeze and build the SUT. Refuses if anything that feeds the compiler # differs from origin/main, so the binary's provenance is checkable. -bench/evidence/run/build_sut.sh +# Pass a commit to build exactly that revision and skip the fetch — do this +# from any wrapper that already fetched and checked out, because otherwise +# this script re-fetches and can resolve a *different* origin/main than the +# one you prepared against. That race's symptom is a drift list naming files +# nobody touched; unargued, the script now detects it and says so rather +# than sending you hunting for local edits that do not exist. +bench/evidence/run/build_sut.sh [] # 2. Fetch the pinned 89-task dataset and classify tasks by resource class. bench/evidence/run/fetch_dataset.sh @@ -109,6 +115,35 @@ To inspect a binary directly: python3 bench/harbor_adapter/stella_harbor/portability.py "$STELLA_BINARY" --json ``` +### The binary ran, and it was the wrong code + +Portability asks "can this run"; the SHA-256 comparison asks "did it arrive +intact"; `STELLA_SOURCE_COMMIT` asks "which commit is it". None of them asks +whether that commit is anywhere near the one a reader will assume was measured. +On 2026-08-07 the path every launch script names as `STELLA_BINARY` was **291 +commits and three days behind `origin/main`** — and it carried its own +`sut_commit.txt` naming that ancient commit, so it looked pinned and passed +everything. Pinning and freshness are different properties. + +`preflight` now also refuses a binary too far behind the reference. The commit +is read from the artifact's own compile-time bytes — `build.rs` stamps +`STELLA_BUILD_GIT_SHA` into a NUL-delimited literal precisely so the identity +can be recovered without executing a cross-compiled binary — so a repointed +path or a hand-refreshed sidecar cannot make an old build look new, and a +sidecar that disagrees with the binary is itself a refusal. + +```bash +python3 bench/harbor_adapter/stella_harbor/freshness.py "$STELLA_BINARY" --json +``` + +Exit status is `0` fresh, `1` stale, `2` undeterminable — and preflight treats +every nonzero status as fatal, because a binary not shown to be near the +reference has not been shown to be near it. `--max-behind` measures older code +on purpose, which puts the distance in the launch command instead of a default +nobody reads. `build_sut.sh` uses the same checker against the commit it just +resolved (`--max-behind 0`) to assert the artifact carries the stamp it asked +for. + ### Is a musl build needed as well? No. glibc 2.17 fixes containers with *older glibc*. It does nothing for a container diff --git a/bench/evidence/run/env.sh b/bench/evidence/run/env.sh index b105efe6d..fe57451ad 100755 --- a/bench/evidence/run/env.sh +++ b/bench/evidence/run/env.sh @@ -97,6 +97,8 @@ assert_portable_binary() { # copy cannot drift from the other. Pass --max-behind to measure older code on # purpose, which puts the distance in the launch command rather than in a # default nobody reads. +# shellcheck disable=SC2120 # build_sut.sh calls this with arguments; shellcheck +# cannot see across the `source` boundary and so reads it as never-parameterised. assert_fresh_sut() { local binary="${1:-$STELLA_BINARY}" [ "$#" -gt 0 ] && shift From 9445eddb3cf39131b05db2b5540fd0d832c84e04 Mon Sep 17 00:00:00 2001 From: Mac Anderson Date: Thu, 6 Aug 2026 22:22:42 -0700 Subject: [PATCH 5/7] chore(bench): regenerate file-size baseline for the adapter tree registration --- scripts/file-size-baseline.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/file-size-baseline.txt b/scripts/file-size-baseline.txt index 9dca27e46..a3a79337a 100644 --- a/scripts/file-size-baseline.txt +++ b/scripts/file-size-baseline.txt @@ -8,7 +8,7 @@ # never silent — it lands as a visible diff here, to be justified in # review like any other change. 2199 bench/harbor_adapter/stella_harbor/__init__.py -4293 bench/harbor_adapter/stella_harbor/secure_launcher.py +4294 bench/harbor_adapter/stella_harbor/secure_launcher.py 2335 bench/harbor_adapter/tests/test_adapter.py 3360 bench/harbor_adapter/tests/test_secure_launcher.py 8272 bench/terminal_bench_analysis/tb21_analysis.py From 038b40a9d5efeebf029c23224457dee44567a5b1 Mon Sep 17 00:00:00 2001 From: Mac Anderson Date: Thu, 6 Aug 2026 22:27:08 -0700 Subject: [PATCH 6/7] fix(bench,arenabench): refuse a SUT binary too far behind the code it is reported as measuring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pinning and freshness are different properties, and only the first was checked. `~/.arenabench/sut/stella` — the path every runbook exports as STELLA_BINARY — sat 291 commits and three days behind origin/main while carrying its own sut_commit.txt naming that ancient commit, so it looked pinned, passed everything #2016/#2020 added, and produced perfectly scoreable trials attributed to code rewritten underneath them. The new guard reads the commit out of the artifact's own compile-time bytes rather than the sidecar beside it: build.rs stamps STELLA_BUILD_GIT_SHA into a NUL-delimited literal precisely so the identity can be recovered without executing a cross-compiled binary. A repointed path or a hand-refreshed sidecar therefore cannot make an old build look new, and a sidecar disagreeing with the binary is itself a refusal. Also names the build_sut.sh fetch race reported on the issue: the script recorded origin/main only after its own fetch, so a branch that moved mid-run produced a drift list of files nobody touched, reading as local contamination. It now takes an optional commit argument, and when it fetches it compares against where origin/main stood beforehand so the refusal can say which of the two causes it is. Closes #2032 Refs #2049, #2050, #2051 --- arenabench/arenabench/runner.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/arenabench/arenabench/runner.py b/arenabench/arenabench/runner.py index cc5642071..b2dfdd084 100644 --- a/arenabench/arenabench/runner.py +++ b/arenabench/arenabench/runner.py @@ -880,11 +880,26 @@ def _agent_environment( binary = os.environ.get("STELLA_BINARY") if binary: env["STELLA_BINARY"] = binary - run.warnings.append( - "SUT not pinned to a commit — running whatever STELLA_BINARY " - "points at, so these numbers cannot be attributed to a " - "specific Stella revision" - ) + # `create_match` has already refused an unpinned seat whose binary + # is measurably stale, so anything reaching here is either close to + # `main` or carries no commit stamp at all. Say which: "cannot be + # attributed" is true of the second and false of the first, and a + # warning that overstates its case is one operators learn to skip. + ambient = sut.ambient_sut() + if ambient is not None and ambient.commit: + run.warnings.append( + f"SUT not pinned — running {ambient.path}, built from " + f"{ambient.commit[:8]}. The commit is recorded here but " + "not in the match, so pin the SUT to make the result " + "self-describing" + ) + else: + run.warnings.append( + "SUT not pinned to a commit — running whatever " + "STELLA_BINARY points at, and that binary carries no " + "compile-time commit stamp, so these numbers cannot be " + "attributed to a specific Stella revision" + ) return env # -- supervision ------------------------------------------------------ From 889bde00f972d7585cc6ba82c5476da1501f6682 Mon Sep 17 00:00:00 2001 From: Mac Anderson Date: Thu, 6 Aug 2026 22:31:09 -0700 Subject: [PATCH 7/7] fix(bench): validate both halves of the freshness rev-range before git sees them --- bench/harbor_adapter/stella_harbor/freshness.py | 9 +++++++++ bench/harbor_adapter/tests/test_freshness.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/bench/harbor_adapter/stella_harbor/freshness.py b/bench/harbor_adapter/stella_harbor/freshness.py index 494b0f9ca..76a82c5ec 100644 --- a/bench/harbor_adapter/stella_harbor/freshness.py +++ b/bench/harbor_adapter/stella_harbor/freshness.py @@ -357,6 +357,15 @@ def measure_distance( from the reference *as this checkout knows it*, which is stated in the verdict so a stale checkout cannot masquerade as a fresh binary. """ + # Both halves of the rev-range are validated before either reaches git. + # `commit` always arrives from `read_identity` in practice, which can only + # produce a full hex SHA — but this is public API interpolated into a + # `A...B` argument, and a value beginning with `-` would be read by git as + # an option rather than as a revision. + if not _FULL_SHA.match((commit or "").lower()): + raise ReferenceUnavailableError( + f"refusing to measure from {commit!r}: not a full 40-hex commit" + ) checkout = Path(repo) if not (checkout / ".git").exists(): raise ReferenceUnavailableError(f"{checkout} is not a git checkout") diff --git a/bench/harbor_adapter/tests/test_freshness.py b/bench/harbor_adapter/tests/test_freshness.py index 18f93e51b..f6a6af208 100644 --- a/bench/harbor_adapter/tests/test_freshness.py +++ b/bench/harbor_adapter/tests/test_freshness.py @@ -258,6 +258,20 @@ def test_an_option_shaped_ref_never_reaches_git(self, repo: Path) -> None: with pytest.raises(SutFreshnessError): measure_distance("a" * 40, repo, "--upload-pack=touch /tmp/pwned") + def test_an_option_shaped_commit_never_reaches_git(self, repo: Path) -> None: + """Both halves of the `A...B` range are validated, not just the ref. + + In practice `commit` only ever arrives from `read_identity`, which can + produce nothing but a full hex SHA — but this is public API, and a + value starting with `-` lands in an argument git reads as an option. + """ + with pytest.raises(SutFreshnessError): + measure_distance("--upload-pack=touch /tmp/pwned", repo) + + def test_a_short_commit_is_refused(self, repo: Path) -> None: + with pytest.raises(SutFreshnessError): + measure_distance(_git(repo, "rev-parse", "--short", "origin/main"), repo) + class TestVerdict: """What is refused, and what the refusal says."""