From e34e05ea921b2e5f775d3b312c8d4b5a8cbb3e7e Mon Sep 17 00:00:00 2001 From: kraxo Date: Thu, 23 Jul 2026 15:29:51 +0200 Subject: [PATCH] ci: fail-closed guard against left-over mutation mutants Incident 2026-07-23: a mutation probe's `if False:` at the SD-JWT key-binding check in bundle.py survived into the working tree (plus a stray .mutbak file); only a manual diff-before-commit caught it. try/finally restore cannot survive a hard kill. Three mechanical layers close the class: 1. mutation_check.py v1.4 isolation: mutants are applied in a throwaway tempdir copy of the tracked tree (git ls-files), the real working tree is never written to, so even SIGKILL mid-probe cannot leave a mutant or backup litter behind. The .mutbak mechanism is gone; restore is in-memory, inside the copy. Belt and braces, the run compares `git status --porcelain` before/after and fails closed on any left-over working tree change. 2. CI clean-assert: the mutation job ends with an always-run step that fails closed if `git status --porcelain` is non-empty (the machine version of the manual diff that caught the incident). 3. mutant_signature_guard.py: a narrow, explainable diff guard (pre-commit hook via scripts/install_git_hooks.sh AND a new blocking CI job, so the hook cannot be bypassed locally) over added lines on src/proofbundle/**: A) trivial-truth branches (if/elif False|True, while False) at a check, B) commented-out verification lines (two-stage: verify-call prefilter, then the content must PARSE as a Python statement; the three false-positive prose shapes found on real v3.6.0..HEAD history are pinned as negative self-test cases), C) `return True` opening a verify/validate/check function. Legitimate exceptions need a visible `# mutant-guard: allow` comment in the diff. --self-test proves each class is caught before a clean verdict is trusted; the guard is quiet over the full v3.6.0..HEAD history. Verified locally: 17 new unittest cases (staged + base + isolation contract), self-test 11/11, SIGKILL mid-run leaves the tree byte-identical, the staged incident replay (`if False:` at bundle.py:425) is blocked by the installed pre-commit hook, ruff clean. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 32 +++ scripts/git-hooks/pre-commit | 9 + scripts/install_git_hooks.sh | 17 ++ scripts/mutant_signature_guard.py | 306 +++++++++++++++++++++++++++ scripts/mutation_check.py | 80 +++++-- tests/test_mutant_signature_guard.py | 130 ++++++++++++ tests/test_mutation_isolation.py | 112 ++++++++++ 7 files changed, 673 insertions(+), 13 deletions(-) create mode 100755 scripts/git-hooks/pre-commit create mode 100755 scripts/install_git_hooks.sh create mode 100644 scripts/mutant_signature_guard.py create mode 100644 tests/test_mutant_signature_guard.py create mode 100644 tests/test_mutation_isolation.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddeeef0..523e9ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,6 +186,38 @@ jobs: run: python -m pip install -e ".[dev,pq,anchors]" - name: Mutation check run: python scripts/mutation_check.py + - name: Fail closed on left-over working tree changes (mutant leftovers) + # Machine version of the manual diff-before-commit that caught the 2026-07-23 live + # mutant. v1.4 mutates only a tempdir copy, so ANY working tree change after the run + # is a leftover and fails the job, even when the mutation step itself already failed. + if: always() + run: | + leftovers="$(git status --porcelain)" + if [ -n "$leftovers" ]; then + echo "::error::left-over working tree change after probe run" + echo "$leftovers" + exit 1 + fi + echo "working tree clean after the mutation run" + + mutant-signature-guard: + # Fail-closed guard against mutation-mutant signatures reaching main (incident 2026-07-23: + # a probe's `if False:` planted at the SD-JWT key-binding check survived into the working + # tree). Diff-scoped and deliberately narrow (scripts/mutant_signature_guard.py); the + # self-test step proves each signature class is actually caught BEFORE the clean verdict + # of the range scan is trusted. The same guard runs locally as a pre-commit hook + # (scripts/install_git_hooks.sh), this job is the non-bypassable half. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: Guard self-test (each planted signature class must be caught) + run: python3 scripts/mutant_signature_guard.py --self-test + - name: Scan the change range for mutant signatures + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + run: python3 scripts/mutant_signature_guard.py --base "$BASE_SHA" anchors: # EXPERIMENTAL [anchors] extra (v2.0 preview): exercises the RFC 3161 TSA verifier against a real diff --git a/scripts/git-hooks/pre-commit b/scripts/git-hooks/pre-commit new file mode 100755 index 0000000..2b95ebe --- /dev/null +++ b/scripts/git-hooks/pre-commit @@ -0,0 +1,9 @@ +#!/bin/sh +# proofbundle pre-commit hook: block mutation-mutant signatures on staged security paths +# (incident 2026-07-23: a probe's `if False:` at the SD-JWT key-binding check survived into +# the working tree). Install with: sh scripts/install_git_hooks.sh +# +# A legitimate exception needs a VISIBLE `# mutant-guard: allow` comment on (or directly +# above) the flagged line. CI runs the same guard on the push/PR range, so skipping this +# hook locally does not skip the check. +exec python3 "$(git rev-parse --show-toplevel)/scripts/mutant_signature_guard.py" --staged diff --git a/scripts/install_git_hooks.sh b/scripts/install_git_hooks.sh new file mode 100755 index 0000000..3b4638d --- /dev/null +++ b/scripts/install_git_hooks.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Install the repo's committed git hooks into .git/hooks (currently: pre-commit mutant guard). +# Refuses to overwrite a foreign pre-existing hook unless called with --force. +set -eu +top="$(git rev-parse --show-toplevel)" +src="$top/scripts/git-hooks/pre-commit" +# --git-path resolves correctly for worktrees (where .git is a file) and core.hooksPath setups. +hooks_dir="$(git rev-parse --git-path hooks)" +mkdir -p "$hooks_dir" +dst="$hooks_dir/pre-commit" +if [ -e "$dst" ] && [ "${1:-}" != "--force" ] && ! grep -q "mutant_signature_guard" "$dst"; then + echo "install_git_hooks: $dst exists and is not ours; re-run with --force to replace" >&2 + exit 1 +fi +cp "$src" "$dst" +chmod +x "$dst" +echo "install_git_hooks: pre-commit mutant guard installed at $dst" diff --git a/scripts/mutant_signature_guard.py b/scripts/mutant_signature_guard.py new file mode 100644 index 0000000..62c53e5 --- /dev/null +++ b/scripts/mutant_signature_guard.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Fail-closed guard against mutation-mutant signatures on security paths (incident 2026-07-23). + +A mutation probe planted `if False:` in place of the SD-JWT key-binding check in bundle.py and +the mutant survived into the working tree; only a manual diff-before-commit caught it. This guard +is the mechanical version of that manual look: it scans the CHANGE (staged diff or a commit +range) for the three narrow signature classes a left-over mutant takes, and blocks fail-closed. + +Signature classes (deliberately narrow and explainable: a safety net, not a linter): + + A a trivial-truth branch added at a check site: `if False:` / `if True:` / + `elif False:` / `elif True:` / `while False:` (also `if False and :`) + B a commented-out verification line: a comment whose content reads like a code statement + calling a verify/validate/check/compare_digest function (prose comments do not match) + C `return True` as the first statement of a function whose name says verify/validate/check + +Scope: added lines under src/proofbundle/**/*.py (the verification library, every path there is +security-relevant). Legitimate exceptions are possible but must be VISIBLE in the diff: put a +`# mutant-guard: allow` comment on the flagged line or the line directly above it. + +Modes: + --staged scan the staged diff (pre-commit hook; content read from the index) + --base scan ..HEAD (CI; all-zero / missing sha falls back + to HEAD~1, and to an empty scan on a root commit) + --self-test prove in a throwaway git repo that every class is caught and that the + negative controls stay quiet (the gate-meta-test; CI runs this first) + +Exit codes: 0 clean · 1 mutant signature found · 2 internal/usage error (fail closed). +stdlib only, offline. +""" +from __future__ import annotations + +import argparse +import ast +import re +import subprocess +import tempfile +from pathlib import Path + + +def _repo_root() -> Path: + """The repo the guard runs IN (cwd-based): the pre-commit hook and CI both execute at + the checkout root; anchoring on the script location would scan the wrong repo when + invoked from elsewhere (e.g. the test fixtures).""" + proc = subprocess.run(["git", "rev-parse", "--show-toplevel"], + capture_output=True, text=True) + if proc.returncode != 0 or not proc.stdout.strip(): + raise SystemExit("mutant_signature_guard: not inside a git repository (fail closed)") + return Path(proc.stdout.strip()) + + +_SECURITY_PATH = re.compile(r"^src/proofbundle/.*\.py$") +_ALLOW_MARKER = "mutant-guard: allow" + +# Class A — trivial-truth branch (word-boundary keeps `if Falsey_thing` out). +_TRIVIAL_TRUTH = re.compile(r"^\s*(?:(?:el)?if\s+(?:False|True)\b|while\s+False\b)") + +# Class B — commented-out verification CODE, two-stage: a cheap prefilter (a comment whose +# content starts like a statement calling a verify/validate/check/compare_digest function), +# then the decisive test: the content must PARSE as a Python statement. Prose that merely +# names a function keeps trailing English words and fails to parse (`# verify_envelope +# (docstring says ...) never gets ...`), commented-out code parses (`# ok = +# hmac.compare_digest(a, b)`). The three false-positive shapes found on real 3.6.0..HEAD +# history are pinned as negative self-test cases below. +_COMMENTED_VERIFY = re.compile( + r"^\s*#\s*(?:if\s+|elif\s+|return\s+|assert\s+|not\s+)?(?:[\w.]+\s*=\s*)?" + r"[\w.]*(?:verify|validate|compare_digest|check)[\w.]*\s*\(") + + +def _commented_content_parses(text: str) -> bool: + content = re.sub(r"^\s*#\s?", "", text).strip() + if content.endswith(":"): + content += "\n pass" # a commented-out `if verify(x):` header needs a body to parse + try: + ast.parse(content) + except SyntaxError: + return False + return True + +# Class C — verification-function names. +_VERIFYISH_NAME = re.compile(r"(?:verify|validate|check)", re.IGNORECASE) + + +def _git(*args: str, cwd: Path) -> str: + proc = subprocess.run(["git", "-C", str(cwd), *args], + capture_output=True, text=True) + if proc.returncode != 0: + raise SystemExit(f"mutant_signature_guard: git {' '.join(args[:2])} failed (fail closed): " + f"{proc.stderr.strip()[:200]}") + return proc.stdout + + +def _added_lines_by_file(diff_text: str) -> dict[str, list[tuple[int, str]]]: + """Parse a -U0 unified diff into {new_path: [(new_lineno, added_line_text), ...]}.""" + out: dict[str, list[tuple[int, str]]] = {} + current: str | None = None + lineno = 0 + for raw in diff_text.splitlines(): + if raw.startswith("+++ "): + path = raw[4:].strip() + current = None if path == "/dev/null" else path.removeprefix("b/") + elif raw.startswith("@@"): + m = re.search(r"\+(\d+)", raw) + lineno = int(m.group(1)) if m else 0 + elif raw.startswith("+") and not raw.startswith("+++"): + if current is not None: + out.setdefault(current, []).append((lineno, raw[1:])) + lineno += 1 + elif raw.startswith(" "): + lineno += 1 + return out + + +def _allowlisted(file_lines: list[str], lineno: int) -> bool: + for candidate in (lineno, lineno - 1): + if 1 <= candidate <= len(file_lines) and _ALLOW_MARKER in file_lines[candidate - 1]: + return True + return False + + +def _class_c_findings(content: str, added: set[int]) -> list[tuple[int, str]]: + """`return True` as first non-docstring statement of a verify-ish function, if the def or + the return line is part of the change (pre-existing code is out of scope for a diff guard).""" + findings: list[tuple[int, str]] = [] + try: + tree = ast.parse(content) + except SyntaxError: + return [] # unparseable staged state: the test/lint gates own that failure + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + if not _VERIFYISH_NAME.search(node.name): + continue + body = node.body + if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) \ + and isinstance(body[0].value.value, str): + body = body[1:] # skip the docstring + if not body or not isinstance(body[0], ast.Return): + continue + val = body[0].value + if isinstance(val, ast.Constant) and val.value is True: + if node.lineno in added or body[0].lineno in added: + findings.append((body[0].lineno, + f"`return True` opens verification function `{node.name}`")) + return findings + + +def _file_content(path: str, *, staged: bool, cwd: Path) -> str: + if staged: + return _git("show", f":{path}", cwd=cwd) + p = cwd / path + return p.read_text(encoding="utf-8") if p.is_file() else "" + + +def scan(diff_text: str, *, staged: bool, cwd: Path) -> list[str]: + findings: list[str] = [] + for path, added in _added_lines_by_file(diff_text).items(): + if not _SECURITY_PATH.match(path): + continue + content = _file_content(path, staged=staged, cwd=cwd) + file_lines = content.splitlines() + added_nums = {n for n, _ in added} + for lineno, text in added: + reason = None + if _TRIVIAL_TRUTH.match(text): + reason = "trivial-truth branch (`if/elif False|True` / `while False`) at a check" + elif _COMMENTED_VERIFY.match(text) and _commented_content_parses(text): + reason = "commented-out verification call" + if reason and not _allowlisted(file_lines, lineno): + findings.append(f"{path}:{lineno}: {reason}\n {text.strip()}") + for lineno, reason in _class_c_findings(content, added_nums): + if not _allowlisted(file_lines, lineno): + findings.append(f"{path}:{lineno}: {reason}") + return findings + + +def _resolve_base(base: str, cwd: Path) -> str | None: + """Turn the CI-provided base sha into a usable merge base; honest fallbacks, never a crash.""" + if not base or set(base) == {"0"}: + base = "HEAD~1" + probe = subprocess.run(["git", "-C", str(cwd), "rev-parse", "--verify", f"{base}^{{commit}}"], + capture_output=True, text=True) + if probe.returncode != 0: + return None + mb = subprocess.run(["git", "-C", str(cwd), "merge-base", base, "HEAD"], + capture_output=True, text=True) + return mb.stdout.strip() if mb.returncode == 0 and mb.stdout.strip() else None + + +def run_staged(cwd: Path) -> list[str]: + diff = _git("diff", "--cached", "-U0", "--no-color", "--", "src/proofbundle", cwd=cwd) + return scan(diff, staged=True, cwd=cwd) + + +def run_base(base: str, cwd: Path) -> list[str]: + resolved = _resolve_base(base, cwd) + if resolved is None: + print("mutant_signature_guard: no usable base commit (root commit / unknown sha) — " + "nothing to diff, scan skipped honestly") + return [] + diff = _git("diff", "-U0", "--no-color", resolved, "HEAD", "--", "src/proofbundle", cwd=cwd) + return scan(diff, staged=False, cwd=cwd) + + +# --- self-test (gate-meta-test: prove each class is caught, and the negatives stay quiet) ----- + +_BENIGN = '''def verify_thing(data): + """Real check.""" + if not isinstance(data, dict): + return False + return bool(data.get("ok")) + + +def helper(x): + # verify the payload first, then compare digests (prose comment, must NOT match) + return x +''' + +_CASES: list[tuple[str, str, bool]] = [ + # (label, replacement content for src/proofbundle/guarded.py, expect_finding) + ("A: if False at a check", + _BENIGN.replace('if not isinstance(data, dict):', 'if False:'), True), + ("A: if True and original check", + _BENIGN.replace('if not isinstance(data, dict):', 'if True and data:'), True), + ("B: commented-out verification call", + _BENIGN.replace(' return bool(data.get("ok"))', + ' # ok = hmac.compare_digest(a, b)\n return True'), True), + ("C: return True opens a verify function", + 'def verify_thing(data):\n return True\n', True), + ("B: commented-out if-header of a check", + _BENIGN.replace('if not isinstance(data, dict):', + '# if _validate_shape(data):\n if data is None:'), True), + ("negative: prose naming a function before a parenthetical (real FP shape, dsse.py)", + _BENIGN + '\n# verify_thing (docstring says only ValueError) never gets a raw error.\n', False), + ("negative: function name with parens inside prose (real FP shape, outcome.py)", + _BENIGN + '\n# verify_thing() is the explicit exception variant.\n', False), + ("negative: code-then-prose narrative (real FP shape, signature.py)", + _BENIGN + '\n# pub.verify(sig, data) and raised a raw TypeError nobody caught.\n', False), + ("negative: benign refactor stays quiet", + _BENIGN.replace('bool(data.get("ok"))', 'bool(data.get("okay"))'), False), + ("negative: allowlist marker suppresses, visibly", + _BENIGN.replace('if not isinstance(data, dict):', + 'if True: # mutant-guard: allow (fixture, reviewed)'), False), +] + + +def self_test() -> int: + failures = 0 + with tempfile.TemporaryDirectory(prefix="mutant-guard-selftest-") as tmp: + repo = Path(tmp) + _git("init", "-q", cwd=repo) + _git("config", "user.email", "guard@selftest.local", cwd=repo) + _git("config", "user.name", "guard-selftest", cwd=repo) + target = repo / "src" / "proofbundle" / "guarded.py" + target.parent.mkdir(parents=True) + target.write_text(_BENIGN, encoding="utf-8") + outside = repo / "scripts" / "not_security.py" + outside.parent.mkdir(parents=True) + outside.write_text("x = 1\n", encoding="utf-8") + _git("add", "-A", cwd=repo) + _git("commit", "-q", "-m", "base", cwd=repo) + for label, content, expect in _CASES: + target.write_text(content, encoding="utf-8") + _git("add", "-A", cwd=repo) + found = bool(run_staged(repo)) + ok = found == expect + print(f" {'ok ' if ok else 'FAIL'} [{label}] " + f"{'caught' if found else 'quiet'} ({'expected' if ok else 'UNEXPECTED'})") + failures += 0 if ok else 1 + _git("checkout", "-q", "--", ".", cwd=repo) + _git("reset", "-q", cwd=repo) + # negative: the same planted signature OUTSIDE the security path stays quiet + outside.write_text("if False:\n pass\n", encoding="utf-8") + _git("add", "-A", cwd=repo) + quiet = not run_staged(repo) + print(f" {'ok ' if quiet else 'FAIL'} [negative: non-security path ignored] " + f"{'quiet' if quiet else 'caught'} ({'expected' if quiet else 'UNEXPECTED'})") + failures += 0 if quiet else 1 + print(f"self-test: {'OK' if failures == 0 else f'FAILED ({failures})'}") + return 0 if failures == 0 else 1 + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + mode = p.add_mutually_exclusive_group(required=True) + mode.add_argument("--staged", action="store_true", help="scan the staged diff (pre-commit)") + mode.add_argument("--base", metavar="SHA", help="scan merge-base(SHA, HEAD)..HEAD (CI)") + mode.add_argument("--self-test", action="store_true", help="prove the guard catches each class") + a = p.parse_args(argv) + if a.self_test: + return self_test() + repo = _repo_root() + findings = run_staged(repo) if a.staged else run_base(a.base, repo) + if findings: + print("mutant_signature_guard: BLOCKED: mutation-mutant signature(s) on security paths:") + for f in findings: + print(f" {f}") + print("If this is intentional and legitimate, add a visible `# mutant-guard: allow` " + "comment on (or directly above) the flagged line so review sees the exception.") + return 1 + print("mutant_signature_guard: clean, no mutant signatures in the scanned change") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mutation_check.py b/scripts/mutation_check.py index a0e7406..889134f 100644 --- a/scripts/mutation_check.py +++ b/scripts/mutation_check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Orthogonal mutation suite — proves the tests still KILL broken implementations (v1.3). +"""Orthogonal mutation suite — proves the tests still KILL broken implementations (v1.4). Anti-Goodhart guard: a green test suite only means something if it goes red when the code is broken. Each operator below mutates ONE independent fault dimension (binding, framing, key @@ -10,6 +10,13 @@ Documented-equivalent mutants are asserted to SURVIVE — if one starts getting killed, the equivalence argument is stale and must be revisited (that is a failure too: honesty both ways). +Isolation contract (v1.4, incident 2026-07-23): mutants are applied in a THROWAWAY COPY of the +tracked tree under a tempdir; the real working tree is never written to, so even a SIGKILL +mid-probe cannot leave a live mutant (or a stray backup file) behind. try/finally alone cannot +give that guarantee: a hard kill skips finally, and exactly that left an `if False:` mutant plus +a .mutbak file in the working tree. Belt and braces, the run additionally compares +`git status --porcelain` before/after and fails closed on any left-over working tree change. + Usage: python3 scripts/mutation_check.py # exit 0 = all as expected, 1 = gap found CI: runs in the mutation job (see .github/workflows/ci.yml). """ @@ -19,6 +26,7 @@ import shutil import subprocess import sys +import tempfile from pathlib import Path ROOT = Path(__file__).resolve().parent.parent @@ -430,7 +438,7 @@ ] -def _red_count() -> int: +def _red_count(work: Path) -> int: # Stale-bytecode defense (real incident during per-sample development): a same-size # mutation + coarse-mtime filesystem leaves a VALID-looking .pyc for the OLD code; -B only # stops WRITING caches — existing ones are still read; and cache dirs may be undeletable on @@ -438,13 +446,13 @@ def _red_count() -> int: # recompilation regardless of what caches survive. import os # noqa: PLC0415 import shutil # noqa: PLC0415 - for cache in ROOT.rglob("__pycache__"): + for cache in work.rglob("__pycache__"): shutil.rmtree(cache, ignore_errors=True) - for src_file in ROOT.glob("src/**/*.py"): + for src_file in work.glob("src/**/*.py"): os.utime(src_file) proc = subprocess.run( [sys.executable, "-B", "-m", "unittest", "discover", "-s", "tests"], - cwd=ROOT, capture_output=True, text=True, + cwd=work, capture_output=True, text=True, env={"PYTHONPATH": "src", "PATH": "/usr/bin:/bin:/usr/local/bin", "HOME": str(Path.home()), "PYTHONDONTWRITEBYTECODE": "1"}) f = re.search(r"failures=(\d+)", proc.stderr) @@ -452,22 +460,47 @@ def _red_count() -> int: return (int(f.group(1)) if f else 0) + (int(e.group(1)) if e else 0) -def main() -> int: - baseline = _red_count() +def _tracked_files(repo: Path) -> list[str]: + """Tracked paths (current working-tree bytes are copied, so uncommitted edits ARE tested).""" + proc = subprocess.run(["git", "-C", str(repo), "ls-files", "-z"], + capture_output=True, text=True) + if proc.returncode != 0: + raise SystemExit("mutation_check: `git ls-files` failed; the isolated work tree needs " + "a git checkout (CI and dev clones both have one)") + return [rel for rel in proc.stdout.split("\0") if rel] + + +def _prepare_workdir(repo: Path, work: Path) -> None: + """Copy every tracked file into the throwaway work tree (CI runs on exactly this file set).""" + for rel in _tracked_files(repo): + src_p = repo / rel + if not src_p.is_file(): + continue # racy delete / submodule entry, the differential baseline absorbs it + dst = work / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_p, dst) + + +def _worktree_status(repo: Path) -> str: + proc = subprocess.run(["git", "-C", str(repo), "status", "--porcelain"], + capture_output=True, text=True) + return proc.stdout if proc.returncode == 0 else "" + + +def _run_operators(work: Path) -> int: + baseline = _red_count(work) print(f"baseline red (environment-only failures allowed): {baseline}") gaps = 0 for rel, old, new, label, expect_killed in MUTATIONS: - path = ROOT / rel + path = work / rel src = path.read_text(encoding="utf-8") if old not in src: print(f" GAP [{label}] pattern not found — operator is stale") gaps += 1 continue - backup = path.with_suffix(path.suffix + ".mutbak") - shutil.copy(path, backup) try: path.write_text(src.replace(old, new, 1), encoding="utf-8") - red = _red_count() + red = _red_count(work) killed = red > baseline ok = killed == expect_killed verdict = "KILLED" if killed else "SURVIVED" @@ -476,11 +509,32 @@ def main() -> int: if not ok: gaps += 1 finally: - shutil.move(backup, path) - final = _red_count() + path.write_text(src, encoding="utf-8") # restore the pristine bytes held in memory + final = _red_count(work) if final != baseline: print(f"GAP: baseline not restored ({final} != {baseline})") gaps += 1 + return gaps + + +def main() -> int: + status_before = _worktree_status(ROOT) + with tempfile.TemporaryDirectory(prefix="proofbundle-mutation-") as tmp: + work = Path(tmp) / "tree" + _prepare_workdir(ROOT, work) + print(f"isolated work tree: {work} (the real working tree is never mutated)") + gaps = _run_operators(work) + # Fail-closed leftover check: the probe run must not have changed the REAL working tree at + # all (v1.4 isolation makes this structurally true; this assert catches any regression). + status_after = _worktree_status(ROOT) + if status_after != status_before: + print("GAP: left-over working tree change after probe run:") + before, after = set(status_before.splitlines()), set(status_after.splitlines()) + for line in sorted(after - before): + print(f" + {line}") + for line in sorted(before - after): + print(f" - {line}") + gaps += 1 print(f"=> {'OK' if gaps == 0 else 'FAILED'} ({len(MUTATIONS)} operators, {gaps} gap(s))") return 0 if gaps == 0 else 1 diff --git a/tests/test_mutant_signature_guard.py b/tests/test_mutant_signature_guard.py new file mode 100644 index 0000000..50c42eb --- /dev/null +++ b/tests/test_mutant_signature_guard.py @@ -0,0 +1,130 @@ +"""Mutant-signature guard (incident 2026-07-23): each signature class a left-over mutation +probe takes must be CAUGHT on security paths, and the negative controls must stay quiet, +in both the pre-commit (--staged) and the CI (--base) mode. Runs the real script via +subprocess against throwaway git repos, per the scripts-test convention.""" +import pathlib +import subprocess +import sys +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "mutant_signature_guard.py" + +BENIGN = '''def verify_thing(data): + """Real check.""" + if not isinstance(data, dict): + return False + return bool(data.get("ok")) +''' + + +def _git(repo, *args): + return subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, + timeout=30) + + +def _guard(repo, *args): + return subprocess.run([sys.executable, str(SCRIPT), *args], cwd=str(repo), + capture_output=True, text=True, timeout=60) + + +class _RepoFixture(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory(prefix="guard-test-") + self.repo = pathlib.Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + _git(self.repo, "init", "-q") + _git(self.repo, "config", "user.email", "t@t.local") + _git(self.repo, "config", "user.name", "t") + self.target = self.repo / "src" / "proofbundle" / "guarded.py" + self.target.parent.mkdir(parents=True) + self.target.write_text(BENIGN, encoding="utf-8") + _git(self.repo, "add", "-A") + _git(self.repo, "commit", "-q", "-m", "base") + + def _stage(self, content, path=None): + (path or self.target).write_text(content, encoding="utf-8") + _git(self.repo, "add", "-A") + + +class TestStagedMode(_RepoFixture): + def test_class_a_trivial_truth_branch_is_blocked_exit_1(self): + self._stage(BENIGN.replace("if not isinstance(data, dict):", "if False:")) + r = _guard(self.repo, "--staged") + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn("trivial-truth branch", r.stdout) + + def test_class_b_commented_out_verification_is_blocked(self): + self._stage(BENIGN.replace(' return bool(data.get("ok"))', + " # ok = hmac.compare_digest(a, b)\n return True")) + r = _guard(self.repo, "--staged") + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn("commented-out verification call", r.stdout) + + def test_class_c_return_true_verify_function_is_blocked(self): + self._stage("def verify_thing(data):\n return True\n") + r = _guard(self.repo, "--staged") + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn("return True", r.stdout) + + def test_benign_edit_stays_quiet_exit_0(self): + self._stage(BENIGN.replace('data.get("ok")', 'data.get("okay")')) + r = _guard(self.repo, "--staged") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + def test_prose_comment_mentioning_verify_stays_quiet(self): + self._stage(BENIGN.replace(' """Real check."""', + ' """Real check."""\n # verify the payload first')) + r = _guard(self.repo, "--staged") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + def test_non_security_path_is_ignored(self): + other = self.repo / "scripts" / "tool.py" + other.parent.mkdir(exist_ok=True) + self._stage("if False:\n pass\n", path=other) + r = _guard(self.repo, "--staged") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + def test_visible_allow_marker_suppresses(self): + self._stage(BENIGN.replace("if not isinstance(data, dict):", + "if True: # mutant-guard: allow (fixture, reviewed)")) + r = _guard(self.repo, "--staged") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + +class TestBaseMode(_RepoFixture): + def test_committed_mutant_in_range_is_blocked(self): + base = _git(self.repo, "rev-parse", "HEAD").stdout.strip() + self.target.write_text(BENIGN.replace("if not isinstance(data, dict):", "if False:"), + encoding="utf-8") + _git(self.repo, "add", "-A") + _git(self.repo, "commit", "-q", "-m", "mutant slips in") + r = _guard(self.repo, "--base", base) + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn("trivial-truth branch", r.stdout) + + def test_all_zero_base_falls_back_to_parent(self): + self.target.write_text(BENIGN.replace("if not isinstance(data, dict):", "if False:"), + encoding="utf-8") + _git(self.repo, "add", "-A") + _git(self.repo, "commit", "-q", "-m", "mutant slips in") + r = _guard(self.repo, "--base", "0" * 40) + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + + def test_root_commit_without_base_skips_honestly_exit_0(self): + r = _guard(self.repo, "--base", "0" * 40) # only one commit, HEAD~1 missing + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("scan skipped honestly", r.stdout) + + +class TestSelfTest(unittest.TestCase): + def test_self_test_passes(self): + r = subprocess.run([sys.executable, str(SCRIPT), "--self-test"], + capture_output=True, text=True, timeout=120) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("self-test: OK", r.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mutation_isolation.py b/tests/test_mutation_isolation.py new file mode 100644 index 0000000..0c11311 --- /dev/null +++ b/tests/test_mutation_isolation.py @@ -0,0 +1,112 @@ +"""Mutation-harness isolation contract (v1.4, incident 2026-07-23): probes mutate a throwaway +tempdir copy, the REAL working tree is never written to, and a left-over working tree change +after a run fails closed. Exercised against a mini fixture repo (a full pb mutation run takes +hours; the contract itself is what these tests pin).""" +import importlib.util +import pathlib +import subprocess +import tempfile +import unittest +from unittest import mock + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "mutation_check.py" + + +def _load_module(): + spec = importlib.util.spec_from_file_location("mutation_check_under_test", SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _git(repo, *args): + return subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, + timeout=30) + + +class _MiniRepo(unittest.TestCase): + """A tiny git repo with one guarded module, one killing test, one mutation operator.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory(prefix="mutiso-test-") + self.repo = pathlib.Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + _git(self.repo, "init", "-q") + _git(self.repo, "config", "user.email", "t@t.local") + _git(self.repo, "config", "user.name", "t") + src = self.repo / "src" / "mini" + src.mkdir(parents=True) + (src / "__init__.py").write_text("", encoding="utf-8") + (src / "core.py").write_text( + "def check(x):\n if x is None:\n return False\n return True\n", + encoding="utf-8") + tests = self.repo / "tests" + tests.mkdir() + (tests / "test_core.py").write_text( + "import unittest\nfrom mini.core import check\n\n\n" + "class T(unittest.TestCase):\n" + " def test_none_is_rejected(self):\n" + " self.assertFalse(check(None))\n", + encoding="utf-8") + _git(self.repo, "add", "-A") + _git(self.repo, "commit", "-q", "-m", "base") + self.mod = _load_module() + self.mod.ROOT = self.repo + self.mod.MUTATIONS = [ + ("src/mini/core.py", "if x is None:", "if False:", "mini: check disabled", True), + ] + + def _real_tree_digest(self): + return {p: p.read_bytes() for p in sorted(self.repo.rglob("*.py"))} + + +class TestIsolationProperty(_MiniRepo): + def test_workdir_copy_mutation_never_touches_real_tree(self): + before = self._real_tree_digest() + with tempfile.TemporaryDirectory() as tmp: + work = pathlib.Path(tmp) / "tree" + self.mod._prepare_workdir(self.repo, work) + copied = work / "src" / "mini" / "core.py" + self.assertTrue(copied.is_file()) + copied.write_text("if False: # mutated in the copy\n", encoding="utf-8") + self.assertEqual(before, self._real_tree_digest()) + + def test_untracked_files_are_not_copied(self): + (self.repo / "leftover.mutbak").write_text("junk", encoding="utf-8") + with tempfile.TemporaryDirectory() as tmp: + work = pathlib.Path(tmp) / "tree" + self.mod._prepare_workdir(self.repo, work) + self.assertFalse((work / "leftover.mutbak").exists()) + + def test_full_run_kills_mutant_and_leaves_real_tree_clean(self): + status_before = self.mod._worktree_status(self.repo) + rc = self.mod.main() + self.assertEqual(rc, 0) + self.assertEqual(status_before, self.mod._worktree_status(self.repo)) + self.assertEqual(_git(self.repo, "status", "--porcelain").stdout, "") + + def test_stale_operator_is_a_gap(self): + self.mod.MUTATIONS = [ + ("src/mini/core.py", "NOT PRESENT", "if False:", "mini: stale", True), + ] + self.assertEqual(self.mod.main(), 1) + + +class TestLeftoverFailClosed(_MiniRepo): + def test_leftover_working_tree_change_fails_the_run(self): + def dirty_and_pass(work): + (self.repo / "planted_leftover.py").write_text("x = 1\n", encoding="utf-8") + return 0 + + with mock.patch.object(self.mod, "_run_operators", dirty_and_pass): + rc = self.mod.main() + self.assertEqual(rc, 1) + + def test_clean_run_with_no_gaps_passes(self): + with mock.patch.object(self.mod, "_run_operators", lambda work: 0): + self.assertEqual(self.mod.main(), 0) + + +if __name__ == "__main__": + unittest.main()