diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 0000000..c6ed5a6 --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,86 @@ +# Portable mutation PR gate (StrykerJS). To adopt in another repo: +# 1. copy scripts/mutation-gate.mjs and this file; +# 2. replace the toolchain setup below (setup-bun + bun install) with your own +# (e.g. drop setup-bun, use actions/setup-node + `npm ci`); +# 3. tune the MUTATION_GATE_* env knobs (full table in the script header); +# 4. add the `mutation` check to branch protection; create the +# mutate-force / mutate-skip labels. +# Modes: changed (default: mutates the PR's changed files) or incremental +# (MUTATION_GATE_MODE: incremental; needs a baseline, see the commented-out +# job at the bottom, plus an actions/cache restore step in this job). +name: mutation +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, labeled, unlabeled] +concurrency: + group: mutation-${{ github.event.pull_request.number }} + cancel-in-progress: true +permissions: + contents: read + pull-requests: write # sticky comment +jobs: + mutation: + runs-on: ubuntu-latest + timeout-minutes: 15 # backstop against a mispredicted run + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} # the PR as authored, not the synthetic merge ref + fetch-depth: 0 # all branches: merge-base needs the real base branch + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.14" # same pin rationale as gates (bunfig coverage semantics) + - uses: actions/setup-node@v5 + with: + node-version: "24" # Stryker CLI host (D-010); >=20 required, 20 is EOL + - run: bun install --frozen-lockfile + - name: gate + id: gate + env: + MUTATION_GATE_BASE: origin/${{ github.event.pull_request.base.ref }} # the branch, never the payload SHA + MUTATION_GATE_FORCE: ${{ contains(github.event.pull_request.labels.*.name, 'mutate-force') && '1' || '' }} + MUTATION_GATE_SKIP: ${{ contains(github.event.pull_request.labels.*.name, 'mutate-skip') && '1' || '' }} + MUTATION_GATE_EXTRA_ARGS: "--concurrency 4" # Task 10 measurement: conc-1 exceeded the 15-min job timeout at 409 lines; conc-4 completed in 7.7 min (27.5 mutants/min) with a clean, non-corrupted report + run: node scripts/mutation-gate.mjs + - name: sticky comment + if: always() && steps.gate.outputs.decision != '' + continue-on-error: true # fork PRs get a read-only token; the verdict is the gate step's alone + env: + GH_TOKEN: ${{ github.token }} + DECISION: ${{ steps.gate.outputs.decision }} + SUMMARY: ${{ steps.gate.outputs.summary }} + PR: ${{ github.event.pull_request.number }} + run: | + BODY_FILE="$(mktemp)" + printf '\n\n%s\n' "$SUMMARY" > "$BODY_FILE" + EXISTING="$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty')" + case "$DECISION" in + pass|pass-empty) [ -z "$EXISTING" ] && exit 0 ;; # quiet pass: only update an existing comment + esac + if [ -n "$EXISTING" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${EXISTING}" -F body=@"$BODY_FILE" + else + gh api -X POST "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" -F body=@"$BODY_FILE" + fi + - name: report artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: mutation-report + path: reports/mutation/ + retention-days: 14 + if-no-files-found: ignore +# baseline: # incremental-mode adopters: produce/refresh the baseline on pushes to main +# # (also add `push: { branches: [main] }` to `on:` and an `if: github.event_name == 'push'` guard) +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v7 +# - +# - uses: actions/cache@v4 +# with: +# path: reports/stryker-incremental.json +# key: stryker-incremental-${{ github.sha }} +# restore-keys: stryker-incremental- +# - run: node_modules/.bin/stryker run --incremental --incrementalFile reports/stryker-incremental.json diff --git a/AGENTS.md b/AGENTS.md index f0676ba..211a419 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,12 +45,12 @@ Guidance for any agent (or human) working in this repo. `CLAUDE.md` is a symlink ## Working notes - **Git identity is the user's to set** — don't run `git config user.*` on their behalf. Commit/push **only when asked**. -- **`bun run mutate` (Stryker) needs a Node >= 20 binary on `PATH`** to host the Stryker CLI process itself — Bun cannot (a `@babel/generator` CJS/ESM-interop crash: `TypeError: generator is not a function`); the runner plugin itself still drives `bun test`. Mutation testing is manual and never a gate. The runner sanitizes bunfig.toml for its child runs (forces coverage=false), which is why the always-on gate and mutation runs compose safely. +- **`bun run mutate` (Stryker) needs a Node >= 20 binary on `PATH`** to host the Stryker CLI process itself — Bun cannot (a `@babel/generator` CJS/ESM-interop crash: `TypeError: generator is not a function`); the runner plugin itself still drives `bun test`. Mutation testing is manual full campaigns plus a changed-file PR gate (`.github/workflows/mutation.yml` + `scripts/mutation-gate.mjs`, D-027): small PRs are gated on zero undetected mutants in touched engine files; large PRs loud-skip with a sticky comment (labels `mutate-force`/`mutate-skip` override); after any Stryker/runner bump, run a full campaign before engine PRs resume. The runner sanitizes bunfig.toml for its child runs (forces coverage=false), which is why the always-on gate and mutation runs compose safely. - **`nvm use default` puts a Node 24 on PATH** for `bun run mutate` / focused `stryker run` invocations. - **`bun test ` may exit 1 with zero failures** — the per-file coverage floor (bunfig.toml) judges partially-imported files. Exit 1 with 0 fails = coverage floor, not a test failure; gate on full `bun test` runs. - **Internal imports**: upward reaches (anything needing `../`) use `#src/…`/`#scripts/…` (package.json `imports`); same-directory and downward stay relative, explicit `.ts` extensions. Enforced by `test/import-style.test.ts` (D-013). - **Never run `biome migrate` unattended** (D-021, D-023). On the v1 config it rewrote `"rules": { "recommended": true }` as `"rules": { "preset": "none" }`, which deletes the rule set instead of preserving it: `biome check .` then exits 0 on code containing `any`, `==` and unused vars, so the lint gate dies silently and CI stays green. The correct spelling is `"preset": "recommended"`, and `test/lint-gate.test.ts` now fails if it ever changes back. After any biome config change, re-verify with a planted violation and check the **exit code**, not the printed summary. - **Biome's "safe" fixes are not all safe here** (D-023). `noUselessEscapeInRegex` unescaped the `\.` in `MQTT_EXTENSION_KEY`, which is a no-op to the regex engine but breaks D-019's character-for-character transcription of the upstream schema key that `test/upstream-drift.test.ts` compares byte-for-byte. It is suppressed inline with that reason. Read what `--write` changed before trusting it; the test suite caught this one, but a less-covered invariant would have slipped through. -- **TypeScript 7 ships `tsc` only** (D-022) — no `tsserver.js`, no programmatic `typescript` module API under `node_modules/typescript/lib`. Nothing in the repo imports it as a module, so gates and mutation runs are unaffected, but an editor set to "use workspace TypeScript version" finds no language server and silently falls back to its own bundled TypeScript. Expect the editor and the `typecheck` gate to be different compilers; when they disagree, `bun run typecheck` is the authority. +- **TypeScript 7 ships `tsc` only** (D-022) — no `tsserver.js`, no programmatic `typescript` module API under `node_modules/typescript/lib`. Nothing in the repo imports it as a module, so the gates are unaffected, but Stryker's core sandbox preprocessing loads `typescript` regardless of `checkers` config (found when mutation first ran in CI, D-027), no-op'd via the `stryker.conf.json` `tsconfigFile` sentinel and pinned by `test/stryker-tsconfig-noop.test.ts`. An editor set to "use workspace TypeScript version" finds no language server and silently falls back to its own bundled TypeScript. Expect the editor and the `typecheck` gate to be different compilers; when they disagree, `bun run typecheck` is the authority. - **Dependency bumps: refresh ≠ range change.** Taking a newer build of an already-declared range is routine; requiring a version you previously did not is a decision. `bun update` conflates them — it rewrites `package.json` floors even for packages whose version did not move — so refresh with `bun update`, then `git checkout package.json && bun install` to keep the change lockfile-only. Range changes get their own entry in `DECISIONS.md`, their own PR, and a measurement (D-020, D-021). -- **CI (GitHub Actions)**: `.github/workflows/ci.yml` runs the gate set (`check-docs` → `lint` → `typecheck` → `demo-app:build` → full `bun test`) on PRs and main pushes; main pushes also upload `demo-app/dist/` + `coverage/` artifacts. Bun is pinned there (1.3.14) so the bunfig coverage-gate semantics stay as verified; bump the pin deliberately, in its own PR. A `main` ruleset requires the `gates` check (repo-admin bypass keeps direct pushes possible). Mutation testing stays out of CI (D-017). +- **CI (GitHub Actions)**: `.github/workflows/ci.yml` runs the gate set (`check-docs` → `lint` → `typecheck` → `demo-app:build` → full `bun test`) on PRs and main pushes; main pushes also upload `demo-app/dist/` + `coverage/` artifacts. Bun is pinned there (1.3.14) so the bunfig coverage-gate semantics stay as verified; bump the pin deliberately, in its own PR. A `main` ruleset requires the `gates` check (repo-admin bypass keeps direct pushes possible). The `mutation` required check runs the changed-file gate on PRs (D-027); full campaigns stay out of CI. diff --git a/DECISIONS.md b/DECISIONS.md index fb2780c..1ab9318 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -256,3 +256,15 @@ Append-only. Each decision has a stable never-reused `D-###` id, what was decide **Obligations**: if the real client's profile ever changes (auth appears, QoS 2 shows up, the protocol level moves, the transport changes), re-run the capture per demo-app.md §9 and update `fixtures/connect/real-client.json` + this entry's facts; none blocking. **From**: the authoritative spike runs against the real browser application (reported 2026-08-03), executed per docs/specs/demo-app.md §9. **Folds into**: REQUIREMENTS.md (R-006, R-007, R-033 text, seeding note), fixtures/connect/real-client.json, fixtures/connect/README.md, src/broker/connect-profile.test.ts, docs/specs/build-plan.md §5, AGENTS.md (Status & next) + +### D-027: A changed-file mutation gate on PRs; full campaigns stay manual +**Date**: 2026-08-04 +**What**: A second required check, `mutation` (`.github/workflows/mutation.yml` + dependency-free `scripts/mutation-gate.mjs`, the two-file portable unit): on every PR targeting main, mutate exactly the changed files that match `stryker.conf.json`'s `mutate` globs, plus sibling sources of changed or deleted test files (existence checked at HEAD), and fail below score 100. Score is Stryker's own metric: detected = Killed + Timeout, undetected = Survived + NoCoverage; Ignored and error statuses stay out of the verdict; zero valid mutants scores 100; the verdict is computed from the JSON report, never Stryker's exit code (thresholds.break defaults to null, so the exit code carries nothing). Above MUTATION_GATE_THRESHOLD_LINES (800 summed whole-file lines, unchanged — see Measured below) the gate loud-skips: green check, step summary, sticky PR comment nudging a local run; labels `mutate-force`/`mutate-skip` override in both directions, force wins. Diff base: the PR head SHA is checked out and merge-based against `origin/`, never the payload base SHA or the synthetic merge ref (stale-payload diffs otherwise fail open via loud-skip). Report artifact uploads on failure only. An incremental mode ships for adopting projects (baseline required by default, loud-skip without it). +**Measured** (ubuntu-latest, Task 10 rehearsal, 2026-08-04): mutating `src/engine/index.ts` + `instances.ts` (409 summed lines) at concurrency 1 hit the job's 15-minute `timeout-minutes` backstop — cancelled mid-campaign, no report written, no mutants/min computable. Concurrency 4 on the identical mutate set completed cleanly in 7m42s (462s): 212 mutants (195 killed, 3 timeout, 1 survived, 13 ignored; score 99.50), **27.53 mutants/min**, zero coverage-correlation warnings in the run log (no sign of the perTest-under-parallel-workers mis-correlation risk the design's Threshold rationale flagged as unverified). Concurrency 1 produced no report, so the design's planned "identical per-status counts" comparison is inapplicable; the timeout-vs-completes-cleanly outcome is itself the decision criterion, and it resolves unambiguously. **Shipped**: `MUTATION_GATE_EXTRA_ARGS: "--concurrency 4"` in `mutation.yml`. **Derived threshold**: 27.53 × 12 min / 0.5 mutants/line = 660.72 → 700 (nearest 100); 700 is within a factor of 2 of the provisional 800 (ratio ≈ 1.14, inside [400, 1600]) → `THRESHOLD_LINES` **stays 800** unchanged in `DEFAULTS`, its test expectations, and the spec's Threshold rationale. +**Why**: Amends D-010 ("run manually, never a gate") and D-017 ("mutation testing is excluded from CI in any form"): both stances priced a full-campaign gate, and a changed-file gate prices per-PR work instead, catching test-strength regressions at merge time where they are cheapest. Whole-file (not changed-line) mutation keeps the D-011 ratchet reading: every file a PR touches ends the PR mutation-clean. Loud-skip keeps the obligation visible on large PRs without holding the check hostage; the label escape hatches keep "mandatory" from eroding at the first heuristic misfire. +**Mitigations / notes**: The gate does not police annotation quality: `Ignored` is excluded, so a `// Stryker disable` comment silences a survivor, and the unobservability argument stays human review (D-011). A Stryker or runner bump can change the mutant set; run a full campaign (`bun run mutate`) after any such bump before engine PRs resume, or the drift lands on the next innocent PR. Test-helper and config changes do not trigger the gate (accepted residual; incremental mode is the answer for projects that care). Widening the `mutate` globs beyond `src/engine/` stays module-by-module, each behind its own kill-or-annotate campaign. **The gate's thesis was validated before it shipped**: the measurement run surfaced a genuine post-D-011 drift survivor, `src/engine/index.ts:311` (`OptionalChaining`), introduced by commit 245ab16 (the L2 scenario runtime, merged after the D-011 campaign closed) and invisible until this rehearsal exercised the file. It was killed by a test (commit 2f2342e), and a full `src/engine` recertification campaign then ran clean under the amended conf: 438 killed + 1 timeout, 0 survived, 0 no-coverage (local, concurrency 4, 3m30s) — exactly the D-011 clean-report bar. **Known flake, documented not fixed**: `@hughescr/stryker-bun-runner` carries an upstream-documented intermittent Bun `TestReporter` id-collision bug that aborts the dry run with a `ConfigError` unrelated to any real mutant. Frequent locally during this work (5 of 7 attempts in one session) but zero occurrences across the seven CI runs measured for this entry. The gate's infra-fail verdict is already distinct from a fail/pass verdict (edge case in the design doc), and a re-run clears it; filing the upstream issue is an open follow-up, not a blocker. +**Discovered during implementation (2026-08-05)**: The first rehearsal run did not produce the expected verdict at all — it infra-failed before instrumenting a single mutant, with every subsequent CI invocation failing identically until fixed. `@stryker-mutator/core@9.6.1`'s `TSConfigPreprocessor` unconditionally calls `ts.parseConfigFileTextToJson` on the sandboxed `tsconfig.json` to rewrite `extends`/`references` paths, regardless of the conf's `checkers` setting; TypeScript 7 (D-022) ships `tsc` only, with no programmatic module API, so the call does not exist and every mutation run in CI crashed with `TypeError: ts.parseConfigFileTextToJson is not a function`. **This amends D-022's mitigation claim** ("stryker.conf.json configures no checkers, so mutation runs never load it either"): that was true of `checkers`, but not of Stryker's own core sandbox preprocessing, which loads `typescript` unconditionally and independent of the `checkers` config — invisible until mutation testing first ran in CI, which D-017 had kept from happening at all until this gate. **Fix**: `stryker.conf.json` sets `"tsconfigFile": "tsconfig.stryker-unused.json"`, a deliberately nonexistent sentinel path. The preprocessor's crash is reached only when the configured path exists in the sandbox's file set, so pointing it at an absent path makes the whole step a clean no-op — offbook's real `tsconfig.json` has no `extends`/`references` and no `typescript-checker` plugin is configured, so the step buys nothing here regardless. **Residual risk, stated explicitly**: a future `@stryker-mutator/core` bump could start validating the configured-but-absent path instead of silently skipping it; the invariant is pinned by `test/stryker-tsconfig-noop.test.ts`, and a focused `stryker run` re-verification is required after any Stryker or runner bump, before trusting the gate again. **Second fix**: past the tsconfig crash, the dry run (which must execute the whole suite once, for perTest coverage) was then killed by `@hughescr/stryker-bun-runner`'s own process-timeout: `bun.timeout` (default 10000ms, documented as "per test") plus a fixed 30000ms drain ceiling, a 40-second hard kill on the *entire* dry-run process — well short of Stryker core's own 5-minute `dryRunTimeoutMinutes`, which never got a chance to apply. The most recent plain `bun test` run in the `ci` workflow took ~68s for the full suite at measurement time, so 40s was never enough on a GitHub-hosted runner. `stryker.conf.json` sets `"bun": { "timeout": 120000 }`, raising the dry-run ceiling to 150s (~2.2x the observed baseline) while keeping the per-mutant ceiling (the same knob) well under the 15-minute job timeout. +**Consequences for earlier entries**: Amends D-022's Mitigations claim as described above; D-022's decision to take TypeScript 7 is unaffected — only its "gates and mutation runs are unaffected" scope needed correcting, since mutation testing had never actually run in CI (D-017) at the time D-022 was written. +**Obligations**: none blocking. Open, non-blocking: file the upstream `@hughescr/stryker-bun-runner` TestReporter id-collision issue; re-verify the `tsconfigFile` no-op with a focused `stryker run` after any future Stryker or runner bump. +**From**: docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md (brainstorm dialog + adversarial agent review, 2026-08-04) +**Folds into**: scripts/mutation-gate.mjs, scripts/mutation-gate.test.ts, .github/workflows/mutation.yml, stryker.conf.json, test/stryker-tsconfig-noop.test.ts, src/engine/index.test.ts, AGENTS.md (working notes), DECISIONS.md (D-022 amendment note) diff --git a/docs/superpowers/plans/2026-08-04-mutation-pr-gate.md b/docs/superpowers/plans/2026-08-04-mutation-pr-gate.md new file mode 100644 index 0000000..8d8dd21 --- /dev/null +++ b/docs/superpowers/plans/2026-08-04-mutation-pr-gate.md @@ -0,0 +1,1641 @@ +# Changed-File Mutation PR Gate Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A required `mutation` PR check that runs StrykerJS over exactly the files a PR touched, fails on any undetected mutant, loud-skips over a size threshold with label overrides, and ships as two copy-able files for other StrykerJS projects. + +**Architecture:** One dependency-free `scripts/mutation-gate.mjs` (pure functions + an orchestrating `main(deps)` with injected I/O) invoked by a self-contained `.github/workflows/mutation.yml`. The verdict comes from Stryker's JSON report, never its exit code. Spec: `docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md` (read it before starting; it is the authority on "why"). + +**Tech Stack:** Bun 1.3.14 (tests, CI), Node 24 (Stryker CLI host), `@stryker-mutator/core` 9.6.1 + `@hughescr/stryker-bun-runner` (already installed), GitHub Actions + `gh`. + +## Global Constraints + +- Branch: all work on `mutation-pr-gate` (already exists, has the spec commits). +- `scripts/mutation-gate.mjs` is dependency-free: imports only `node:` builtins, nothing from the repo; runs under Node 18+ and Bun; assumes POSIX paths and repo-root cwd; `MUTATION_GATE_STRYKER_CMD` is split on spaces (no spaces in paths). +- The CLI entry guard is `realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)`. Never `import.meta.main` (absent before Node 24; the failure mode is a silent no-op green gate). +- Score formula (fixed by spec): detected = `Killed` + `Timeout`; undetected = `Survived` + `NoCoverage`; valid = detected + undetected; score = 100 × detected / valid; score = 100 when valid = 0. `Ignored`, `CompileError`, `RuntimeError`, `Pending` never enter the verdict but are counted. Unknown status = throw. +- Defaults: `MUTATION_GATE_BREAK=100`, `MUTATION_GATE_THRESHOLD_LINES=800` (provisional until Task 10's measurement), report at `reports/mutation/mutation.json`, incremental file at `reports/stryker-incremental.json`. +- Exit codes: 0 pass/skip, 1 gate failure, 2 infra failure. +- No new packages. No changes to `stryker.conf.json`, `bunfig.toml`, `biome.json`, `.github/workflows/ci.yml`, or the coverage floors. +- **Focused `bun test scripts/mutation-gate.test.ts` may exit 1 with ZERO failing tests** (the bunfig per-file coverage floor judges partially-imported files). For the red/green TDD steps below, judge by the printed fail count. Gate every commit on full `bun test` exit 0 (`bun test; echo "exit=$?"`), which is authoritative (AGENTS.md). +- Stryker's CLI host needs Node >= 20 on PATH locally: run `nvm use default` (Node 24) once per shell before any step that spawns Stryker. +- Doc edits must keep `bun scripts/check-docs.ts` exit 0. +- Commits: imperative house style (`feat: ...`, `docs: ...`, `ci: ...`). Never add Co-Authored-By or any AI-attribution trailer. +- `scripts/` is excluded from Biome and from tsconfig `include`; do not "fix" that. Lint/typecheck make no claims about these files; the tests and rehearsal are the safety net. + +--- + +### Task 1: Diff parsing and line counting + +**Files:** +- Create: `scripts/mutation-gate.mjs` +- Create: `scripts/mutation-gate.test.ts` + +**Interfaces:** +- Produces: `parseNameStatusZ(raw: string) -> { changed: string[], deleted: string[] }` (throws on unhandled status); `countLines(content: string) -> number`. Task 7 consumes both. + +- [ ] **Step 1: Write the failing tests** + +Create `scripts/mutation-gate.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { parseNameStatusZ, countLines } from "./mutation-gate.mjs"; + +test("parseNameStatusZ splits adds/modifies from deletes", () => { + const raw = "A\0src/engine/new.ts\0M\0src/engine/dispatch.ts\0D\0src/engine/old.test.ts\0"; + expect(parseNameStatusZ(raw)).toEqual({ + changed: ["src/engine/new.ts", "src/engine/dispatch.ts"], + deleted: ["src/engine/old.test.ts"], + }); +}); + +test("parseNameStatusZ reads the three-field rename record and keeps the new path", () => { + const raw = "R100\0src/engine/a.ts\0src/engine/b.ts\0M\0src/engine/c.ts\0"; + expect(parseNameStatusZ(raw)).toEqual({ changed: ["src/engine/b.ts", "src/engine/c.ts"], deleted: [] }); +}); + +test("parseNameStatusZ handles copies and typechanges as changes", () => { + const raw = "C75\0src/a.ts\0src/b.ts\0T\0src/c.ts\0"; + expect(parseNameStatusZ(raw)).toEqual({ changed: ["src/b.ts", "src/c.ts"], deleted: [] }); +}); + +test("parseNameStatusZ throws on an unhandled status instead of mis-pairing the rest", () => { + expect(() => parseNameStatusZ("U\0src/conflicted.ts\0")).toThrow('unhandled diff status "U"'); +}); + +test("parseNameStatusZ of an empty diff is empty", () => { + expect(parseNameStatusZ("")).toEqual({ changed: [], deleted: [] }); +}); + +test("countLines counts content lines with and without trailing newline", () => { + expect(countLines("")).toBe(0); + expect(countLines("a\n")).toBe(1); + expect(countLines("a\nb")).toBe(2); + expect(countLines("a\nb\n")).toBe(2); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: failures (module `./mutation-gate.mjs` not found). Judge by printed failures, not exit code. + +- [ ] **Step 3: Write the implementation** + +Create `scripts/mutation-gate.mjs`: + +```js +// mutation-gate: a changed-file StrykerJS mutation gate for PRs. +// Portable unit: this file + .github/workflows/mutation.yml (copy both). +// Spec: docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md +// +// Configuration (env, all optional): +// MUTATION_GATE_MODE changed (default) | incremental +// MUTATION_GATE_BASE base ref; default origin/HEAD, then main +// MUTATION_GATE_THRESHOLD_LINES loud-skip above this summed line count (800) +// MUTATION_GATE_BREAK minimum score, fail below it (100) +// MUTATION_GATE_CONFIG stryker config path (stryker.conf.json) +// MUTATION_GATE_GLOBS comma-separated mutate globs, overrides config +// MUTATION_GATE_TEST_SIBLINGS changed/deleted X.test.ts pulls X.ts (true) +// MUTATION_GATE_FORCE run + block even over threshold (labels) +// MUTATION_GATE_SKIP loud-skip regardless of size (labels; FORCE wins) +// MUTATION_GATE_REQUIRE_BASELINE incremental: skip when baseline missing (true) +// MUTATION_GATE_INCREMENTAL_FILE reports/stryker-incremental.json +// MUTATION_GATE_STRYKER_CMD node_modules/.bin/stryker run (split on spaces) +// MUTATION_GATE_EXTRA_ARGS appended to the stryker invocation +// MUTATION_GATE_REPORT reports/mutation/mutation.json +// Exit codes: 0 pass/skip, 1 gate failure (undetected mutants), 2 infra failure. + +export function parseNameStatusZ(raw) { + const tokens = raw.split("\0").filter((t) => t.length > 0); + const changed = []; + const deleted = []; + let i = 0; + while (i < tokens.length) { + const status = tokens[i]; + const kind = status[0]; + if (kind === "R" || kind === "C") { + changed.push(tokens[i + 2]); + i += 3; + } else if (kind === "D") { + deleted.push(tokens[i + 1]); + i += 2; + } else if (kind === "A" || kind === "M" || kind === "T") { + changed.push(tokens[i + 1]); + i += 2; + } else { + throw new Error(`mutation-gate: unhandled diff status "${status}"`); + } + } + return { changed, deleted }; +} + +export function countLines(content) { + if (content === "") return 0; + const lines = content.split("\n"); + if (lines[lines.length - 1] === "") lines.pop(); + return lines.length; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: 6 pass, 0 fail (exit code may still be 1 from the coverage floor; that is fine on focused runs). + +- [ ] **Step 5: Full-suite gate and commit** + +```bash +bun test; echo "exit=$?" # expect exit=0 +git add scripts/mutation-gate.mjs scripts/mutation-gate.test.ts +git commit -m "feat: mutation-gate diff parsing and line counting" +``` + +--- + +### Task 2: Glob subset matcher with loud refusal + +**Files:** +- Modify: `scripts/mutation-gate.mjs` (append) +- Modify: `scripts/mutation-gate.test.ts` (append) + +**Interfaces:** +- Produces: `UnsupportedGlobError` (Error subclass); `globToRegExp(pattern: string) -> RegExp` (throws `UnsupportedGlobError`); `matchesMutateGlobs(path: string, globs: string[]) -> boolean` (ordered, `!`-negation unsets). Tasks 3 and 7 consume `matchesMutateGlobs`. + +- [ ] **Step 1: Write the failing tests** (append to `scripts/mutation-gate.test.ts`; extend the first import line with the new names) + +```ts +import { globToRegExp, matchesMutateGlobs, UnsupportedGlobError } from "./mutation-gate.mjs"; + +test("offbook's real globs: engine source in, engine tests out, broker out", () => { + const globs = ["src/engine/**/*.ts", "!src/engine/**/*.test.ts"]; + expect(matchesMutateGlobs("src/engine/scheduler.ts", globs)).toBe(true); + expect(matchesMutateGlobs("src/engine/sub/dir/x.ts", globs)).toBe(true); + expect(matchesMutateGlobs("src/engine/scheduler.test.ts", globs)).toBe(false); + expect(matchesMutateGlobs("src/broker/index.ts", globs)).toBe(false); +}); + +test("** matches zero segments", () => { + expect(globToRegExp("src/**/*.ts").test("src/index.ts")).toBe(true); +}); + +test("* stays inside one segment; ? matches exactly one char", () => { + expect(globToRegExp("src/*.ts").test("src/a/b.ts")).toBe(false); + expect(globToRegExp("src/?.ts").test("src/a.ts")).toBe(true); + expect(globToRegExp("src/?.ts").test("src/ab.ts")).toBe(false); +}); + +test("single-level braces expand", () => { + const re = globToRegExp("{src,lib}/a.ts"); + expect(re.test("src/a.ts")).toBe(true); + expect(re.test("lib/a.ts")).toBe(true); + expect(re.test("bin/a.ts")).toBe(false); +}); + +test("negation is ordered unset-on-match, later patterns win", () => { + expect(matchesMutateGlobs("src/a.test.ts", ["src/**/*.ts", "!src/**/*.test.ts"])).toBe(false); + expect(matchesMutateGlobs("src/a.test.ts", ["!src/**/*.test.ts", "src/**/*.ts"])).toBe(true); +}); + +test("extglobs, character classes, escapes, wildcards-in-braces are refused, never mis-matched", () => { + for (const bad of ["src/!(*.test).ts", "src/[ab].ts", "src/a\\*.ts", "{src/*,lib}/a.ts"]) { + expect(() => globToRegExp(bad)).toThrow(UnsupportedGlobError); + } +}); + +test("regex metacharacters in glob literals are escaped (a dot is a dot)", () => { + expect(globToRegExp("src/a.ts").test("src/axts")).toBe(false); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: the 7 new tests fail (exports missing); the Task 1 tests still pass. + +- [ ] **Step 3: Write the implementation** (append to `scripts/mutation-gate.mjs`) + +```js +export class UnsupportedGlobError extends Error { + constructor(pattern) { + super( + `mutation-gate: glob "${pattern}" uses syntax outside the supported subset ` + + `(**, *, ?, {a,b}, leading !). Set MUTATION_GATE_GLOBS to equivalent simple globs.`, + ); + this.name = "UnsupportedGlobError"; + } +} + +const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +export function globToRegExp(pattern) { + if (/[()[\]\\]/.test(pattern)) throw new UnsupportedGlobError(pattern); + let out = ""; + let i = 0; + while (i < pattern.length) { + const ch = pattern[i]; + if (ch === "*") { + if (pattern.startsWith("**/", i)) { + out += "(?:[^/]+/)*"; + i += 3; + } else if (pattern.startsWith("**", i)) { + out += ".*"; + i += 2; + } else { + out += "[^/]*"; + i += 1; + } + } else if (ch === "?") { + out += "[^/]"; + i += 1; + } else if (ch === "{") { + const end = pattern.indexOf("}", i); + const body = end === -1 ? "" : pattern.slice(i + 1, end); + if (end === -1 || /[*?{]/.test(body)) throw new UnsupportedGlobError(pattern); + out += `(?:${body.split(",").map(escapeRegExp).join("|")})`; + i = end + 1; + } else { + out += escapeRegExp(ch); + i += 1; + } + } + return new RegExp(`^${out}$`); +} + +export function matchesMutateGlobs(path, globs) { + let included = false; + for (const glob of globs) { + const negated = glob.startsWith("!"); + const pattern = negated ? glob.slice(1) : glob; + if (globToRegExp(pattern).test(path)) included = !negated; + } + return included; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: 13 pass, 0 fail. + +- [ ] **Step 5: Full-suite gate and commit** + +```bash +bun test; echo "exit=$?" # expect exit=0 +git add scripts/mutation-gate.mjs scripts/mutation-gate.test.ts +git commit -m "feat: mutation-gate glob subset matcher with loud refusal" +``` + +--- + +### Task 3: Sibling rule and mutate-set selection + +**Files:** +- Modify: `scripts/mutation-gate.mjs` (append) +- Modify: `scripts/mutation-gate.test.ts` (append) + +**Interfaces:** +- Consumes: `matchesMutateGlobs` (Task 2). +- Produces: `siblingOf(path: string) -> string | null`; `selectMutateSet({ changed, deleted, globs, testSiblings, exists }) -> string[]` (sorted, deduped; `exists: (path) => boolean` is existence **at HEAD**). Task 7 consumes `selectMutateSet`. + +- [ ] **Step 1: Write the failing tests** (append; extend the import with `siblingOf, selectMutateSet`) + +```ts +test("siblingOf derives the source next to a test file", () => { + expect(siblingOf("src/engine/scheduler.test.ts")).toBe("src/engine/scheduler.ts"); + expect(siblingOf("src/engine/faker.spec.tsx")).toBe("src/engine/faker.tsx"); + expect(siblingOf("src/engine/scheduler.ts")).toBe(null); +}); + +const ENGINE_GLOBS = ["src/engine/**/*.ts", "!src/engine/**/*.test.ts"]; + +test("selectMutateSet keeps matching changed sources, drops the rest", () => { + const files = selectMutateSet({ + changed: ["src/engine/dispatch.ts", "src/broker/index.ts", "README.md"], + deleted: [], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: () => true, + }); + expect(files).toEqual(["src/engine/dispatch.ts"]); +}); + +test("a changed test file pulls its existing sibling source", () => { + const files = selectMutateSet({ + changed: ["src/engine/scheduler.test.ts"], + deleted: [], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: (p) => p === "src/engine/scheduler.ts", + }); + expect(files).toEqual(["src/engine/scheduler.ts"]); +}); + +test("a deleted test file pulls its surviving sibling (the test-deletion evasion)", () => { + const files = selectMutateSet({ + changed: [], + deleted: ["src/engine/scheduler.test.ts"], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: (p) => p === "src/engine/scheduler.ts", + }); + expect(files).toEqual(["src/engine/scheduler.ts"]); +}); + +test("deleting module and test together pulls nothing (existence-at-HEAD)", () => { + const files = selectMutateSet({ + changed: [], + deleted: ["src/engine/gone.ts", "src/engine/gone.test.ts"], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: () => false, + }); + expect(files).toEqual([]); +}); + +test("testSiblings=false disables the rule; output is deduped and sorted", () => { + expect( + selectMutateSet({ + changed: ["src/engine/scheduler.test.ts"], + deleted: [], + globs: ENGINE_GLOBS, + testSiblings: false, + exists: () => true, + }), + ).toEqual([]); + expect( + selectMutateSet({ + changed: ["src/engine/b.ts", "src/engine/a.ts", "src/engine/a.test.ts"], + deleted: [], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: () => true, + }), + ).toEqual(["src/engine/a.ts", "src/engine/b.ts"]); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: the 6 new tests fail; all earlier tests pass. + +- [ ] **Step 3: Write the implementation** (append to `scripts/mutation-gate.mjs`) + +```js +export function siblingOf(path) { + const m = path.match(/^(.*)\.(test|spec)(\.[^./]+)$/); + return m ? `${m[1]}${m[3]}` : null; +} + +export function selectMutateSet({ changed, deleted, globs, testSiblings, exists }) { + const set = new Set(); + for (const path of changed) { + if (matchesMutateGlobs(path, globs)) set.add(path); + } + if (testSiblings) { + for (const path of [...changed, ...deleted]) { + const sibling = siblingOf(path); + if (sibling && exists(sibling) && matchesMutateGlobs(sibling, globs)) set.add(sibling); + } + } + return [...set].sort(); +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: 19 pass, 0 fail. + +- [ ] **Step 5: Full-suite gate and commit** + +```bash +bun test; echo "exit=$?" # expect exit=0 +git add scripts/mutation-gate.mjs scripts/mutation-gate.test.ts +git commit -m "feat: mutation-gate sibling rule and mutate-set selection" +``` + +--- + +### Task 4: Env configuration and the size decision + +**Files:** +- Modify: `scripts/mutation-gate.mjs` (append) +- Modify: `scripts/mutation-gate.test.ts` (append) + +**Interfaces:** +- Produces: `DEFAULTS` (frozen object); `readConfig(env: Record) -> cfg` with fields `mode, base, thresholdLines, breakScore, configPath, globsOverride, testSiblings, force, skip, requireBaseline, incrementalFile, strykerCmd (string[]), extraArgs (string[]), reportPath` (throws on non-numeric numbers); `decide({ files, totalLines, thresholdLines, force, skip }) -> "pass-empty" | "skip-label" | "skip-size" | "run"`. Task 7 consumes both. + +- [ ] **Step 1: Write the failing tests** (append; extend the import with `DEFAULTS, readConfig, decide`) + +```ts +test("decide: empty set passes, labels and threshold act, force wins over both", () => { + expect(decide({ files: [], totalLines: 0, thresholdLines: 800, force: false, skip: false })).toBe("pass-empty"); + expect(decide({ files: ["a"], totalLines: 10, thresholdLines: 800, force: false, skip: false })).toBe("run"); + expect(decide({ files: ["a"], totalLines: 900, thresholdLines: 800, force: false, skip: false })).toBe("skip-size"); + expect(decide({ files: ["a"], totalLines: 10, thresholdLines: 800, force: false, skip: true })).toBe("skip-label"); + expect(decide({ files: ["a"], totalLines: 900, thresholdLines: 800, force: true, skip: true })).toBe("run"); +}); + +test("readConfig defaults", () => { + const cfg = readConfig({}); + expect(cfg.mode).toBe("changed"); + expect(cfg.base).toBeUndefined(); + expect(cfg.thresholdLines).toBe(800); + expect(cfg.breakScore).toBe(100); + expect(cfg.configPath).toBe("stryker.conf.json"); + expect(cfg.globsOverride).toBeUndefined(); + expect(cfg.testSiblings).toBe(true); + expect(cfg.force).toBe(false); + expect(cfg.skip).toBe(false); + expect(cfg.requireBaseline).toBe(true); + expect(cfg.incrementalFile).toBe("reports/stryker-incremental.json"); + expect(cfg.strykerCmd).toEqual(["node_modules/.bin/stryker", "run"]); + expect(cfg.extraArgs).toEqual([]); + expect(cfg.reportPath).toBe("reports/mutation/mutation.json"); +}); + +test("readConfig parses overrides", () => { + const cfg = readConfig({ + MUTATION_GATE_MODE: "incremental", + MUTATION_GATE_BASE: "origin/develop", + MUTATION_GATE_THRESHOLD_LINES: "200", + MUTATION_GATE_BREAK: "90", + MUTATION_GATE_GLOBS: "lib/**/*.js, !lib/**/*.spec.js", + MUTATION_GATE_TEST_SIBLINGS: "false", + MUTATION_GATE_FORCE: "1", + MUTATION_GATE_EXTRA_ARGS: "--concurrency 4", + }); + expect(cfg.mode).toBe("incremental"); + expect(cfg.base).toBe("origin/develop"); + expect(cfg.thresholdLines).toBe(200); + expect(cfg.breakScore).toBe(90); + expect(cfg.globsOverride).toEqual(["lib/**/*.js", "!lib/**/*.spec.js"]); + expect(cfg.testSiblings).toBe(false); + expect(cfg.force).toBe(true); + expect(cfg.extraArgs).toEqual(["--concurrency", "4"]); +}); + +test("readConfig treats 0/false/no/empty as false for flags and rejects non-numbers", () => { + expect(readConfig({ MUTATION_GATE_FORCE: "false" }).force).toBe(false); + expect(readConfig({ MUTATION_GATE_SKIP: "0" }).skip).toBe(false); + expect(readConfig({ MUTATION_GATE_TEST_SIBLINGS: "" }).testSiblings).toBe(true); + expect(() => readConfig({ MUTATION_GATE_THRESHOLD_LINES: "many" })).toThrow("not a number"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: the 4 new tests fail; all earlier tests pass. + +- [ ] **Step 3: Write the implementation** (append to `scripts/mutation-gate.mjs`) + +```js +export const DEFAULTS = Object.freeze({ + mode: "changed", + thresholdLines: 800, + breakScore: 100, + configPath: "stryker.conf.json", + incrementalFile: "reports/stryker-incremental.json", + strykerCmd: "node_modules/.bin/stryker run", + reportPath: "reports/mutation/mutation.json", +}); + +const FALSY = new Set(["0", "false", "no"]); +const asBool = (v, dflt) => (v === undefined || v === "" ? dflt : !FALSY.has(v.toLowerCase())); +const asFlag = (v) => v !== undefined && v !== "" && !FALSY.has(v.toLowerCase()); +const asNum = (v, dflt) => { + if (v === undefined || v === "") return dflt; + const n = Number(v); + if (!Number.isFinite(n)) throw new Error(`mutation-gate: not a number: "${v}"`); + return n; +}; +const asList = (v) => + v === undefined + ? undefined + : v + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + +export function readConfig(env) { + return { + mode: env.MUTATION_GATE_MODE || DEFAULTS.mode, + base: env.MUTATION_GATE_BASE || undefined, + thresholdLines: asNum(env.MUTATION_GATE_THRESHOLD_LINES, DEFAULTS.thresholdLines), + breakScore: asNum(env.MUTATION_GATE_BREAK, DEFAULTS.breakScore), + configPath: env.MUTATION_GATE_CONFIG || DEFAULTS.configPath, + globsOverride: asList(env.MUTATION_GATE_GLOBS), + testSiblings: asBool(env.MUTATION_GATE_TEST_SIBLINGS, true), + force: asFlag(env.MUTATION_GATE_FORCE), + skip: asFlag(env.MUTATION_GATE_SKIP), + requireBaseline: asBool(env.MUTATION_GATE_REQUIRE_BASELINE, true), + incrementalFile: env.MUTATION_GATE_INCREMENTAL_FILE || DEFAULTS.incrementalFile, + strykerCmd: (env.MUTATION_GATE_STRYKER_CMD || DEFAULTS.strykerCmd).split(" ").filter(Boolean), + extraArgs: (env.MUTATION_GATE_EXTRA_ARGS || "").split(" ").filter(Boolean), + reportPath: env.MUTATION_GATE_REPORT || DEFAULTS.reportPath, + }; +} + +export function decide({ files, totalLines, thresholdLines, force, skip }) { + if (files.length === 0) return "pass-empty"; + if (force) return "run"; + if (skip) return "skip-label"; + if (totalLines > thresholdLines) return "skip-size"; + return "run"; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: 23 pass, 0 fail. + +- [ ] **Step 5: Full-suite gate and commit** + +```bash +bun test; echo "exit=$?" # expect exit=0 +git add scripts/mutation-gate.mjs scripts/mutation-gate.test.ts +git commit -m "feat: mutation-gate env config and size decision" +``` + +--- + +### Task 5: Report interpretation (the verdict) + +**Files:** +- Modify: `scripts/mutation-gate.mjs` (append) +- Modify: `scripts/mutation-gate.test.ts` (append) + +**Interfaces:** +- Produces: `interpretReport(report, breakScore) -> { counts, undetected: [{file, line, mutator}], score, verdict: "pass" | "fail" }`. `report` is mutation-testing-report-schema JSON (`files..mutants[].{status, mutatorName, location.start.line}`). Task 7 consumes it. The test-file fixture builder `makeReport` is reused by Task 7's tests. + +- [ ] **Step 1: Write the failing tests** (append; extend the import with `interpretReport`) + +```ts +type FixtureMutant = { mutator: string; status: string; line?: number }; +export function makeReport(mutantsByFile: Record) { + return { + schemaVersion: "2", + thresholds: { high: 80, low: 60 }, + files: Object.fromEntries( + Object.entries(mutantsByFile).map(([file, mutants]) => [ + file, + { + language: "typescript", + source: "", + mutants: mutants.map((m, i) => ({ + id: String(i), + mutatorName: m.mutator, + status: m.status, + location: { start: { line: m.line ?? 1, column: 1 }, end: { line: m.line ?? 1, column: 2 } }, + })), + }, + ]), + ), + }; +} + +test("all killed passes at break 100; Timeout counts as detected (the D-011 reading)", () => { + const r = interpretReport( + makeReport({ "src/engine/a.ts": [{ mutator: "X", status: "Killed" }, { mutator: "X", status: "Timeout" }] }), + 100, + ); + expect(r.score).toBe(100); + expect(r.verdict).toBe("pass"); +}); + +test("a survivor fails at break 100 and is listed as file:line mutator", () => { + const r = interpretReport( + makeReport({ + "src/engine/a.ts": [ + { mutator: "EqualityOperator", status: "Survived", line: 12 }, + { mutator: "StringLiteral", status: "Killed" }, + ], + }), + 100, + ); + expect(r.verdict).toBe("fail"); + expect(r.score).toBe(50); + expect(r.undetected).toEqual([{ file: "src/engine/a.ts", line: 12, mutator: "EqualityOperator" }]); +}); + +test("NoCoverage is undetected; Ignored/CompileError/RuntimeError/Pending stay out of the verdict", () => { + const r = interpretReport( + makeReport({ + "a.ts": [ + { mutator: "X", status: "Killed" }, + { mutator: "X", status: "NoCoverage", line: 3 }, + { mutator: "X", status: "Ignored" }, + { mutator: "X", status: "CompileError" }, + { mutator: "X", status: "RuntimeError" }, + { mutator: "X", status: "Pending" }, + ], + }), + 100, + ); + expect(r.score).toBe(50); + expect(r.verdict).toBe("fail"); + expect(r.undetected).toEqual([{ file: "a.ts", line: 3, mutator: "X" }]); + expect(r.counts).toEqual({ + Killed: 1, Survived: 0, NoCoverage: 1, Timeout: 0, CompileError: 1, RuntimeError: 1, Ignored: 1, Pending: 1, + }); +}); + +test("an errors-only run scores 100 (zero valid mutants is a pass, not NaN)", () => { + const r = interpretReport(makeReport({ "a.ts": [{ mutator: "X", status: "RuntimeError" }] }), 100); + expect(r.score).toBe(100); + expect(r.verdict).toBe("pass"); +}); + +test("break below 100 tolerates survivors down to the threshold, exact score passes", () => { + const twoOfThree = makeReport({ + "a.ts": [ + { mutator: "X", status: "Killed" }, + { mutator: "X", status: "Killed" }, + { mutator: "X", status: "Survived" }, + ], + }); + expect(interpretReport(twoOfThree, 66).verdict).toBe("pass"); + expect(interpretReport(twoOfThree, 67).verdict).toBe("fail"); +}); + +test("an unknown status throws (schema drift surfaces loudly)", () => { + expect(() => interpretReport(makeReport({ "a.ts": [{ mutator: "X", status: "Vanished" }] }), 100)).toThrow( + 'unknown mutant status "Vanished"', + ); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: the 6 new tests fail; all earlier tests pass. + +- [ ] **Step 3: Write the implementation** (append to `scripts/mutation-gate.mjs`) + +```js +export function interpretReport(report, breakScore) { + const counts = { + Killed: 0, Survived: 0, NoCoverage: 0, Timeout: 0, CompileError: 0, RuntimeError: 0, Ignored: 0, Pending: 0, + }; + const undetected = []; + for (const [file, data] of Object.entries(report.files ?? {})) { + for (const mutant of data.mutants) { + if (!(mutant.status in counts)) { + throw new Error(`mutation-gate: unknown mutant status "${mutant.status}"`); + } + counts[mutant.status] += 1; + if (mutant.status === "Survived" || mutant.status === "NoCoverage") { + undetected.push({ file, line: mutant.location.start.line, mutator: mutant.mutatorName }); + } + } + } + const detected = counts.Killed + counts.Timeout; + const valid = detected + counts.Survived + counts.NoCoverage; + const score = valid === 0 ? 100 : (100 * detected) / valid; + return { counts, undetected, score, verdict: score < breakScore ? "fail" : "pass" }; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: 29 pass, 0 fail. + +- [ ] **Step 5: Full-suite gate and commit** + +```bash +bun test; echo "exit=$?" # expect exit=0 +git add scripts/mutation-gate.mjs scripts/mutation-gate.test.ts +git commit -m "feat: mutation-gate report interpretation over the full status enum" +``` + +--- + +### Task 6: Summaries and GitHub outputs + +**Files:** +- Modify: `scripts/mutation-gate.mjs` (append) +- Modify: `scripts/mutation-gate.test.ts` (append) + +**Interfaces:** +- Consumes: `interpretReport`'s result shape (Task 5). +- Produces: `renderSkip({ decision, files, totalLines, thresholdLines }) -> string`; `renderResult({ files, result, breakScore }) -> string`; `renderInfra(message: string) -> string`; `formatGithubOutputs(outputs: Record) -> string` (heredoc format). Task 7 consumes all four. + +- [ ] **Step 1: Write the failing tests** (append; extend the import with the four names) + +```ts +test("renderSkip names the numbers, the labels, and the local command", () => { + const md = renderSkip({ decision: "skip-size", files: ["src/engine/index.ts"], totalLines: 950, thresholdLines: 800 }); + expect(md).toContain("skip-size"); + expect(md).toContain("950"); + expect(md).toContain("800"); + expect(md).toContain("bun run mutate"); + expect(md).toContain("mutate-force"); + expect(renderSkip({ decision: "pass-empty", files: [], totalLines: 0, thresholdLines: 800 })).toContain( + "no mutable files changed", + ); + expect( + renderSkip({ decision: "skip-no-baseline", files: ["a.ts"], totalLines: 1, thresholdLines: 800 }), + ).toContain("baseline"); +}); + +test("renderResult on failure lists each undetected mutant with the kill-or-annotate instruction", () => { + const result = { + counts: { Killed: 1, Survived: 1, NoCoverage: 0, Timeout: 0, CompileError: 0, RuntimeError: 0, Ignored: 0, Pending: 0 }, + undetected: [{ file: "src/engine/a.ts", line: 12, mutator: "EqualityOperator" }], + score: 50, + verdict: "fail" as const, + }; + const md = renderResult({ files: ["src/engine/a.ts"], result, breakScore: 100 }); + expect(md).toContain("fail"); + expect(md).toContain("50.00"); + expect(md).toContain("src/engine/a.ts:12"); + expect(md).toContain("EqualityOperator"); + expect(md).toContain("Stryker disable next-line"); +}); + +test("renderResult on pass reports the score and mutant counts", () => { + const result = { + counts: { Killed: 3, Survived: 0, NoCoverage: 0, Timeout: 1, CompileError: 0, RuntimeError: 0, Ignored: 2, Pending: 0 }, + undetected: [], + score: 100, + verdict: "pass" as const, + }; + const md = renderResult({ files: ["src/engine/a.ts"], result, breakScore: 100 }); + expect(md).toContain("pass"); + expect(md).toContain("100.00"); + expect(md).toContain("3 killed"); +}); + +test("renderInfra says it is not a verdict", () => { + expect(renderInfra("boom")).toContain("boom"); + expect(renderInfra("boom")).toContain("not a verdict"); +}); + +test("formatGithubOutputs emits the heredoc form for each key", () => { + expect(formatGithubOutputs({ decision: "fail", summary: "line1\nline2" })).toBe( + "decision<<__MUTATION_GATE_EOF__\nfail\n__MUTATION_GATE_EOF__\n" + + "summary<<__MUTATION_GATE_EOF__\nline1\nline2\n__MUTATION_GATE_EOF__\n", + ); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: the 5 new tests fail; all earlier tests pass. + +- [ ] **Step 3: Write the implementation** (append to `scripts/mutation-gate.mjs`) + +```js +const SKIP_REASONS = { + "pass-empty": "no mutable files changed; nothing to mutate.", + "skip-size": "the change is over the size threshold for a CI mutation run.", + "skip-label": "the mutate-skip label is set.", + "skip-no-baseline": "incremental mode has no baseline incremental file; refusing a surprise full campaign.", +}; + +export function renderSkip({ decision, files, totalLines, thresholdLines }) { + const lines = [`## mutation gate: ${decision}`, "", SKIP_REASONS[decision] ?? decision]; + if (decision !== "pass-empty") { + lines.push( + "", + `Mutable files in this change: ${files.length} (${totalLines} lines; threshold ${thresholdLines}).`, + "The gate did not run. Before merging, run the mutation check locally: `bun run mutate`", + "(or `MUTATION_GATE_BASE= node scripts/mutation-gate.mjs` for the changed-file run).", + "Labels: `mutate-force` runs the gate anyway; `mutate-skip` waves it off.", + ); + } + return `${lines.join("\n")}\n`; +} + +export function renderResult({ files, result, breakScore }) { + const c = result.counts; + const lines = [ + `## mutation gate: ${result.verdict} (score ${result.score.toFixed(2)}, break ${breakScore})`, + "", + `Mutated ${files.length} file(s): ${files.join(", ")}`, + `Mutants: ${c.Killed} killed, ${c.Timeout} timeout, ${c.Survived} survived, ${c.NoCoverage} no-coverage; ` + + `${c.Ignored} ignored, ${c.CompileError + c.RuntimeError} errored, ${c.Pending} pending.`, + ]; + if (result.undetected.length > 0) { + lines.push("", "Undetected mutants (kill each with a test, or annotate with a reasoned", + "`// Stryker disable next-line : `):", ""); + for (const m of result.undetected) { + lines.push(`- \`${m.file}:${m.line}\` ${m.mutator}`); + } + } + return `${lines.join("\n")}\n`; +} + +export function renderInfra(message) { + return `## mutation gate: infra failure\n\n${message}\n\nThis is an infrastructure error, not a verdict on the PR's tests.\n`; +} + +export function formatGithubOutputs(outputs) { + const lines = []; + for (const [key, value] of Object.entries(outputs)) { + lines.push(`${key}<<__MUTATION_GATE_EOF__`, String(value), "__MUTATION_GATE_EOF__"); + } + return `${lines.join("\n")}\n`; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: 34 pass, 0 fail. + +- [ ] **Step 5: Full-suite gate and commit** + +```bash +bun test; echo "exit=$?" # expect exit=0 +git add scripts/mutation-gate.mjs scripts/mutation-gate.test.ts +git commit -m "feat: mutation-gate summaries and GitHub output formatting" +``` + +--- + +### Task 7: `main()` orchestration (changed mode), real deps, CLI entry + +**Files:** +- Modify: `scripts/mutation-gate.mjs` (append) +- Modify: `scripts/mutation-gate.test.ts` (append) + +**Interfaces:** +- Consumes: everything from Tasks 1-6, plus `makeReport` from the Task 5 test block. +- Produces: `EXIT` (`{ ok: 0, gateFail: 1, infra: 2 }`); `resolveDefaultBase(deps) -> string`; `readMutateGlobs(deps, configPath) -> string[]`; `main(deps?) -> number` (exit code); `realDeps() -> deps`. Deps shape: `{ env, exec(argv, opts?) -> { code, stdout }, readFile(p) -> string, exists(p) -> boolean, writeSummary(md), writeOutputs(obj), log(msg) }`. Task 8 extends `main` for incremental mode. + +- [ ] **Step 1: Write the failing tests** (append; extend the import with `EXIT, resolveDefaultBase, readMutateGlobs, main, realDeps`) + +```ts +const CONF = JSON.stringify({ mutate: ["src/engine/**/*.ts", "!src/engine/**/*.test.ts"] }); + +function fakeDeps({ + files = {} as Record, + execs = [] as Array<{ code: number; stdout: string }>, + env = {} as Record, +} = {}) { + const calls: string[][] = []; + const outputs: Array> = []; + const summaries: string[] = []; + const deps = { + env, + exec(argv: string[], _opts?: { inherit?: boolean }) { + calls.push(argv); + const next = execs.shift(); + if (!next) throw new Error(`unexpected exec: ${argv.join(" ")}`); + return next; + }, + readFile(p: string) { + if (p in files) return files[p]; + throw new Error(`ENOENT: ${p}`); + }, + exists: (p: string) => p in files, + writeSummary: (md: string) => summaries.push(md), + writeOutputs: (o: Record) => outputs.push(o), + log: (_msg: string) => {}, + }; + return { deps, calls, outputs, summaries }; +} + +const MB_OK = { code: 0, stdout: "abc123\n" }; +const diffOf = (raw: string) => ({ code: 0, stdout: raw }); + +test("main: missing merge-base is an infra failure naming fetch-depth", () => { + const { deps, outputs } = fakeDeps({ execs: [{ code: 128, stdout: "" }], env: { MUTATION_GATE_BASE: "origin/main" } }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].decision).toBe("infra"); + expect(outputs[0].summary).toContain("fetch-depth: 0"); +}); + +test("main: no mutable changes passes empty without spawning stryker", () => { + const { deps, calls, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF }, + execs: [MB_OK, diffOf("M\0README.md\0")], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("pass-empty"); + expect(calls.length).toBe(2); +}); + +test("main: over-threshold loud-skips green without spawning stryker; force runs it", () => { + const bigFile = "x\n".repeat(900); + const base = { + files: { "stryker.conf.json": CONF, "src/engine/index.ts": bigFile }, + execs: [MB_OK, diffOf("M\0src/engine/index.ts\0")], + }; + const skip = fakeDeps({ ...base, env: { MUTATION_GATE_BASE: "origin/main" } }); + expect(main(skip.deps)).toBe(EXIT.ok); + expect(skip.outputs[0].decision).toBe("skip-size"); + expect(skip.calls.length).toBe(2); + + const forced = fakeDeps({ + files: { + "stryker.conf.json": CONF, + "src/engine/index.ts": bigFile, + "reports/mutation/mutation.json": JSON.stringify(makeReport({ "src/engine/index.ts": [{ mutator: "X", status: "Killed" }] })), + }, + execs: [MB_OK, diffOf("M\0src/engine/index.ts\0"), { code: 0, stdout: "" }], + env: { MUTATION_GATE_BASE: "origin/main", MUTATION_GATE_FORCE: "1" }, + }); + expect(main(forced.deps)).toBe(EXIT.ok); + expect(forced.outputs[0].decision).toBe("pass"); +}); + +test("main: a clean run passes and the stryker argv carries --mutate and --reporters", () => { + const { deps, calls, outputs } = fakeDeps({ + files: { + "stryker.conf.json": CONF, + "src/engine/dispatch.ts": "a\nb\n", + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/dispatch.ts": [{ mutator: "X", status: "Killed" }] }), + ), + }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0"), { code: 0, stdout: "" }], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("pass"); + const stryker = calls[2]; + expect(stryker.slice(0, 2)).toEqual(["node_modules/.bin/stryker", "run"]); + expect(stryker).toContain("--mutate"); + expect(stryker[stryker.indexOf("--mutate") + 1]).toBe("src/engine/dispatch.ts"); + expect(stryker[stryker.indexOf("--reporters") + 1]).toBe("clear-text,progress,json,html"); +}); + +test("main: survivors fail the gate with exit 1 and the mutant named", () => { + const { deps, outputs } = fakeDeps({ + files: { + "stryker.conf.json": CONF, + "src/engine/dispatch.ts": "a\n", + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/dispatch.ts": [{ mutator: "EqualityOperator", status: "Survived", line: 7 }] }), + ), + }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0"), { code: 0, stdout: "" }], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.gateFail); + expect(outputs[0].decision).toBe("fail"); + expect(outputs[0].summary).toContain("src/engine/dispatch.ts:7"); +}); + +test("main: stryker exiting without a report is infra, not a verdict", () => { + const { deps, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/dispatch.ts": "a\n" }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0"), { code: 1, stdout: "" }], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].decision).toBe("infra"); + expect(outputs[0].summary).toContain("without writing"); +}); + +test("main: an unsupported conf glob is refused with the GLOBS remedy", () => { + const { deps, outputs } = fakeDeps({ + files: { "stryker.conf.json": JSON.stringify({ mutate: ["src/!(*.test).ts"] }) }, + execs: [MB_OK, diffOf("M\0src/a.ts\0")], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].summary).toContain("MUTATION_GATE_GLOBS"); +}); + +test("main: skip label loud-skips a small PR", () => { + const { deps, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/dispatch.ts": "a\n" }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0")], + env: { MUTATION_GATE_BASE: "origin/main", MUTATION_GATE_SKIP: "1" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("skip-label"); +}); + +test("resolveDefaultBase prefers origin/HEAD, falls back to main", () => { + const viaHead = fakeDeps({ execs: [{ code: 0, stdout: "refs/remotes/origin/trunk\n" }] }); + expect(resolveDefaultBase(viaHead.deps)).toBe("origin/trunk"); + const noHead = fakeDeps({ execs: [{ code: 1, stdout: "" }] }); + expect(resolveDefaultBase(noHead.deps)).toBe("main"); +}); + +test("readMutateGlobs reads the conf and rejects a missing mutate array", () => { + const ok = fakeDeps({ files: { "stryker.conf.json": CONF } }); + expect(readMutateGlobs(ok.deps, "stryker.conf.json")).toEqual(["src/engine/**/*.ts", "!src/engine/**/*.test.ts"]); + const bad = fakeDeps({ files: { "stryker.conf.json": "{}" } }); + expect(() => readMutateGlobs(bad.deps, "stryker.conf.json")).toThrow("MUTATION_GATE_GLOBS"); +}); + +test("realDeps exec runs a real command and captures stdout", () => { + const d = realDeps(); + const r = d.exec(["git", "--version"]); + expect(r.code).toBe(0); + expect(r.stdout).toContain("git version"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: the 11 new tests fail; all earlier tests pass. + +- [ ] **Step 3: Write the implementation** + +Add these imports to `scripts/mutation-gate.mjs`, immediately after the header comment block (before the first export): + +```js +import { spawnSync } from "node:child_process"; +import { appendFileSync, existsSync, readFileSync, realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +``` + +Append at the bottom: + +```js +export const EXIT = Object.freeze({ ok: 0, gateFail: 1, infra: 2 }); + +export function resolveDefaultBase(deps) { + const head = deps.exec(["git", "symbolic-ref", "-q", "refs/remotes/origin/HEAD"]); + if (head.code === 0 && head.stdout.trim() !== "") { + return head.stdout.trim().replace(/^refs\/remotes\//, ""); + } + return "main"; +} + +export function readMutateGlobs(deps, configPath) { + const conf = JSON.parse(deps.readFile(configPath)); + if (!Array.isArray(conf.mutate) || conf.mutate.length === 0) { + throw new Error(`mutation-gate: no "mutate" array in ${configPath}; set MUTATION_GATE_GLOBS`); + } + return conf.mutate; +} + +function finish(deps, decision, summaryMd, exitCode) { + deps.log(summaryMd); + deps.writeSummary(summaryMd); + deps.writeOutputs({ decision, summary: summaryMd }); + return exitCode; +} + +export function main(deps) { + const d = deps ?? realDeps(); + try { + const cfg = readConfig(d.env); + if (cfg.mode !== "changed" && cfg.mode !== "incremental") { + throw new Error(`mutation-gate: unknown MUTATION_GATE_MODE "${cfg.mode}"`); + } + const base = cfg.base ?? resolveDefaultBase(d); + const mb = d.exec(["git", "merge-base", base, "HEAD"]); + if (mb.code !== 0) { + return finish(d, "infra", renderInfra( + `no merge-base between "${base}" and HEAD. In CI, check out with fetch-depth: 0 so the base branch history is present.`, + ), EXIT.infra); + } + const diff = d.exec(["git", "diff", "--name-status", "-z", "-M", mb.stdout.trim(), "HEAD"]); + if (diff.code !== 0) { + return finish(d, "infra", renderInfra("git diff --name-status failed"), EXIT.infra); + } + const { changed, deleted } = parseNameStatusZ(diff.stdout); + const globs = cfg.globsOverride ?? readMutateGlobs(d, cfg.configPath); + const files = selectMutateSet({ changed, deleted, globs, testSiblings: cfg.testSiblings, exists: d.exists }); + const totalLines = files.reduce((n, f) => n + countLines(d.readFile(f)), 0); + const decision = decide({ files, totalLines, thresholdLines: cfg.thresholdLines, force: cfg.force, skip: cfg.skip }); + if (decision !== "run") { + return finish(d, decision, renderSkip({ decision, files, totalLines, thresholdLines: cfg.thresholdLines }), EXIT.ok); + } + let strykerArgs; + if (cfg.mode === "incremental") { + if (cfg.requireBaseline && !d.exists(cfg.incrementalFile)) { + return finish(d, "skip-no-baseline", + renderSkip({ decision: "skip-no-baseline", files, totalLines, thresholdLines: cfg.thresholdLines }), EXIT.ok); + } + strykerArgs = [...cfg.strykerCmd, "--incremental", "--incrementalFile", cfg.incrementalFile, + "--reporters", "clear-text,progress,json,html", ...cfg.extraArgs]; + } else { + strykerArgs = [...cfg.strykerCmd, "--mutate", files.join(","), + "--reporters", "clear-text,progress,json,html", ...cfg.extraArgs]; + } + const run = d.exec(strykerArgs, { inherit: true }); + if (!d.exists(cfg.reportPath)) { + return finish(d, "infra", renderInfra( + `stryker exited ${run.code} without writing ${cfg.reportPath}. Read the run log above; this may be a crash, not a test-strength verdict.`, + ), EXIT.infra); + } + const result = interpretReport(JSON.parse(d.readFile(cfg.reportPath)), cfg.breakScore); + const summaryMd = renderResult({ files, result, breakScore: cfg.breakScore }); + return finish(d, result.verdict, summaryMd, result.verdict === "pass" ? EXIT.ok : EXIT.gateFail); + } catch (err) { + return finish(d, "infra", renderInfra(err.message), EXIT.infra); + } +} + +export function realDeps() { + return { + env: process.env, + exec(argv, opts = {}) { + const r = spawnSync(argv[0], argv.slice(1), { + encoding: "utf8", + stdio: opts.inherit ? ["ignore", "inherit", "inherit"] : ["ignore", "pipe", "pipe"], + maxBuffer: 64 * 1024 * 1024, + }); + if (r.error) throw r.error; + return { code: r.status ?? 1, stdout: r.stdout ?? "" }; + }, + readFile: (p) => readFileSync(p, "utf8"), + exists: existsSync, + writeSummary(md) { + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${md}\n`); + }, + writeOutputs(outputs) { + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, formatGithubOutputs(outputs)); + }, + log: (msg) => console.error(msg), + }; +} + +const isCliEntry = (() => { + if (!process.argv[1]) return false; + try { + return realpathSync(process.argv[1]) === fileURLToPath(import.meta.url); + } catch { + return false; + } +})(); +if (isCliEntry) process.exit(main()); +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: 45 pass, 0 fail. + +- [ ] **Step 5: Self-run on the repo (pass-empty smoke test)** + +From the repo root on `mutation-pr-gate` (which touches only docs): + +```bash +nvm use default # Node >= 20 on PATH +MUTATION_GATE_BASE=main node scripts/mutation-gate.mjs; echo "exit=$?" +``` + +Expected: the pass-empty summary printed to stderr and `exit=0`. Also confirm the guard: `bun -e 'await import("./scripts/mutation-gate.mjs"); console.log("import-only ok")'` prints `import-only ok` and exits 0 (no gate run on import). + +- [ ] **Step 6: Full-suite gate and commit** + +```bash +bun test; echo "exit=$?" # expect exit=0 +git add scripts/mutation-gate.mjs scripts/mutation-gate.test.ts +git commit -m "feat: mutation-gate main orchestration, real deps, CLI entry" +``` + +--- + +### Task 8: Incremental mode + +**Files:** +- Modify: `scripts/mutation-gate.test.ts` (append; the `main` from Task 7 already implements the branch, so these tests pin it) + +**Interfaces:** +- Consumes: `main`, `EXIT`, `makeReport`, `fakeDeps` (Task 7 test block), `CONF`. +- Produces: pinned behavior for `MUTATION_GATE_MODE=incremental`; no new exports. + +- [ ] **Step 1: Write the tests** (append) + +```ts +test("incremental: missing baseline loud-skips without spawning stryker", () => { + const { deps, calls, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/dispatch.ts": "a\n" }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0")], + env: { MUTATION_GATE_BASE: "origin/main", MUTATION_GATE_MODE: "incremental" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("skip-no-baseline"); + expect(calls.length).toBe(2); +}); + +test("incremental: with a baseline, argv has --incremental and no --mutate", () => { + const { deps, calls, outputs } = fakeDeps({ + files: { + "stryker.conf.json": CONF, + "src/engine/dispatch.ts": "a\n", + "reports/stryker-incremental.json": "{}", + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/dispatch.ts": [{ mutator: "X", status: "Killed" }] }), + ), + }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0"), { code: 0, stdout: "" }], + env: { MUTATION_GATE_BASE: "origin/main", MUTATION_GATE_MODE: "incremental" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("pass"); + const stryker = calls[2]; + expect(stryker).toContain("--incremental"); + expect(stryker[stryker.indexOf("--incrementalFile") + 1]).toBe("reports/stryker-incremental.json"); + expect(stryker).not.toContain("--mutate"); +}); + +test("incremental: REQUIRE_BASELINE=false runs without a baseline", () => { + const { deps, outputs } = fakeDeps({ + files: { + "stryker.conf.json": CONF, + "src/engine/dispatch.ts": "a\n", + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/dispatch.ts": [{ mutator: "X", status: "Killed" }] }), + ), + }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0"), { code: 0, stdout: "" }], + env: { + MUTATION_GATE_BASE: "origin/main", + MUTATION_GATE_MODE: "incremental", + MUTATION_GATE_REQUIRE_BASELINE: "false", + }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("pass"); +}); + +test("an unknown mode is an infra failure", () => { + const { deps, outputs } = fakeDeps({ env: { MUTATION_GATE_MODE: "yolo" } }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].summary).toContain('unknown MUTATION_GATE_MODE "yolo"'); +}); +``` + +- [ ] **Step 2: Run tests to verify they pass** (the implementation landed in Task 7; these pin it) + +Run: `bun test scripts/mutation-gate.test.ts` +Expected: 49 pass, 0 fail. If any of the four fail, fix `main()` in `scripts/mutation-gate.mjs`, not the tests. + +- [ ] **Step 3: Full-suite gate and commit** + +```bash +bun test; echo "exit=$?" # expect exit=0 +git add scripts/mutation-gate.test.ts +git commit -m "test: pin mutation-gate incremental mode" +``` + +--- + +### Task 9: The workflow and the labels + +**Files:** +- Create: `.github/workflows/mutation.yml` + +**Interfaces:** +- Consumes: the script's env knobs, `decision`/`summary` outputs, exit codes (Tasks 4, 6, 7). +- Produces: the `mutation` check; `mutate-force`/`mutate-skip` labels. Task 10 exercises them. + +- [ ] **Step 1: Write the workflow** + +Create `.github/workflows/mutation.yml`: + +```yaml +# Portable mutation PR gate (StrykerJS). To adopt in another repo: +# 1. copy scripts/mutation-gate.mjs and this file; +# 2. replace the toolchain setup below (setup-bun + bun install) with your own +# (e.g. drop setup-bun, use actions/setup-node + `npm ci`); +# 3. tune the MUTATION_GATE_* env knobs (full table in the script header); +# 4. add the `mutation` check to branch protection; create the +# mutate-force / mutate-skip labels. +# Modes: changed (default: mutates the PR's changed files) or incremental +# (MUTATION_GATE_MODE: incremental; needs a baseline, see the commented-out +# job at the bottom, plus an actions/cache restore step in this job). +name: mutation +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, labeled, unlabeled] +concurrency: + group: mutation-${{ github.event.pull_request.number }} + cancel-in-progress: true +permissions: + contents: read + pull-requests: write # sticky comment +jobs: + mutation: + runs-on: ubuntu-latest + timeout-minutes: 15 # backstop against a mispredicted run + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} # the PR as authored, not the synthetic merge ref + fetch-depth: 0 # all branches: merge-base needs the real base branch + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.14" # same pin rationale as gates (bunfig coverage semantics) + - uses: actions/setup-node@v5 + with: + node-version: "24" # Stryker CLI host (D-010); >=20 required, 20 is EOL + - run: bun install --frozen-lockfile + - name: gate + id: gate + env: + MUTATION_GATE_BASE: origin/${{ github.event.pull_request.base.ref }} # the branch, never the payload SHA + MUTATION_GATE_FORCE: ${{ contains(github.event.pull_request.labels.*.name, 'mutate-force') && '1' || '' }} + MUTATION_GATE_SKIP: ${{ contains(github.event.pull_request.labels.*.name, 'mutate-skip') && '1' || '' }} + # MUTATION_GATE_EXTRA_ARGS: "--concurrency 4" # set from the Task 10 measurement + run: node scripts/mutation-gate.mjs + - name: sticky comment + if: always() && steps.gate.outputs.decision != '' + continue-on-error: true # fork PRs get a read-only token; the verdict is the gate step's alone + env: + GH_TOKEN: ${{ github.token }} + DECISION: ${{ steps.gate.outputs.decision }} + SUMMARY: ${{ steps.gate.outputs.summary }} + PR: ${{ github.event.pull_request.number }} + run: | + BODY_FILE="$(mktemp)" + printf '\n\n%s\n' "$SUMMARY" > "$BODY_FILE" + EXISTING="$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty')" + case "$DECISION" in + pass|pass-empty) [ -z "$EXISTING" ] && exit 0 ;; # quiet pass: only update an existing comment + esac + if [ -n "$EXISTING" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${EXISTING}" -F body=@"$BODY_FILE" + else + gh api -X POST "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" -F body=@"$BODY_FILE" + fi + - name: report artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: mutation-report + path: reports/mutation/ + retention-days: 14 + if-no-files-found: ignore +# baseline: # incremental-mode adopters: produce/refresh the baseline on pushes to main +# # (also add `push: { branches: [main] }` to `on:` and an `if: github.event_name == 'push'` guard) +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v7 +# - +# - uses: actions/cache@v4 +# with: +# path: reports/stryker-incremental.json +# key: stryker-incremental-${{ github.sha }} +# restore-keys: stryker-incremental- +# - run: node_modules/.bin/stryker run --incremental --incrementalFile reports/stryker-incremental.json +``` + +- [ ] **Step 2: Sanity-check the YAML parses** + +```bash +bun -e 'const { parse } = await import("yaml"); parse(await Bun.file(".github/workflows/mutation.yml").text()); console.log("yaml ok")' +``` + +Expected: `yaml ok`. + +- [ ] **Step 3: Create the labels** + +```bash +gh label create mutate-force --color B60205 --description "mutation gate: run and block even over the size threshold" +gh label create mutate-skip --color C5DEF5 --description "mutation gate: wave off the gate for this PR" +gh label list | grep mutate +``` + +Expected: both labels listed. + +- [ ] **Step 4: Commit and push (the rehearsal branches off this)** + +```bash +git add .github/workflows/mutation.yml +git commit -m "ci: mutation PR gate workflow with label overrides and sticky comment" +git push -u origin mutation-pr-gate +``` + +--- + +### Task 10: E2E rehearsal and the concurrency measurement + +This task exercises the real check on a scratch PR. Nothing from this branch is merged; it exists to prove red/green/skip/labels/artifact/comment and to measure mutants-per-minute. `pull_request` workflows run from the head branch, so the scratch PR carries the new workflow. Verify every outcome by check conclusion or exit code, never log prose. The `gates` check may go red on the scratch PR while the probe is untested; that is expected and irrelevant here. + +**Files:** +- Scratch branch `mutation-gate-rehearsal` (off `mutation-pr-gate`): temporary edits to `src/engine/instances.ts`, `src/engine/instances.test.ts`, `src/engine/index.ts`, `.github/workflows/mutation.yml`. All discarded when the PR closes. +- Create: `/tmp/claude-1000/-home-nn-Projects-offbook/c85f6d34-1e52-4951-b55d-61ccfd1f864d/scratchpad/mutation-gate-measurements.md` (the numbers Task 11 needs). + +**Interfaces:** +- Consumes: the pushed workflow (Task 9). +- Produces: measured mutants-per-minute at concurrency 1 and 4; the decision which `MUTATION_GATE_EXTRA_ARGS` ships; recorded in the scratchpad file for Task 11. + +- [ ] **Step 1: Open the rehearsal PR with a planted survivor** + +```bash +git checkout -b mutation-gate-rehearsal +cat >> src/engine/instances.ts <<'EOF' + +export function rehearsalProbe(n: number): string { + return n > 10 ? "big" : "small"; +} +EOF +git add src/engine/instances.ts +git commit -m "rehearsal: plant an untested probe (never merge)" +git push -u origin mutation-gate-rehearsal +gh pr create --base main --head mutation-gate-rehearsal --draft \ + --title "rehearsal: mutation gate (never merge)" \ + --body "Scratch PR exercising the mutation gate end to end. Close without merging." +``` + +- [ ] **Step 2: Verify the red run names the probe's mutants** + +```bash +gh run list --workflow=mutation --branch mutation-gate-rehearsal --limit 1 # note the run id once it appears +gh run watch --exit-status; echo "exit=$?" +``` + +Expected: `exit=1` (the check fails). Then confirm the verdict content and the artifact: + +```bash +gh run view --json conclusion --jq .conclusion # expect: failure +gh api "repos/{owner}/{repo}/actions/runs//artifacts" --jq '.artifacts[].name' # expect: mutation-report +gh pr view --json comments --jq '.comments[].body' | grep -c 'mutation-gate' # expect: 1 +``` + +The job summary (run page) must list `src/engine/instances.ts:` mutants (ConditionalExpression / EqualityOperator / StringLiteral) with the kill-or-annotate instruction. + +- [ ] **Step 3: Exercise the label overrides while red** + +```bash +gh pr edit --add-label mutate-skip +# wait for the labeled-event run: +gh run list --workflow=mutation --branch mutation-gate-rehearsal --limit 1 +gh run watch --exit-status; echo "exit=$?" # expect exit=0 (loud skip is green) +gh run view --json conclusion --jq .conclusion # expect: success +gh pr view --json comments --jq '[.comments[] | select(.body | startswith(""))] | length' # expect: 1 (upserted, not duplicated) +gh pr edit --remove-label mutate-skip # triggers the red run again +``` + +- [ ] **Step 4: Exercise skip-size cheaply, then restore** + +On the rehearsal branch, add `MUTATION_GATE_THRESHOLD_LINES: "1"` under the gate step's `env:` in `.github/workflows/mutation.yml`, commit (`rehearsal: force skip-size`), push. Expected: the new run is green with decision `skip-size` in the job summary. Then `git revert HEAD && git push` to restore. + +- [ ] **Step 5: Kill the probe, verify green** + +```bash +cat >> src/engine/instances.test.ts <<'EOF' + +// rehearsal probe kill (never merged) +import { rehearsalProbe } from "./instances.ts"; + +test("rehearsalProbe boundary and literals", () => { + expect(rehearsalProbe(11)).toBe("big"); + expect(rehearsalProbe(10)).toBe("small"); +}); +EOF +bun test src/engine/instances.test.ts # judge by fail count: expect 0 fail +git add src/engine/instances.test.ts +git commit -m "rehearsal: kill the probe" +git push +``` + +Note: `instances.test.ts` already imports `test`/`expect` from `bun:test`, so the appended block only imports `rehearsalProbe`; an `import` declaration is legal at any top-level position in an ESM file, so the append works as-is (this is a scratch branch, tidiness is optional). Expected: next mutation run green, conclusion `success`, sticky comment updated to the pass state, still exactly one comment. + +- [ ] **Step 6: Measure concurrency 1 vs 4 on a big file** + +Temporarily (still on the rehearsal branch): flip the artifact step to `if: always()` and append a no-op line (`// rehearsal touch`) to `src/engine/index.ts` so the biggest engine file (363 lines) enters the mutate set. Commit, push, let the run finish (expect green; this is the concurrency-1 measurement). Then set `MUTATION_GATE_EXTRA_ARGS: "--concurrency 4"` in the workflow env, commit, push, let it finish (the concurrency-4 measurement). + +For each of the two runs record into `mutation-gate-measurements.md` in the scratchpad: + +```bash +SCRATCH=/tmp/claude-1000/-home-nn-Projects-offbook/c85f6d34-1e52-4951-b55d-61ccfd1f864d/scratchpad +gh run view --json jobs \ + --jq '.jobs[0].steps[] | select(.name == "gate") | {startedAt, completedAt}' +gh run download --name mutation-report --dir "$SCRATCH/run-" +bun -e 'const r = JSON.parse(await Bun.file("/run-/mutation.json").text()); + let t = 0, k = {}; for (const f of Object.values(r.files)) for (const m of f.mutants) { t++; k[m.status]=(k[m.status]??0)+1 } + console.log(JSON.stringify({ total: t, byStatus: k }))' +``` + +Record: gate-step wall time, total mutants, per-status counts, computed mutants/minute for each run. **The two runs' per-status counts must be identical**; if concurrency 4 diverges (perTest coverage mis-correlation) or crashes, record that and the decision is concurrency 1. Otherwise the faster setting wins. Derive the threshold: `sustainable = rate_winner (mutants/min) x 12 (min) / 0.5 (mutants/line)`, rounded to the nearest 100; if it is within a factor of 2 of 800, keep 800. + +- [ ] **Step 7: Close out the rehearsal** + +```bash +gh pr close --delete-branch +git checkout mutation-pr-gate +``` + +Then apply the measurement's outcome on `mutation-pr-gate`: +- If concurrency 4 won: uncomment/set `MUTATION_GATE_EXTRA_ARGS: "--concurrency 4"` in `.github/workflows/mutation.yml`. +- If the derived threshold differs from 800 by more than 2x: change `thresholdLines` in `DEFAULTS` in `scripts/mutation-gate.mjs`, the `800` expectations in `scripts/mutation-gate.test.ts` (readConfig defaults test), and the Threshold rationale numbers in the spec. + +```bash +bun test; echo "exit=$?" # expect exit=0 +git add -A +git commit -m "ci: apply the measured mutation-gate concurrency/threshold" # only if anything changed +git push +``` + +--- + +### Task 11: D-027, AGENTS.md, spec status + +**Files:** +- Modify: `DECISIONS.md` (append D-027 at the end) +- Modify: `AGENTS.md` (two working-notes lines) +- Modify: `docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md` (Status line + measured numbers) + +**Interfaces:** +- Consumes: the measurements file from Task 10. + +- [ ] **Step 1: Append D-027 to `DECISIONS.md`** + +Append after the last entry (D-026), filling the four `<...>` slots from `/tmp/claude-1000/-home-nn-Projects-offbook/c85f6d34-1e52-4951-b55d-61ccfd1f864d/scratchpad/mutation-gate-measurements.md`: + +```markdown +### D-027: A changed-file mutation gate on PRs; full campaigns stay manual +**Date**: 2026-08-04 +**What**: A second required check, `mutation` (`.github/workflows/mutation.yml` + dependency-free `scripts/mutation-gate.mjs`, the two-file portable unit): on every PR targeting main, mutate exactly the changed files that match `stryker.conf.json`'s `mutate` globs, plus sibling sources of changed or deleted test files (existence checked at HEAD), and fail below score 100. Score is Stryker's own metric: detected = Killed + Timeout, undetected = Survived + NoCoverage; Ignored and error statuses stay out of the verdict; zero valid mutants scores 100; the verdict is computed from the JSON report, never Stryker's exit code (thresholds.break defaults to null, so the exit code carries nothing). Above MUTATION_GATE_THRESHOLD_LINES ( summed whole-file lines) the gate loud-skips: green check, step summary, sticky PR comment nudging a local run; labels `mutate-force`/`mutate-skip` override in both directions, force wins. Diff base: the PR head SHA is checked out and merge-based against `origin/`, never the payload base SHA or the synthetic merge ref (stale-payload diffs otherwise fail open via loud-skip). Report artifact uploads on failure only. An incremental mode ships for adopting projects (baseline required by default, loud-skip without it). Measured on ubuntu-latest (Task 10 rehearsal, 2026-08-04): mutants over src/engine/index.ts at concurrency 1 in (/min) vs concurrency 4 in (/min), per-status counts ; shipping . +**Why**: Amends D-010 ("run manually, never a gate") and D-017 ("mutation testing is excluded from CI in any form"): both stances priced a full-campaign gate, and a changed-file gate prices per-PR work instead, catching test-strength regressions at merge time where they are cheapest. Whole-file (not changed-line) mutation keeps the D-011 ratchet reading: every file a PR touches ends the PR mutation-clean. Loud-skip keeps the obligation visible on large PRs without holding the check hostage; the label escape hatches keep "mandatory" from eroding at the first heuristic misfire. +**Mitigations / notes**: The gate does not police annotation quality: `Ignored` is excluded, so a `// Stryker disable` comment silences a survivor, and the unobservability argument stays human review (D-011). A Stryker or runner bump can change the mutant set; run a full campaign (`bun run mutate`) after any such bump before engine PRs resume, or the drift lands on the next innocent PR. Test-helper and config changes do not trigger the gate (accepted residual; incremental mode is the answer for projects that care). Widening the `mutate` globs beyond `src/engine/` stays module-by-module, each behind its own kill-or-annotate campaign. +**From**: docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md (brainstorm dialog + adversarial agent review, 2026-08-04) +**Folds into**: scripts/mutation-gate.mjs, scripts/mutation-gate.test.ts, .github/workflows/mutation.yml, AGENTS.md (working notes), main ruleset (`mutation` required check) +``` + +- [ ] **Step 2: Update the two AGENTS.md working-notes lines** + +In `AGENTS.md`, replace the sentence `Mutation testing is manual and never a gate.` (inside the `bun run mutate` bullet) with: + +``` +Mutation testing is manual full campaigns plus a changed-file PR gate (`.github/workflows/mutation.yml` + `scripts/mutation-gate.mjs`, D-027): small PRs are gated on zero undetected mutants in touched engine files; large PRs loud-skip with a sticky comment (labels `mutate-force`/`mutate-skip` override); after any Stryker/runner bump, run a full campaign before engine PRs resume. +``` + +And in the CI bullet, replace `Mutation testing stays out of CI (D-017).` with: + +``` +The `mutation` required check runs the changed-file gate on PRs (D-027); full campaigns stay out of CI. +``` + +- [ ] **Step 3: Update the spec's Status line and measured numbers** + +In `docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md`: change the `**Status**:` line to `implemented 2026-08-04 (see D-027)` and replace the Threshold rationale's "provisional until that measurement" sentence with the measured rates and the shipped setting (same numbers as D-027). + +- [ ] **Step 4: Run the full gate set and commit** + +```bash +bun scripts/check-docs.ts; echo "exit=$?" # expect exit=0 (D-027 keeps ids contiguous) +bun run lint; echo "exit=$?" # expect exit=0 +bun run typecheck; echo "exit=$?" # expect exit=0 +bun test; echo "exit=$?" # expect exit=0 +git add DECISIONS.md AGENTS.md docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md +git commit -m "docs: D-027 mutation PR gate; amend D-010/D-017 working notes" +git push +``` + +--- + +### Task 12: Ruleset, PR, final verification + +**Files:** none (GitHub state + the PR). + +- [ ] **Step 1: Add `mutation` to the main ruleset's required checks** + +```bash +SCRATCH=/tmp/claude-1000/-home-nn-Projects-offbook/c85f6d34-1e52-4951-b55d-61ccfd1f864d/scratchpad +RID=$(gh api repos/{owner}/{repo}/rulesets --jq '.[0].id') +gh api "repos/{owner}/{repo}/rulesets/$RID" \ + --jq '{name, target, enforcement, bypass_actors, conditions, rules}' > "$SCRATCH/ruleset.json" +jq '(.rules |= map(if .type == "required_status_checks" + then (.parameters.required_status_checks += [{"context": "mutation"}]) else . end))' \ + "$SCRATCH/ruleset.json" > "$SCRATCH/ruleset-new.json" +gh api -X PUT "repos/{owner}/{repo}/rulesets/$RID" --input "$SCRATCH/ruleset-new.json" +gh api "repos/{owner}/{repo}/rulesets/$RID" --jq \ + '.rules[] | select(.type == "required_status_checks").parameters.required_status_checks[].context' +``` + +Expected final output: `gates` and `mutation`. (If more than one ruleset exists, pick the one targeting `~DEFAULT_BRANCH` when reading `RID`.) + +- [ ] **Step 2: Open the real PR** + +```bash +gh pr create --base main --head mutation-pr-gate \ + --title "feat: changed-file mutation gate as a second required PR check (D-027)" \ + --body "Implements docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md: scripts/mutation-gate.mjs + .github/workflows/mutation.yml (the portable two-file unit), changed-file mode gating on zero undetected mutants, loud-skip over the size threshold with mutate-force/mutate-skip label overrides, incremental mode for adopters, report artifact on failure. Rehearsed end to end on a scratch PR (red on a planted survivor, green after the kill, both labels, skip-size, sticky comment upsert, artifact); concurrency and threshold set from the measured rate (D-027)." +``` + +- [ ] **Step 3: Verify both checks on the real PR** + +This PR touches `scripts/`, `.github/`, and docs, no engine files, so the expected `mutation` outcome is a green `pass-empty` with no sticky comment. + +```bash +gh pr checks --watch; echo "exit=$?" # expect exit=0, both gates and mutation green +gh run view --json conclusion --jq .conclusion # expect: success +``` + +Confirm in the run's job summary that the decision is `pass-empty`. Merge is the user's call; stop here and report. + +--- + +## Verification checklist (mirrors the spec's Verification section) + +- [ ] Full `bun test` exit 0 with all mutation-gate unit tests in (rename fixture, deleted-test sibling, both-deleted, full status enum, zero-valid, heredoc outputs). +- [ ] Rehearsal PR: red run named the planted mutants; green after the kill; `mutate-skip` flipped red to green without a push; `mutate-force` ran an over-threshold change; skip-size exercised via `THRESHOLD_LINES=1`; exactly one sticky comment throughout; `mutation-report` artifact on the red run. +- [ ] Concurrency 1 vs 4 measured, per-status counts compared, winner + threshold recorded in D-027 and the spec. +- [ ] `check-docs`, `lint`, `typecheck`, full `bun test` all exit 0 after the doc edits. +- [ ] Ruleset lists `gates` and `mutation`; the real PR shows both green with `mutation` = `pass-empty`. diff --git a/docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md b/docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md new file mode 100644 index 0000000..455d160 --- /dev/null +++ b/docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md @@ -0,0 +1,155 @@ +# Changed-file mutation gate for PRs: design + +**Date**: 2026-08-04 +**Status**: implemented 2026-08-05 (see D-027) +**Provenance**: brainstorm dialog 2026-08-04. Engine size figures measured during the dialog (`wc -l`; D-011 campaign counts). Stryker CLI spellings verified against the installed `@stryker-mutator/core` 9.6.1 during the adversarial review: `--mutate`, `--incremental`, `--incrementalFile`, and `--reporters` exist as comma-split CLI options; CLI-supplied arrays replace the conf's wholesale (`@stryker-mutator/util` deepMerge), so the conf file is never modified; the JSON reporter's default output is exactly `reports/mutation/mutation.json`; `thresholds.break` defaults to null, so Stryker exits 0 with survivors, which makes report interpretation (not exit-code gating) mandatory, not merely preferable. + +## Problem + +Mutation testing is manual and never a gate (D-010), and D-017 excludes it from CI in any form. That leaves test-strength regressions invisible at merge time exactly where they are cheapest to catch: small PRs touching already-clean modules. A full-campaign CI gate is out of the question on GitHub Actions minutes grounds, and large refactors would hold a required check hostage for tens of minutes. Wanted: a PR gate that mutates only what the PR touched, is mandatory for small PRs, steps aside loudly for large ones, and is portable to other StrykerJS projects with minimal ceremony. + +## Scope and stances (from the design dialog) + +- **Gate width = the project's own `mutate` globs** (engine-only in offbook today). Widening the globs is a deferred, module-by-module follow-up: each module's glob lands only after that module passes a D-011-style kill-or-annotate campaign, so "survivor = red" stays true forever. Repo-wide expansion now was rejected (it would require six-plus campaigns as a prerequisite). +- **Pass = zero undetected mutants over the mutated set** (score 100, the D-011 reading). The break threshold is a knob so adopting projects can gate at 80 or 90 while they climb. +- **Large PRs get a loud skip**: the check goes green, but the job writes a step summary and a sticky PR comment naming the measured size, the threshold, and the nudge to run `bun run mutate` locally before merging. Two label overrides: `mutate-force` (run and block even over the threshold) and `mutate-skip` (a maintainer waves the gate off when the heuristic misfires). Force wins if both are present. +- **Portability shape: two copied files** (script + workflow), dependency-free, no cross-repo `uses:` coupling. A composite action in offbook and a shared-repo reusable workflow were considered and rejected for reachability coupling; either can wrap this script later without rework. +- **Mechanism: changed-file `--mutate` override** (whole files) for offbook, with **Stryker incremental mode as a script mode for adopters** (it also catches test-only weakening, at the price of baseline infrastructure). Changed-line ranges were rejected: weaker assertion (a PR deleting an assertion that covered unchanged neighboring lines would pass), fiddly diff mapping around moves, and unverified multi-range support. + +## Components + +1. **`scripts/mutation-gate.mjs`** (portable piece 1): single-file, dependency-free, plain JS, runs under Node 18+ or Bun, imports only `node:` builtins and nothing from the repo. All logic lives here. Pure functions are exported for testing; the CLI entry is guarded by comparing `realpath(process.argv[1])` against `fileURLToPath(import.meta.url)`. The guard is explicitly **not** `import.meta.main`: Node below 24 lacks it, and the failure mode is the script silently no-op-ing under `node`, a green gate that ran nothing. +2. **`.github/workflows/mutation.yml`** (portable piece 2): a separate workflow, not a new `ci.yml` job, because it needs `labeled`/`unlabeled` trigger types (label toggles re-evaluate the gate without a push) and because the portable unit should be exactly two files. One job, check name `mutation`, added to the `main` ruleset as a second required check. The `gates` workflow is untouched. +3. **Doc integration (offbook-only)**: new decision **D-027** amending D-010 ("never a gate") and D-017 ("excluded from CI in any form"), plus the AGENTS.md working-notes update. **No new R-###**: like D-017's CI, this is repo infrastructure, decision-only; `check-docs` is unaffected. +4. **`scripts/mutation-gate.test.ts`** (offbook-only): unit tests over the exported functions (glob matcher, diff parsing, sibling derivation, size decision, report interpretation against fixture JSON). The bunfig per-file coverage floors judge the `.mjs` (verified empirically: an imported `.mjs` is instrumented, judged per-file, and a floor miss is exit 1 with zero failed tests and no message), and `functions = 0.64` means most of the script must execute under tests, not just the pure helpers. Therefore `main()` and the I/O edges take injectable seams (spawn, env, fs, output writers) and are unit-tested through them; the un-injected defaults are exercised by the e2e rehearsal. + +## The script + +Configuration is environment variables only (CI-native, no arg parser), prefixed `MUTATION_GATE_`: + +| Knob | Default | Meaning | +|---|---|---| +| `MODE` | `changed` | `changed` (offbook) or `incremental` (adopters) | +| `BASE` | `origin/HEAD`, falling back to `main` | base ref; the workflow passes `origin/` explicitly, so the default serves local runs | +| `THRESHOLD_LINES` | `800`, provisional | summed whole-file line count of the mutate set, above which the gate loud-skips; re-derived from the measured rehearsal rate (see Threshold rationale) | +| `BREAK` | `100` | minimum mutation score over the mutated set; below it the gate fails | +| `CONFIG` | `stryker.conf.json` | Stryker config path, read for `mutate` globs | +| `GLOBS` | the conf's `mutate` | explicit glob override for projects whose patterns exceed the supported subset | +| `TEST_SIBLINGS` | `true` | a changed **or deleted** `X.test.ts`/`X.spec.ts` pulls in sibling `X.ts` when it exists and matches the globs | +| `FORCE` / `SKIP` | unset | set by the workflow from PR labels; `FORCE` wins if both. (`FORCE` is the gate's label override; it is unrelated to Stryker's own `--force` incremental-rebuild flag) | +| `REQUIRE_BASELINE` | `true` | incremental mode only: missing incremental file means loud-skip, never a surprise full campaign | +| `STRYKER_CMD` | `node_modules/.bin/stryker run` | how to spawn Stryker | +| `EXTRA_ARGS` | empty | appended to the Stryker invocation (e.g. `--concurrency 4` once measured; see Threshold rationale) | +| `REPORT` | `reports/mutation/mutation.json` | where the JSON report lands (the reporter's verified default) | + +**Flow, `changed` mode:** + +1. Resolve the diff base as `git merge-base $BASE HEAD` and take `git diff --name-status -z HEAD`. Kept for mutation: added, modified, and renamed (new path) files. Deleted paths are retained for step 3 before being dropped from the mutate set. Rename records in `-z` output are three-field (`R\0old\0new\0`); the parser handles them explicitly and a rename fixture is a required test case, since a status/path-alternating parser silently mis-pairs everything after the first rename. +2. Intersect with the `mutate` globs via a built-in matcher supporting `**`, `*`, `?`, `{a,b}`, and leading-`!` negation. That subset covers offbook's globs exactly. An unsupported pattern (e.g. Stryker's extglob defaults) is detected and refused with "set `MUTATION_GATE_GLOBS`", never silently mismatched. Paths are repo-relative POSIX, as git emits them. Two knowing divergences from Stryker's minimatch, both inert for offbook's globs and for the supported subset's intended use: the matcher's `*`/`**` also match dotfiles (minimatch defaults to `dot: false`), and a mid-segment `**` (e.g. `a**b.ts`) crosses directory separators where minimatch treats it as `*`. Projects whose globs depend on either behavior should set `MUTATION_GATE_GLOBS` to unambiguous spellings. +3. Apply the test-sibling rule to changed **and deleted** test files (strip the `.test`/`.spec` segment, include the sibling if it exists **at HEAD, i.e. in the PR's tree**, and matches the globs). Existence-at-HEAD is what keeps a legitimately removed module honest: a PR deleting both `X.test.ts` and `X.ts` derives a sibling that no longer exists, so nothing is included and Stryker is never handed a nonexistent path. Deleting `src/engine/scheduler.test.ts` is the maximal test-weakening event, and it must pull `scheduler.ts` into the mutate set; the adversarial review confirmed the earlier drop-deletions-first ordering let it evade the gate entirely. An empty resulting set passes green ("no mutable files changed"). +4. Size decision: sum the files' line counts. Over `THRESHOLD_LINES` (or `SKIP` set) is a loud skip, green with a notice. Under (or `FORCE` set) proceeds. +5. Run `$STRYKER_CMD --mutate ` with reporters `clear-text,progress,json,html`. The CLI `--mutate` overrides the conf's globs; the conf file is never modified (verified: CLI arrays replace conf arrays wholesale). +6. Interpret the JSON report in the script, over the full status enum (`Killed`, `Survived`, `NoCoverage`, `Timeout`, `CompileError`, `RuntimeError`, `Ignored`, `Pending`): **detected = Killed + Timeout; undetected = Survived + NoCoverage; valid = detected + undetected; score = 100 x detected / valid, defined as 100 when valid = 0.** This is Stryker's own metric, and the D-011 reading (its 100% includes 2 timeouts). `Ignored`, `CompileError`, `RuntimeError`, and `Pending` never enter the verdict but are counted in the summary, so an error-heavy run is visible without red-ing an innocent PR. Score below `BREAK` fails; the failure output lists each undetected mutant as `file:line mutatorName` plus the kill-or-annotate instruction. A nonzero Stryker exit with no report is an infra failure, reported distinctly from a gate verdict. + +**Flow, `incremental` mode:** steps 1-4 run identically (the diff still drives the size decision and labels), but step 5 becomes `$STRYKER_CMD --incremental --incrementalFile ` (default `reports/stryker-incremental.json`, the verified Stryker default) with no `--mutate` override: Stryker's own change detection picks the mutants, which also catches test-only weakening. Missing baseline with `REQUIRE_BASELINE=true` loud-skips. The break threshold then applies to the full-scope score, so incremental adopters set `BREAK` to their earned level. Two documented caveats for adopters: Stryker documents incremental as an approximation (schedule periodic full runs), and **the line-count skip heuristic does not bound incremental work**: Stryker's invalidation is its own diffing, so a small change to a shared test helper can invalidate arbitrarily many mutants and run toward the timeout under a tiny-diff green light. Baseline production (a main-push or scheduled job saving the incremental file via `actions/cache`) ships as a commented-out job in the workflow file. + +**Outputs:** exit 0 (pass or loud skip), 1 (gate failure), 2 (infra failure). The script writes a Markdown block to `GITHUB_STEP_SUMMARY` and sets `decision`/`summary` via `GITHUB_OUTPUT` for the comment step. Local preflight: `node scripts/mutation-gate.mjs` on a branch (the `BASE` default resolves `origin/HEAD`). + +## The workflow + +Condensed sketch; exact action majors verified at implementation time: + +```yaml +name: mutation +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, labeled, unlabeled] +concurrency: + group: mutation-${{ github.event.pull_request.number }} + cancel-in-progress: true +permissions: + contents: read + pull-requests: write # sticky comment +jobs: + mutation: + runs-on: ubuntu-latest + timeout-minutes: 15 # backstop against a mispredicted run + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} # the PR as authored, not the synthetic merge ref + fetch-depth: 0 # all branches: merge-base needs the real base branch + - uses: oven-sh/setup-bun@v2 + with: { bun-version: "1.3.14" } # same pin rationale as gates + - uses: actions/setup-node@v5 + with: { node-version: "24" } # Stryker CLI host (D-010; engines >=20, but 20 is EOL and local practice is 24) + - run: bun install --frozen-lockfile + - name: gate + id: gate + env: + MUTATION_GATE_BASE: origin/${{ github.event.pull_request.base.ref }} # the branch, never the payload SHA + MUTATION_GATE_FORCE: ${{ contains(github.event.pull_request.labels.*.name, 'mutate-force') && '1' || '' }} + MUTATION_GATE_SKIP: ${{ contains(github.event.pull_request.labels.*.name, 'mutate-skip') && '1' || '' }} + run: node scripts/mutation-gate.mjs + - name: sticky comment + if: always() && steps.gate.outputs.decision != '' + continue-on-error: true # fork PRs get a read-only token; the verdict is the gate step's alone + env: { GH_TOKEN: ${{ github.token }} } + run: # upsert one comment marked , via gh api + - name: report artifact + if: failure() + uses: actions/upload-artifact@v7 + with: { name: mutation-report, path: reports/mutation/, retention-days: 14, if-no-files-found: ignore } +``` + +Notes: + +- **Diff-base correctness is a design decision, not an implementation accident.** `pull_request.base.sha` in the event payload can be stale when the base branch has moved, and the default checkout is the synthetic merge ref; the adversarial review demonstrated that combination making the gate see base-branch commits as PR changes, inflating the size sum and failing open via loud-skip exactly when main is busy. Hence: check out the PR **head SHA**, fetch all branches, and merge-base against `origin/`. Testing the PR as authored (not merged with latest main) is safe here because the D-017 ruleset already requires branches to be up to date with main before merging. +- The decision logic runs inside an always-starting job, so the required check always reports one of: pass, loud-skip (green), gate-fail, infra-fail. No job-level `if:`, no skipped-check ambiguity. A green loud-skip satisfying a required check is standard GitHub semantics (verified). +- **Accepted cost**: any label add or remove, including unrelated labels, re-runs the job, and `cancel-in-progress` kills an in-flight campaign. A label-name filter was considered and rejected: the always-report property is load-bearing, and offbook's label traffic is negligible. Adopters with chatty label automation are pointed at this note. +- The sticky comment is upserted (found by the HTML sentinel, edited in place): posted on skip or fail, updated to the pass state if it already exists, never duplicated across `synchronize` events. The step never carries the verdict (`continue-on-error: true`), so a fork PR's read-only token cannot red the check for infra reasons. +- The report artifact uploads only on failure (that is when the HTML drill-down earns its keep; the full engine report is ~850 KB, a scoped run smaller). Adopters wanting it on every run flip the step to `if: always()`; the step is self-contained and deletable. +- For adopters, the file carries a header comment: replace the toolchain setup steps (a Node-only project drops setup-bun and uses `npm ci`), keep the rest; set the env knobs as needed. +- One-time setup: add `mutation` to the `main` ruleset's required checks alongside `gates` (admin bypass unchanged), and create the `mutate-force`/`mutate-skip` labels. + +## Threshold rationale + +Engine's mutable source is 917 lines across 7 files and produced 461 mutants in the D-011 campaign (427 non-ignored + 34 ignored), about 0.5 mutants per line. The largest single file (`index.ts`, 363 lines) bounds the smallest usable threshold: below ~400, a single-file PR to the biggest engine file could never be gated. + +The conf pins `concurrency: 1` as its baseline — that default still governs a local `bun run mutate` full campaign — but the gate's own workflow now overrides it via `MUTATION_GATE_EXTRA_ARGS: "--concurrency 4"`, the measured setting below; the naive arithmetic (800 lines ≈ 400 mutants inside 15 minutes) no longer describes an unmeasured **serial** pipeline. The old coverage-tooling design note ("requires `--concurrency 1`") predates the installed runner 1.3.8, whose README documents parallel workers; perTest coverage under parallel workers was measured clean under concurrency 4 (no coverage-correlation warnings in the run log). **Measured** (Task 10 rehearsal, ubuntu-latest, 2026-08-04): mutating a 409-line set (`src/engine/index.ts` + `instances.ts`) at concurrency 1 hit the 15-minute job timeout with no report produced; concurrency 4 on the identical set completed cleanly in 7m42s at **27.53 mutants/min**. Concurrency 4 ships. The derived sustainable threshold (27.53 × 12 / 0.5 ≈ 661, rounded to 700) sits within a factor of 2 of 800, so `THRESHOLD_LINES` **stays 800**, unchanged. Full numbers, the drift-survivor finding, and the stryker.conf.json fixes required to get the rehearsal running at all are recorded in D-027. `timeout-minutes: 15` stays the hard backstop either way. + +## Edge cases + +- **Shallow history** (no merge-base): infra-fail naming the fix (`fetch-depth: 0`), never a false pass. +- **Renames** gate the new path; the three-field `-z` record is parsed explicitly (see flow step 1). +- **Deleted test files** pull their sibling source into the mutate set (see flow step 3); other deletions are dropped. +- **Zero mutants in the selected files** (everything annotated): pass, with a "0 mutants" note. The script never invokes Stryker with an empty `--mutate` list (the empty set passes before spawning). +- **Test helpers, config, `package.json` changes** do not trigger the gate. Accepted residual risk, documented; incremental mode is the answer for projects that care. +- **Stryker crash vs. clean run** is distinguished by report presence; the verdict is computed from the JSON report and exit codes only, never from printed text. +- **Unsupported glob pattern** in the conf: refused loudly with the `MUTATION_GATE_GLOBS` remedy. +- **Windows runners**: out of scope, documented (the script assumes POSIX paths from git and a POSIX spawn of `node_modules/.bin/stryker`). + +## Repo integration + +- `DECISIONS.md`: new **D-027** recording the gate (mechanism, diff-base decision, score formula, thresholds and the measured runner rate, labels, artifact policy, portability intent), amending D-010's "never a gate" and D-017's "excluded from CI in any form". Two explicit caveat sentences: the gate excludes `Ignored`, so a `// Stryker disable` annotation silences a survivor and annotation *quality* (the unobservability argument, D-011) remains human review; and a Stryker or runner bump can change the mutant set, so a full D-011 campaign runs after any such bump before engine PRs resume, else the next innocent PR inherits the drift. +- `AGENTS.md` working notes: the "manual and never a gate" line becomes "manual full campaigns plus a changed-file PR gate"; labels, the loud-skip behavior, and the post-bump full-campaign rule get one line each. +- `bun run mutate` and the local full-campaign workflow (D-011 hygiene) are untouched. +- No new `R-###`; no `check-docs` changes. (Verified during review: `check-docs` is green with this spec referencing D-027, which is the contiguous next id.) +- Biome excludes `scripts/` entirely (`biome.json`), so the `.mjs` gets no lint coverage; correctness relies on its unit tests and the e2e rehearsal, and `bun run lint` makes no claim about it. + +## Out of scope + +- Widening offbook's `mutate` globs beyond `src/engine/` (module-by-module follow-ups, each behind its own campaign). +- An offbook-side incremental baseline or scheduled full mutation runs (documented as adopter opt-ins only). +- Changed-line mutation ranges. +- Non-GitHub CI and Windows support. +- Any change to the `gates` workflow, the coverage floors, or Biome's `scripts/` exclusion. + +## Verification + +- Unit tests green under full `bun test` (per-file coverage floors judge the `.mjs`; gate on exit code, not printed counts). Required fixtures: a rename record (`R100`, three-field), a deleted-test-file diff that must pull the sibling source, its counterpart deleting both test and source that must pull nothing (existence-at-HEAD), and a report JSON exercising every status in the enum including the zero-valid case. +- E2E rehearsal on a scratch branch: plant a surviving mutant in an engine file; the `mutation` check goes red naming exactly that mutant; kill it; the check goes green. Set `THRESHOLD_LINES=1` on a test run to exercise the loud-skip path cheaply; toggle `mutate-force`/`mutate-skip` and confirm the decision flips without a push. Measure mutants-per-minute at concurrency 1 and 4 (see Threshold rationale) and record both in D-027. +- Sticky comment: exactly one comment across multiple pushes; artifact appears on the red run only. +- `bun scripts/check-docs.ts`, `bun run typecheck`, and full `bun test` all green after the doc and script edits (`bun run lint` is unaffected by design: Biome excludes `scripts/`). +- Ruleset: PR merge blocked while `mutation` is red; loud-skip satisfies the check; admin bypass still works. diff --git a/scripts/mutation-gate.mjs b/scripts/mutation-gate.mjs new file mode 100644 index 0000000..3d09839 --- /dev/null +++ b/scripts/mutation-gate.mjs @@ -0,0 +1,371 @@ +// mutation-gate: a changed-file StrykerJS mutation gate for PRs. +// Portable unit: this file + .github/workflows/mutation.yml (copy both). +// Spec: docs/superpowers/specs/2026-08-04-mutation-pr-gate-design.md +// +// Configuration (env, all optional): +// MUTATION_GATE_MODE changed (default) | incremental +// MUTATION_GATE_BASE base ref; default origin/HEAD, then main +// MUTATION_GATE_THRESHOLD_LINES loud-skip above this summed line count (800) +// MUTATION_GATE_BREAK minimum score, fail below it (100) +// MUTATION_GATE_CONFIG stryker config path (stryker.conf.json) +// MUTATION_GATE_GLOBS comma-separated mutate globs, overrides config +// MUTATION_GATE_TEST_SIBLINGS changed/deleted X.test.ts pulls X.ts (true) +// MUTATION_GATE_FORCE run + block even over threshold (labels) +// MUTATION_GATE_SKIP loud-skip regardless of size (labels; FORCE wins) +// MUTATION_GATE_REQUIRE_BASELINE incremental: skip when baseline missing (true) +// MUTATION_GATE_INCREMENTAL_FILE reports/stryker-incremental.json +// MUTATION_GATE_STRYKER_CMD node_modules/.bin/stryker run (split on spaces) +// MUTATION_GATE_EXTRA_ARGS appended to the stryker invocation +// MUTATION_GATE_REPORT reports/mutation/mutation.json +// Exit codes: 0 pass/skip, 1 gate failure (undetected mutants), 2 infra failure. + +import { spawnSync } from "node:child_process"; +import { appendFileSync, existsSync, readFileSync, realpathSync, rmSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +export function parseNameStatusZ(raw) { + const tokens = raw.split("\0").filter((t) => t.length > 0); + const changed = []; + const deleted = []; + let i = 0; + while (i < tokens.length) { + const status = tokens[i]; + const kind = status[0]; + if (kind === "R" || kind === "C") { + changed.push(tokens[i + 2]); + i += 3; + } else if (kind === "D") { + deleted.push(tokens[i + 1]); + i += 2; + } else if (kind === "A" || kind === "M" || kind === "T") { + changed.push(tokens[i + 1]); + i += 2; + } else { + throw new Error(`mutation-gate: unhandled diff status "${status}"`); + } + } + return { changed, deleted }; +} + +export function countLines(content) { + if (content === "") return 0; + const lines = content.split("\n"); + if (lines[lines.length - 1] === "") lines.pop(); + return lines.length; +} + +export class UnsupportedGlobError extends Error { + constructor(pattern) { + super( + `mutation-gate: glob "${pattern}" uses syntax outside the supported subset ` + + `(**, *, ?, {a,b}, leading !). Set MUTATION_GATE_GLOBS to equivalent simple globs.`, + ); + this.name = "UnsupportedGlobError"; + } +} + +const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +export function globToRegExp(pattern) { + if (/[()[\]\\]/.test(pattern)) throw new UnsupportedGlobError(pattern); + let out = ""; + let i = 0; + while (i < pattern.length) { + const ch = pattern[i]; + if (ch === "*") { + if (pattern.startsWith("**/", i)) { + out += "(?:[^/]+/)*"; + i += 3; + } else if (pattern.startsWith("**", i)) { + out += ".*"; + i += 2; + } else { + out += "[^/]*"; + i += 1; + } + } else if (ch === "?") { + out += "[^/]"; + i += 1; + } else if (ch === "{") { + const end = pattern.indexOf("}", i); + const body = end === -1 ? "" : pattern.slice(i + 1, end); + if (end === -1 || /[*?{]/.test(body)) throw new UnsupportedGlobError(pattern); + out += `(?:${body.split(",").map(escapeRegExp).join("|")})`; + i = end + 1; + } else { + out += escapeRegExp(ch); + i += 1; + } + } + return new RegExp(`^${out}$`); +} + +export function matchesMutateGlobs(path, globs) { + let included = false; + for (const glob of globs) { + const negated = glob.startsWith("!"); + const pattern = negated ? glob.slice(1) : glob; + if (globToRegExp(pattern).test(path)) included = !negated; + } + return included; +} + +export function siblingOf(path) { + const m = path.match(/^(.*)\.(test|spec)(\.[^./]+)$/); + return m ? `${m[1]}${m[3]}` : null; +} + +export function selectMutateSet({ changed, deleted, globs, testSiblings, exists }) { + const set = new Set(); + for (const path of changed) { + if (matchesMutateGlobs(path, globs)) set.add(path); + } + if (testSiblings) { + for (const path of [...changed, ...deleted]) { + const sibling = siblingOf(path); + if (sibling && exists(sibling) && matchesMutateGlobs(sibling, globs)) set.add(sibling); + } + } + return [...set].sort(); +} + +export const DEFAULTS = Object.freeze({ + mode: "changed", + thresholdLines: 800, + breakScore: 100, + configPath: "stryker.conf.json", + incrementalFile: "reports/stryker-incremental.json", + strykerCmd: "node_modules/.bin/stryker run", + reportPath: "reports/mutation/mutation.json", +}); + +const FALSY = new Set(["0", "false", "no"]); +const asBool = (v, dflt) => (v === undefined || v === "" ? dflt : !FALSY.has(v.toLowerCase())); +const asFlag = (v) => v !== undefined && v !== "" && !FALSY.has(v.toLowerCase()); +const asNum = (v, dflt) => { + if (v === undefined || v === "") return dflt; + const n = Number(v); + if (!Number.isFinite(n)) throw new Error(`mutation-gate: not a number: "${v}"`); + return n; +}; +const asList = (v) => + v === undefined + ? undefined + : v + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + +export function readConfig(env) { + return { + mode: env.MUTATION_GATE_MODE || DEFAULTS.mode, + base: env.MUTATION_GATE_BASE || undefined, + thresholdLines: asNum(env.MUTATION_GATE_THRESHOLD_LINES, DEFAULTS.thresholdLines), + breakScore: asNum(env.MUTATION_GATE_BREAK, DEFAULTS.breakScore), + configPath: env.MUTATION_GATE_CONFIG || DEFAULTS.configPath, + globsOverride: asList(env.MUTATION_GATE_GLOBS), + testSiblings: asBool(env.MUTATION_GATE_TEST_SIBLINGS, true), + force: asFlag(env.MUTATION_GATE_FORCE), + skip: asFlag(env.MUTATION_GATE_SKIP), + requireBaseline: asBool(env.MUTATION_GATE_REQUIRE_BASELINE, true), + incrementalFile: env.MUTATION_GATE_INCREMENTAL_FILE || DEFAULTS.incrementalFile, + strykerCmd: (env.MUTATION_GATE_STRYKER_CMD || DEFAULTS.strykerCmd).split(" ").filter(Boolean), + extraArgs: (env.MUTATION_GATE_EXTRA_ARGS || "").split(" ").filter(Boolean), + reportPath: env.MUTATION_GATE_REPORT || DEFAULTS.reportPath, + }; +} + +export function decide({ files, totalLines, thresholdLines, force, skip }) { + if (files.length === 0) return "pass-empty"; + if (force) return "run"; + if (skip) return "skip-label"; + if (totalLines > thresholdLines) return "skip-size"; + return "run"; +} + +export function interpretReport(report, breakScore) { + const counts = { + Killed: 0, Survived: 0, NoCoverage: 0, Timeout: 0, CompileError: 0, RuntimeError: 0, Ignored: 0, Pending: 0, + }; + const undetected = []; + for (const [file, data] of Object.entries(report.files ?? {})) { + for (const mutant of data.mutants) { + if (!Object.hasOwn(counts, mutant.status)) { + throw new Error(`mutation-gate: unknown mutant status "${mutant.status}"`); + } + counts[mutant.status] += 1; + if (mutant.status === "Survived" || mutant.status === "NoCoverage") { + undetected.push({ file, line: mutant.location.start.line, mutator: mutant.mutatorName }); + } + } + } + const detected = counts.Killed + counts.Timeout; + const valid = detected + counts.Survived + counts.NoCoverage; + const score = valid === 0 ? 100 : (100 * detected) / valid; + return { counts, undetected, score, verdict: score < breakScore ? "fail" : "pass" }; +} + +const SKIP_REASONS = { + "pass-empty": "no mutable files changed; nothing to mutate.", + "skip-size": "the change is over the size threshold for a CI mutation run.", + "skip-label": "the mutate-skip label is set.", + "skip-no-baseline": "incremental mode has no baseline incremental file; refusing a surprise full campaign.", +}; + +export function renderSkip({ decision, files, totalLines, thresholdLines }) { + const lines = [`## mutation gate: ${decision}`, "", SKIP_REASONS[decision] ?? decision]; + if (decision !== "pass-empty") { + lines.push( + "", + `Mutable files in this change: ${files.length} (${totalLines} lines; threshold ${thresholdLines}).`, + "The gate did not run. Before merging, run the mutation check locally: `bun run mutate`", + "(or `MUTATION_GATE_BASE= node scripts/mutation-gate.mjs` for the changed-file run).", + "Labels: `mutate-force` runs the gate anyway; `mutate-skip` waves it off.", + ); + } + return `${lines.join("\n")}\n`; +} + +export function renderResult({ files, result, breakScore }) { + const c = result.counts; + const lines = [ + `## mutation gate: ${result.verdict} (score ${result.score.toFixed(2)}, break ${breakScore})`, + "", + `Mutated ${files.length} file(s): ${files.join(", ")}`, + `Mutants: ${c.Killed} killed, ${c.Timeout} timeout, ${c.Survived} survived, ${c.NoCoverage} no-coverage; ` + + `${c.Ignored} ignored, ${c.CompileError + c.RuntimeError} errored, ${c.Pending} pending.`, + ]; + if (result.undetected.length > 0) { + lines.push("", "Undetected mutants (kill each with a test, or annotate with a reasoned", + "`// Stryker disable next-line : `):", ""); + for (const m of result.undetected) { + lines.push(`- \`${m.file}:${m.line}\` ${m.mutator}`); + } + } + return `${lines.join("\n")}\n`; +} + +export function renderInfra(message) { + return `## mutation gate: infra failure\n\n${message}\n\nThis is an infrastructure error, not a verdict on the PR's tests.\n`; +} + +export function formatGithubOutputs(outputs) { + const lines = []; + for (const [key, value] of Object.entries(outputs)) { + lines.push(`${key}<<__MUTATION_GATE_EOF__`, String(value), "__MUTATION_GATE_EOF__"); + } + return `${lines.join("\n")}\n`; +} + +export const EXIT = Object.freeze({ ok: 0, gateFail: 1, infra: 2 }); + +export function resolveDefaultBase(deps) { + const head = deps.exec(["git", "symbolic-ref", "-q", "refs/remotes/origin/HEAD"]); + if (head.code === 0 && head.stdout.trim() !== "") { + return head.stdout.trim().replace(/^refs\/remotes\//, ""); + } + return "main"; +} + +export function readMutateGlobs(deps, configPath) { + const conf = JSON.parse(deps.readFile(configPath)); + if (!Array.isArray(conf.mutate) || conf.mutate.length === 0) { + throw new Error(`mutation-gate: no "mutate" array in ${configPath}; set MUTATION_GATE_GLOBS`); + } + return conf.mutate; +} + +function finish(deps, decision, summaryMd, exitCode) { + deps.log(summaryMd); + deps.writeSummary(summaryMd); + deps.writeOutputs({ decision, summary: summaryMd }); + return exitCode; +} + +export function main(deps) { + const d = deps ?? realDeps(); + try { + const cfg = readConfig(d.env); + if (cfg.mode !== "changed" && cfg.mode !== "incremental") { + throw new Error(`mutation-gate: unknown MUTATION_GATE_MODE "${cfg.mode}"`); + } + const base = cfg.base ?? resolveDefaultBase(d); + const mb = d.exec(["git", "merge-base", base, "HEAD"]); + if (mb.code !== 0) { + return finish(d, "infra", renderInfra( + `no merge-base between "${base}" and HEAD. In CI, check out with fetch-depth: 0 so the base branch history is present.`, + ), EXIT.infra); + } + const diff = d.exec(["git", "diff", "--name-status", "-z", "-M", mb.stdout.trim(), "HEAD"]); + if (diff.code !== 0) { + return finish(d, "infra", renderInfra("git diff --name-status failed"), EXIT.infra); + } + const { changed, deleted } = parseNameStatusZ(diff.stdout); + const globs = cfg.globsOverride ?? readMutateGlobs(d, cfg.configPath); + const files = selectMutateSet({ changed, deleted, globs, testSiblings: cfg.testSiblings, exists: d.exists }); + const totalLines = files.reduce((n, f) => n + countLines(d.readFile(f)), 0); + const decision = decide({ files, totalLines, thresholdLines: cfg.thresholdLines, force: cfg.force, skip: cfg.skip }); + if (decision !== "run") { + return finish(d, decision, renderSkip({ decision, files, totalLines, thresholdLines: cfg.thresholdLines }), EXIT.ok); + } + let strykerArgs; + if (cfg.mode === "incremental") { + if (cfg.requireBaseline && !d.exists(cfg.incrementalFile)) { + return finish(d, "skip-no-baseline", + renderSkip({ decision: "skip-no-baseline", files, totalLines, thresholdLines: cfg.thresholdLines }), EXIT.ok); + } + strykerArgs = [...cfg.strykerCmd, "--incremental", "--incrementalFile", cfg.incrementalFile, + "--reporters", "clear-text,progress,json,html", ...cfg.extraArgs]; + } else { + strykerArgs = [...cfg.strykerCmd, "--mutate", files.join(","), + "--reporters", "clear-text,progress,json,html", ...cfg.extraArgs]; + } + if (d.exists(cfg.reportPath)) d.remove(cfg.reportPath); + const run = d.exec(strykerArgs, { inherit: true }); + if (!d.exists(cfg.reportPath)) { + return finish(d, "infra", renderInfra( + `stryker exited ${run.code} without writing ${cfg.reportPath}. Read the run log above; this may be a crash, not a test-strength verdict.`, + ), EXIT.infra); + } + const result = interpretReport(JSON.parse(d.readFile(cfg.reportPath)), cfg.breakScore); + const summaryMd = renderResult({ files, result, breakScore: cfg.breakScore }); + return finish(d, result.verdict, summaryMd, result.verdict === "pass" ? EXIT.ok : EXIT.gateFail); + } catch (err) { + return finish(d, "infra", renderInfra(err.message), EXIT.infra); + } +} + +export function realDeps() { + return { + env: process.env, + exec(argv, opts = {}) { + const r = spawnSync(argv[0], argv.slice(1), { + encoding: "utf8", + stdio: opts.inherit ? ["ignore", "inherit", "inherit"] : ["ignore", "pipe", "pipe"], + maxBuffer: 64 * 1024 * 1024, + }); + if (r.error) throw r.error; + return { code: r.status ?? 1, stdout: r.stdout ?? "" }; + }, + readFile: (p) => readFileSync(p, "utf8"), + exists: existsSync, + remove: (p) => rmSync(p, { force: true }), + writeSummary(md) { + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${md}\n`); + }, + writeOutputs(outputs) { + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, formatGithubOutputs(outputs)); + }, + log: (msg) => console.error(msg), + }; +} + +const isCliEntry = (() => { + if (!process.argv[1]) return false; + try { + return realpathSync(process.argv[1]) === fileURLToPath(import.meta.url); + } catch { + return false; + } +})(); +if (isCliEntry) process.exit(main()); diff --git a/scripts/mutation-gate.test.ts b/scripts/mutation-gate.test.ts new file mode 100644 index 0000000..84cfbf0 --- /dev/null +++ b/scripts/mutation-gate.test.ts @@ -0,0 +1,649 @@ +import { test, expect } from "bun:test"; +import { parseNameStatusZ, countLines, globToRegExp, matchesMutateGlobs, UnsupportedGlobError, siblingOf, selectMutateSet, DEFAULTS, readConfig, decide, interpretReport, renderSkip, renderResult, renderInfra, formatGithubOutputs, EXIT, resolveDefaultBase, readMutateGlobs, main, realDeps } from "./mutation-gate.mjs"; + +test("parseNameStatusZ splits adds/modifies from deletes", () => { + const raw = "A\0src/engine/new.ts\0M\0src/engine/dispatch.ts\0D\0src/engine/old.test.ts\0"; + expect(parseNameStatusZ(raw)).toEqual({ + changed: ["src/engine/new.ts", "src/engine/dispatch.ts"], + deleted: ["src/engine/old.test.ts"], + }); +}); + +test("parseNameStatusZ reads the three-field rename record and keeps the new path", () => { + const raw = "R100\0src/engine/a.ts\0src/engine/b.ts\0M\0src/engine/c.ts\0"; + expect(parseNameStatusZ(raw)).toEqual({ changed: ["src/engine/b.ts", "src/engine/c.ts"], deleted: [] }); +}); + +test("parseNameStatusZ handles copies and typechanges as changes", () => { + const raw = "C75\0src/a.ts\0src/b.ts\0T\0src/c.ts\0"; + expect(parseNameStatusZ(raw)).toEqual({ changed: ["src/b.ts", "src/c.ts"], deleted: [] }); +}); + +test("parseNameStatusZ throws on an unhandled status instead of mis-pairing the rest", () => { + expect(() => parseNameStatusZ("U\0src/conflicted.ts\0")).toThrow('unhandled diff status "U"'); +}); + +test("parseNameStatusZ of an empty diff is empty", () => { + expect(parseNameStatusZ("")).toEqual({ changed: [], deleted: [] }); +}); + +test("countLines counts content lines with and without trailing newline", () => { + expect(countLines("")).toBe(0); + expect(countLines("a\n")).toBe(1); + expect(countLines("a\nb")).toBe(2); + expect(countLines("a\nb\n")).toBe(2); +}); + +test("offbook's real globs: engine source in, engine tests out, broker out", () => { + const globs = ["src/engine/**/*.ts", "!src/engine/**/*.test.ts"]; + expect(matchesMutateGlobs("src/engine/scheduler.ts", globs)).toBe(true); + expect(matchesMutateGlobs("src/engine/sub/dir/x.ts", globs)).toBe(true); + expect(matchesMutateGlobs("src/engine/scheduler.test.ts", globs)).toBe(false); + expect(matchesMutateGlobs("src/broker/index.ts", globs)).toBe(false); +}); + +test("** matches zero segments", () => { + expect(globToRegExp("src/**/*.ts").test("src/index.ts")).toBe(true); +}); + +test("* stays inside one segment; ? matches exactly one char", () => { + expect(globToRegExp("src/*.ts").test("src/a/b.ts")).toBe(false); + expect(globToRegExp("src/?.ts").test("src/a.ts")).toBe(true); + expect(globToRegExp("src/?.ts").test("src/ab.ts")).toBe(false); +}); + +test("single-level braces expand", () => { + const re = globToRegExp("{src,lib}/a.ts"); + expect(re.test("src/a.ts")).toBe(true); + expect(re.test("lib/a.ts")).toBe(true); + expect(re.test("bin/a.ts")).toBe(false); +}); + +test("negation is ordered unset-on-match, later patterns win", () => { + expect(matchesMutateGlobs("src/a.test.ts", ["src/**/*.ts", "!src/**/*.test.ts"])).toBe(false); + expect(matchesMutateGlobs("src/a.test.ts", ["!src/**/*.test.ts", "src/**/*.ts"])).toBe(true); +}); + +test("extglobs, character classes, escapes, wildcards-in-braces are refused, never mis-matched", () => { + for (const bad of ["src/!(*.test).ts", "src/[ab].ts", "src/a\\*.ts", "{src/*,lib}/a.ts"]) { + expect(() => globToRegExp(bad)).toThrow(UnsupportedGlobError); + } +}); + +test("regex metacharacters in glob literals are escaped (a dot is a dot)", () => { + expect(globToRegExp("src/a.ts").test("src/axts")).toBe(false); +}); + +test("siblingOf derives the source next to a test file", () => { + expect(siblingOf("src/engine/scheduler.test.ts")).toBe("src/engine/scheduler.ts"); + expect(siblingOf("src/engine/faker.spec.tsx")).toBe("src/engine/faker.tsx"); + expect(siblingOf("src/engine/scheduler.ts")).toBe(null); +}); + +const ENGINE_GLOBS = ["src/engine/**/*.ts", "!src/engine/**/*.test.ts"]; + +test("selectMutateSet keeps matching changed sources, drops the rest", () => { + const files = selectMutateSet({ + changed: ["src/engine/dispatch.ts", "src/broker/index.ts", "README.md"], + deleted: [], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: () => true, + }); + expect(files).toEqual(["src/engine/dispatch.ts"]); +}); + +test("a changed test file pulls its existing sibling source", () => { + const files = selectMutateSet({ + changed: ["src/engine/scheduler.test.ts"], + deleted: [], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: (p) => p === "src/engine/scheduler.ts", + }); + expect(files).toEqual(["src/engine/scheduler.ts"]); +}); + +test("a deleted test file pulls its surviving sibling (the test-deletion evasion)", () => { + const files = selectMutateSet({ + changed: [], + deleted: ["src/engine/scheduler.test.ts"], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: (p) => p === "src/engine/scheduler.ts", + }); + expect(files).toEqual(["src/engine/scheduler.ts"]); +}); + +test("deleting module and test together pulls nothing (existence-at-HEAD)", () => { + const files = selectMutateSet({ + changed: [], + deleted: ["src/engine/gone.ts", "src/engine/gone.test.ts"], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: () => false, + }); + expect(files).toEqual([]); +}); + +test("testSiblings=false disables the rule; output is deduped and sorted", () => { + expect( + selectMutateSet({ + changed: ["src/engine/scheduler.test.ts"], + deleted: [], + globs: ENGINE_GLOBS, + testSiblings: false, + exists: () => true, + }), + ).toEqual([]); + expect( + selectMutateSet({ + changed: ["src/engine/b.ts", "src/engine/a.ts", "src/engine/a.test.ts"], + deleted: [], + globs: ENGINE_GLOBS, + testSiblings: true, + exists: () => true, + }), + ).toEqual(["src/engine/a.ts", "src/engine/b.ts"]); +}); + +test("decide: empty set passes, labels and threshold act, force wins over both", () => { + expect(decide({ files: [], totalLines: 0, thresholdLines: 800, force: false, skip: false })).toBe("pass-empty"); + expect(decide({ files: ["a"], totalLines: 10, thresholdLines: 800, force: false, skip: false })).toBe("run"); + expect(decide({ files: ["a"], totalLines: 900, thresholdLines: 800, force: false, skip: false })).toBe("skip-size"); + expect(decide({ files: ["a"], totalLines: 10, thresholdLines: 800, force: false, skip: true })).toBe("skip-label"); + expect(decide({ files: ["a"], totalLines: 900, thresholdLines: 800, force: true, skip: true })).toBe("run"); + expect(decide({ files: [], totalLines: 0, thresholdLines: 800, force: true, skip: false })).toBe("pass-empty"); +}); + +test("readConfig defaults", () => { + const cfg = readConfig({}); + expect(cfg.mode).toBe("changed"); + expect(cfg.base).toBeUndefined(); + expect(cfg.thresholdLines).toBe(800); + expect(cfg.breakScore).toBe(100); + expect(cfg.configPath).toBe("stryker.conf.json"); + expect(cfg.globsOverride).toBeUndefined(); + expect(cfg.testSiblings).toBe(true); + expect(cfg.force).toBe(false); + expect(cfg.skip).toBe(false); + expect(cfg.requireBaseline).toBe(true); + expect(cfg.incrementalFile).toBe("reports/stryker-incremental.json"); + expect(cfg.strykerCmd).toEqual(["node_modules/.bin/stryker", "run"]); + expect(cfg.extraArgs).toEqual([]); + expect(cfg.reportPath).toBe("reports/mutation/mutation.json"); +}); + +test("readConfig parses overrides", () => { + const cfg = readConfig({ + MUTATION_GATE_MODE: "incremental", + MUTATION_GATE_BASE: "origin/develop", + MUTATION_GATE_THRESHOLD_LINES: "200", + MUTATION_GATE_BREAK: "90", + MUTATION_GATE_GLOBS: "lib/**/*.js, !lib/**/*.spec.js", + MUTATION_GATE_TEST_SIBLINGS: "false", + MUTATION_GATE_FORCE: "1", + MUTATION_GATE_EXTRA_ARGS: "--concurrency 4", + }); + expect(cfg.mode).toBe("incremental"); + expect(cfg.base).toBe("origin/develop"); + expect(cfg.thresholdLines).toBe(200); + expect(cfg.breakScore).toBe(90); + expect(cfg.globsOverride).toEqual(["lib/**/*.js", "!lib/**/*.spec.js"]); + expect(cfg.testSiblings).toBe(false); + expect(cfg.force).toBe(true); + expect(cfg.extraArgs).toEqual(["--concurrency", "4"]); +}); + +test("readConfig treats 0/false/no/empty as false for flags and rejects non-numbers", () => { + expect(readConfig({ MUTATION_GATE_FORCE: "false" }).force).toBe(false); + expect(readConfig({ MUTATION_GATE_SKIP: "0" }).skip).toBe(false); + expect(readConfig({ MUTATION_GATE_TEST_SIBLINGS: "" }).testSiblings).toBe(true); + expect(() => readConfig({ MUTATION_GATE_THRESHOLD_LINES: "many" })).toThrow("not a number"); +}); + +type FixtureMutant = { mutator: string; status: string; line?: number }; +export function makeReport(mutantsByFile: Record) { + return { + schemaVersion: "2", + thresholds: { high: 80, low: 60 }, + files: Object.fromEntries( + Object.entries(mutantsByFile).map(([file, mutants]) => [ + file, + { + language: "typescript", + source: "", + mutants: mutants.map((m, i) => ({ + id: String(i), + mutatorName: m.mutator, + status: m.status, + location: { start: { line: m.line ?? 1, column: 1 }, end: { line: m.line ?? 1, column: 2 } }, + })), + }, + ]), + ), + }; +} + +test("all killed passes at break 100; Timeout counts as detected (the D-011 reading)", () => { + const r = interpretReport( + makeReport({ "src/engine/a.ts": [{ mutator: "X", status: "Killed" }, { mutator: "X", status: "Timeout" }] }), + 100, + ); + expect(r.score).toBe(100); + expect(r.verdict).toBe("pass"); +}); + +test("a survivor fails at break 100 and is listed as file:line mutator", () => { + const r = interpretReport( + makeReport({ + "src/engine/a.ts": [ + { mutator: "EqualityOperator", status: "Survived", line: 12 }, + { mutator: "StringLiteral", status: "Killed" }, + ], + }), + 100, + ); + expect(r.verdict).toBe("fail"); + expect(r.score).toBe(50); + expect(r.undetected).toEqual([{ file: "src/engine/a.ts", line: 12, mutator: "EqualityOperator" }]); +}); + +test("NoCoverage is undetected; Ignored/CompileError/RuntimeError/Pending stay out of the verdict", () => { + const r = interpretReport( + makeReport({ + "a.ts": [ + { mutator: "X", status: "Killed" }, + { mutator: "X", status: "NoCoverage", line: 3 }, + { mutator: "X", status: "Ignored" }, + { mutator: "X", status: "CompileError" }, + { mutator: "X", status: "RuntimeError" }, + { mutator: "X", status: "Pending" }, + ], + }), + 100, + ); + expect(r.score).toBe(50); + expect(r.verdict).toBe("fail"); + expect(r.undetected).toEqual([{ file: "a.ts", line: 3, mutator: "X" }]); + expect(r.counts).toEqual({ + Killed: 1, Survived: 0, NoCoverage: 1, Timeout: 0, CompileError: 1, RuntimeError: 1, Ignored: 1, Pending: 1, + }); +}); + +test("an errors-only run scores 100 (zero valid mutants is a pass, not NaN)", () => { + const r = interpretReport(makeReport({ "a.ts": [{ mutator: "X", status: "RuntimeError" }] }), 100); + expect(r.score).toBe(100); + expect(r.verdict).toBe("pass"); +}); + +test("break below 100 tolerates survivors down to the threshold, exact score passes", () => { + const twoOfThree = makeReport({ + "a.ts": [ + { mutator: "X", status: "Killed" }, + { mutator: "X", status: "Killed" }, + { mutator: "X", status: "Survived" }, + ], + }); + expect(interpretReport(twoOfThree, 66).verdict).toBe("pass"); + expect(interpretReport(twoOfThree, 67).verdict).toBe("fail"); +}); + +test("an unknown status throws (schema drift surfaces loudly)", () => { + expect(() => interpretReport(makeReport({ "a.ts": [{ mutator: "X", status: "Vanished" }] }), 100)).toThrow( + 'unknown mutant status "Vanished"', + ); +}); + +test("renderSkip names the numbers, the labels, and the local command", () => { + const md = renderSkip({ decision: "skip-size", files: ["src/engine/index.ts"], totalLines: 950, thresholdLines: 800 }); + expect(md).toContain("skip-size"); + expect(md).toContain("950"); + expect(md).toContain("800"); + expect(md).toContain("bun run mutate"); + expect(md).toContain("mutate-force"); + expect(renderSkip({ decision: "pass-empty", files: [], totalLines: 0, thresholdLines: 800 })).toContain( + "no mutable files changed", + ); + expect( + renderSkip({ decision: "skip-no-baseline", files: ["a.ts"], totalLines: 1, thresholdLines: 800 }), + ).toContain("baseline"); +}); + +test("renderResult on failure lists each undetected mutant with the kill-or-annotate instruction", () => { + const result = { + counts: { Killed: 1, Survived: 1, NoCoverage: 0, Timeout: 0, CompileError: 0, RuntimeError: 0, Ignored: 0, Pending: 0 }, + undetected: [{ file: "src/engine/a.ts", line: 12, mutator: "EqualityOperator" }], + score: 50, + verdict: "fail" as const, + }; + const md = renderResult({ files: ["src/engine/a.ts"], result, breakScore: 100 }); + expect(md).toContain("fail"); + expect(md).toContain("50.00"); + expect(md).toContain("src/engine/a.ts:12"); + expect(md).toContain("EqualityOperator"); + expect(md).toContain("Stryker disable next-line"); +}); + +test("renderResult on pass reports the score and mutant counts", () => { + const result = { + counts: { Killed: 3, Survived: 0, NoCoverage: 0, Timeout: 1, CompileError: 0, RuntimeError: 0, Ignored: 2, Pending: 0 }, + undetected: [], + score: 100, + verdict: "pass" as const, + }; + const md = renderResult({ files: ["src/engine/a.ts"], result, breakScore: 100 }); + expect(md).toContain("pass"); + expect(md).toContain("100.00"); + expect(md).toContain("3 killed"); +}); + +test("renderInfra says it is not a verdict", () => { + expect(renderInfra("boom")).toContain("boom"); + expect(renderInfra("boom")).toContain("not a verdict"); +}); + +test("formatGithubOutputs emits the heredoc form for each key", () => { + expect(formatGithubOutputs({ decision: "fail", summary: "line1\nline2" })).toBe( + "decision<<__MUTATION_GATE_EOF__\nfail\n__MUTATION_GATE_EOF__\n" + + "summary<<__MUTATION_GATE_EOF__\nline1\nline2\n__MUTATION_GATE_EOF__\n", + ); +}); + +const CONF = JSON.stringify({ mutate: ["src/engine/**/*.ts", "!src/engine/**/*.test.ts"] }); + +function fakeDeps({ + files = {} as Record, + execs = [] as Array<{ code: number; stdout: string; writes?: Record }>, + env = {} as Record, +} = {}) { + const calls: string[][] = []; + const outputs: Array> = []; + const summaries: string[] = []; + const deps = { + env, + exec(argv: string[], _opts?: { inherit?: boolean }) { + calls.push(argv); + const next = execs.shift(); + if (!next) throw new Error(`unexpected exec: ${argv.join(" ")}`); + if (next.writes) Object.assign(files, next.writes); + return next; + }, + readFile(p: string) { + if (p in files) return files[p]; + throw new Error(`ENOENT: ${p}`); + }, + exists: (p: string) => p in files, + remove: (p: string) => { + delete files[p]; + }, + writeSummary: (md: string) => summaries.push(md), + writeOutputs: (o: Record) => outputs.push(o), + log: (_msg: string) => {}, + }; + return { deps, calls, outputs, summaries }; +} + +const MB_OK = { code: 0, stdout: "abc123\n" }; +const diffOf = (raw: string) => ({ code: 0, stdout: raw }); + +test("main: missing merge-base is an infra failure naming fetch-depth", () => { + const { deps, outputs } = fakeDeps({ execs: [{ code: 128, stdout: "" }], env: { MUTATION_GATE_BASE: "origin/main" } }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].decision).toBe("infra"); + expect(outputs[0].summary).toContain("fetch-depth: 0"); +}); + +test("main: a failing git diff is an infra failure naming the failing command", () => { + const { deps, outputs } = fakeDeps({ + execs: [MB_OK, { code: 128, stdout: "" }], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].decision).toBe("infra"); + expect(outputs[0].summary).toContain("git diff"); +}); + +test("main: no mutable changes passes empty without spawning stryker", () => { + const { deps, calls, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF }, + execs: [MB_OK, diffOf("M\0README.md\0")], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("pass-empty"); + expect(calls.length).toBe(2); +}); + +test("main: over-threshold loud-skips green without spawning stryker; force runs it", () => { + const bigFile = "x\n".repeat(900); + const base = { + files: { "stryker.conf.json": CONF, "src/engine/index.ts": bigFile }, + execs: [MB_OK, diffOf("M\0src/engine/index.ts\0")], + }; + const skip = fakeDeps({ ...base, env: { MUTATION_GATE_BASE: "origin/main" } }); + expect(main(skip.deps)).toBe(EXIT.ok); + expect(skip.outputs[0].decision).toBe("skip-size"); + expect(skip.calls.length).toBe(2); + + const forced = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/index.ts": bigFile }, + execs: [ + MB_OK, + diffOf("M\0src/engine/index.ts\0"), + { + code: 0, + stdout: "", + writes: { + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/index.ts": [{ mutator: "X", status: "Killed" }] }), + ), + }, + }, + ], + env: { MUTATION_GATE_BASE: "origin/main", MUTATION_GATE_FORCE: "1" }, + }); + expect(main(forced.deps)).toBe(EXIT.ok); + expect(forced.outputs[0].decision).toBe("pass"); +}); + +test("main: a clean run passes and the stryker argv carries --mutate and --reporters", () => { + const { deps, calls, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/dispatch.ts": "a\nb\n" }, + execs: [ + MB_OK, + diffOf("M\0src/engine/dispatch.ts\0"), + { + code: 0, + stdout: "", + writes: { + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/dispatch.ts": [{ mutator: "X", status: "Killed" }] }), + ), + }, + }, + ], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("pass"); + const stryker = calls[2]; + expect(stryker.slice(0, 2)).toEqual(["node_modules/.bin/stryker", "run"]); + expect(stryker).toContain("--mutate"); + expect(stryker[stryker.indexOf("--mutate") + 1]).toBe("src/engine/dispatch.ts"); + expect(stryker[stryker.indexOf("--reporters") + 1]).toBe("clear-text,progress,json,html"); +}); + +test("main: survivors fail the gate with exit 1 and the mutant named", () => { + const { deps, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/dispatch.ts": "a\n" }, + execs: [ + MB_OK, + diffOf("M\0src/engine/dispatch.ts\0"), + { + code: 0, + stdout: "", + writes: { + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/dispatch.ts": [{ mutator: "EqualityOperator", status: "Survived", line: 7 }] }), + ), + }, + }, + ], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.gateFail); + expect(outputs[0].decision).toBe("fail"); + expect(outputs[0].summary).toContain("src/engine/dispatch.ts:7"); +}); + +test("main: stryker exiting without a report is infra, not a verdict", () => { + const { deps, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/dispatch.ts": "a\n" }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0"), { code: 1, stdout: "" }], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].decision).toBe("infra"); + expect(outputs[0].summary).toContain("without writing"); +}); + +test("main: a stale pre-existing report is removed before spawning; a crash that writes nothing is still infra", () => { + const { deps, outputs } = fakeDeps({ + files: { + "stryker.conf.json": CONF, + "src/engine/dispatch.ts": "a\n", + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/dispatch.ts": [{ mutator: "X", status: "Killed" }] }), + ), + }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0"), { code: 1, stdout: "" }], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].decision).toBe("infra"); + expect(outputs[0].summary).toContain("without writing"); +}); + +test("main: an unsupported conf glob is refused with the GLOBS remedy", () => { + const { deps, outputs } = fakeDeps({ + files: { "stryker.conf.json": JSON.stringify({ mutate: ["src/!(*.test).ts"] }) }, + execs: [MB_OK, diffOf("M\0src/a.ts\0")], + env: { MUTATION_GATE_BASE: "origin/main" }, + }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].summary).toContain("MUTATION_GATE_GLOBS"); +}); + +test("main: skip label loud-skips a small PR", () => { + const { deps, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/dispatch.ts": "a\n" }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0")], + env: { MUTATION_GATE_BASE: "origin/main", MUTATION_GATE_SKIP: "1" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("skip-label"); +}); + +test("resolveDefaultBase prefers origin/HEAD, falls back to main", () => { + const viaHead = fakeDeps({ execs: [{ code: 0, stdout: "refs/remotes/origin/trunk\n" }] }); + expect(resolveDefaultBase(viaHead.deps)).toBe("origin/trunk"); + const noHead = fakeDeps({ execs: [{ code: 1, stdout: "" }] }); + expect(resolveDefaultBase(noHead.deps)).toBe("main"); +}); + +test("readMutateGlobs reads the conf and rejects a missing mutate array", () => { + const ok = fakeDeps({ files: { "stryker.conf.json": CONF } }); + expect(readMutateGlobs(ok.deps, "stryker.conf.json")).toEqual(["src/engine/**/*.ts", "!src/engine/**/*.test.ts"]); + const bad = fakeDeps({ files: { "stryker.conf.json": "{}" } }); + expect(() => readMutateGlobs(bad.deps, "stryker.conf.json")).toThrow("MUTATION_GATE_GLOBS"); +}); + +test("realDeps exec runs a real command and captures stdout", () => { + const d = realDeps(); + const r = d.exec(["git", "--version"]); + expect(r.code).toBe(0); + expect(r.stdout).toContain("git version"); +}); + +test("incremental: missing baseline loud-skips without spawning stryker", () => { + const { deps, calls, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/dispatch.ts": "a\n" }, + execs: [MB_OK, diffOf("M\0src/engine/dispatch.ts\0")], + env: { MUTATION_GATE_BASE: "origin/main", MUTATION_GATE_MODE: "incremental" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("skip-no-baseline"); + expect(calls.length).toBe(2); +}); + +test("incremental: with a baseline, argv has --incremental and no --mutate", () => { + const { deps, calls, outputs } = fakeDeps({ + files: { + "stryker.conf.json": CONF, + "src/engine/dispatch.ts": "a\n", + "reports/stryker-incremental.json": "{}", + }, + execs: [ + MB_OK, + diffOf("M\0src/engine/dispatch.ts\0"), + { + code: 0, + stdout: "", + writes: { + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/dispatch.ts": [{ mutator: "X", status: "Killed" }] }), + ), + }, + }, + ], + env: { MUTATION_GATE_BASE: "origin/main", MUTATION_GATE_MODE: "incremental" }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("pass"); + const stryker = calls[2]; + expect(stryker).toContain("--incremental"); + expect(stryker[stryker.indexOf("--incrementalFile") + 1]).toBe("reports/stryker-incremental.json"); + expect(stryker).not.toContain("--mutate"); +}); + +test("incremental: REQUIRE_BASELINE=false runs without a baseline", () => { + const { deps, outputs } = fakeDeps({ + files: { "stryker.conf.json": CONF, "src/engine/dispatch.ts": "a\n" }, + execs: [ + MB_OK, + diffOf("M\0src/engine/dispatch.ts\0"), + { + code: 0, + stdout: "", + writes: { + "reports/mutation/mutation.json": JSON.stringify( + makeReport({ "src/engine/dispatch.ts": [{ mutator: "X", status: "Killed" }] }), + ), + }, + }, + ], + env: { + MUTATION_GATE_BASE: "origin/main", + MUTATION_GATE_MODE: "incremental", + MUTATION_GATE_REQUIRE_BASELINE: "false", + }, + }); + expect(main(deps)).toBe(EXIT.ok); + expect(outputs[0].decision).toBe("pass"); +}); + +test("an unknown mode is an infra failure", () => { + const { deps, outputs } = fakeDeps({ env: { MUTATION_GATE_MODE: "yolo" } }); + expect(main(deps)).toBe(EXIT.infra); + expect(outputs[0].summary).toContain('unknown MUTATION_GATE_MODE "yolo"'); +}); + +test("CLI entry: invoking the script directly under bun actually runs main() (no silent no-op)", () => { + const result = Bun.spawnSync({ + cmd: ["bun", "scripts/mutation-gate.mjs"], + env: { ...process.env, MUTATION_GATE_MODE: "yolo", GITHUB_STEP_SUMMARY: undefined, GITHUB_OUTPUT: undefined }, + }); + expect(result.exitCode).toBe(2); + expect(result.stderr.toString()).toContain('unknown MUTATION_GATE_MODE "yolo"'); +}); diff --git a/src/engine/index.test.ts b/src/engine/index.test.ts index 5ab0efc..2b42cc6 100644 --- a/src/engine/index.test.ts +++ b/src/engine/index.test.ts @@ -576,6 +576,39 @@ test("inbound with no registration, or a handler without onInbound, is a silent expect(errors).toEqual([]); // neither path threw inside the scheduler task }); +test("inbound with no matching handler and a scenarios dep whose call returns undefined is a silent no-op (L2 seam optional at both levels, R-016)", async () => { + const errors: unknown[][] = []; + const orig = console.error; + console.error = (...a: unknown[]) => { + errors.push(a); + }; + try { + const config = loadConfig({ seed: 1 }); + const emitted: NormalizedMessage[] = []; + const engine = createEngine({ + config, + broker: { + emit: async (m) => { + emitted.push(m); + }, + }, + registry: makeRegistry, + record: (v) => ({ ...v, seq: 1, observedAt: "t" }) as Violation, + dispatch: createDispatchRegistry(), // no handler registered: select finds no match + scenarios: () => undefined, // L2 wired but no scenario runtime active yet + }); + engine.onInbound({ + message: { topic: "state/d1", payload: { status: "ok" } }, + meta: { clientId: "c", seq: 1, receivedAt: 0 }, + }); + await engine.idle(); + expect(emitted).toEqual([]); + } finally { + console.error = orig; + } + expect(errors).toEqual([]); // deps.scenarios?.()?.onInbound: a getter returning undefined never throws +}); + test("a '+' inside a topic level is not a wildcard: the subscribe materializes (level-exact detection)", async () => { const { engine, emitted } = buildEngine(); engine.onSubscribe("state/x+y"); diff --git a/stryker.conf.json b/stryker.conf.json index 4c4634e..a180729 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -6,5 +6,7 @@ "concurrency": 1, "mutate": ["src/engine/**/*.ts", "!src/engine/**/*.test.ts"], "reporters": ["clear-text", "progress", "html"], - "tempDirName": ".stryker-tmp" + "tempDirName": ".stryker-tmp", + "tsconfigFile": "tsconfig.stryker-unused.json", + "bun": { "timeout": 120000 } } diff --git a/test/stryker-tsconfig-noop.test.ts b/test/stryker-tsconfig-noop.test.ts new file mode 100644 index 0000000..39f28c3 --- /dev/null +++ b/test/stryker-tsconfig-noop.test.ts @@ -0,0 +1,26 @@ +// Tripwire for the stryker.conf.json tsconfigFile no-op (D-027). +// +// Stryker core's TSConfigPreprocessor loads the `typescript` module and calls +// ts.parseConfigFileTextToJson on the configured tsconfigFile whenever that +// path exists in the sandbox's file set — independent of any `checkers` +// config. TypeScript 7 ships tsc only (D-022), so that call crashes every +// mutation run. The conf therefore points tsconfigFile at a deliberately +// nonexistent path, which makes the preprocessor a no-op (files.get(...) +// returns undefined and the vulnerable call is skipped). +// +// That no-op holds only while (1) the sentinel path stays nonexistent and +// (2) the conf keeps declaring it. If this test fails: either someone +// created the sentinel file (delete or rename it), or the conf dropped the +// knob. A Stryker bump may also start validating the path — see D-027's +// residual-risk note; after any Stryker/runner bump, re-verify with a +// focused `stryker run` before trusting the gate. + +import { expect, test } from "bun:test"; +import { existsSync, readFileSync } from "node:fs"; + +test("stryker.conf.json pins the TSConfigPreprocessor no-op: sentinel declared and absent", () => { + const conf = JSON.parse(readFileSync("stryker.conf.json", "utf8")); + expect(conf.tsconfigFile).toBe("tsconfig.stryker-unused.json"); + expect(conf.checkers ?? []).toEqual([]); + expect(existsSync("tsconfig.stryker-unused.json")).toBe(false); +});