chore: cut local cargo test from 6m43s to 20s, fix two loop defects

.github/workflows/ci.yml

@@ -62,8 +62,13 @@ jobs:
       - name: cargo clippy
         run: cargo clippy --workspace --all-targets -- -D warnings
 
+      # `--include-ignored`: the slow acceptance/robustness gates
+      # (cross_product_matrix, aarch64_elf_fuzz, dotnet_dnfile_fuzz -- ~383 s
+      # of the suite's ~402 s) are `#[ignore]`d so a local `cargo test` stays
+      # a ~20 s inner loop. CI is where they must actually run, so it opts
+      # back in. Dropping this flag silently removes them from coverage.
       - name: cargo test
-        run: cargo test --workspace
+        run: cargo test --workspace -- --include-ignored
 
       # Gate J2 (docs/BENCHMARKS.md): `--jobs N` output must be byte-identical to
       # `--jobs 1`. `cargo test` above already runs the library-level half
@@ -125,6 +130,11 @@ jobs:
 
       # Build and test, not just check: `cargo check` would miss anything that
       # only fails at codegen or link time.
+      # Deliberately without `--include-ignored`, unlike the `build` job: this
+      # job asks "does the workspace still compile and pass on the MSRV", and
+      # the ignored gates are still *compiled* here (that is the codegen/link
+      # coverage MSRV cares about). Re-running ~383 s of acceptance matrix on
+      # a second runner would not test anything about the Rust version.
       - name: cargo test on the declared MSRV
         run: cargo test --workspace --locked
 

AGENTS.md

@@ -17,14 +17,24 @@ results.
 
 ## Feedback loops
 
-| Loop | Command | When |
-|---|---|---|
-| Inner | `cargo test -p capa-x --test features_parity` | Every change |
-| Mid | `python3 scripts/difftest.py --mode full --samples scripts/corpus-smoke.txt --capa-cli target/release/capa-x --jobs 6` | Every commit |
-| Outer | The mid command with `scripts/corpus-outer.txt` | Before merge or release |
+| Loop | Command | When | Cost |
+|---|---|---|---|
+| Inner | `cargo test -p capa-x --test features_parity` | Every change | ~7 s |
+| Mid | `python3 scripts/difftest.py --mode full --samples scripts/corpus-smoke.txt --capa-cli target/release/capa-x --jobs 6` | Every commit | ~40 s |
+| Outer | The mid command with `scripts/corpus-outer.txt` | Before merge or release | ~5 min |
+
+Use the cheapest loop that can observe the change. `cargo test --workspace`
+(~20 s) is the broader local check; the slow acceptance and robustness gates
+are `#[ignore]`d and belong to CI, which runs them with `--include-ignored`.
+Run them locally with the same flag before merge, not on every edit.
 
 `--jobs 1` is the semantic baseline. Any byte difference across job counts is
 a bug. Never carry a measurement forward from a note or cached report.
 
+Both difftest corpora carry a per-sample baseline (`<corpus>.expected.json`),
+so a run reports regressions against known diffs rather than the raw presence
+of a diff. Re-record with `--write-expected` only for a deliberate change, in
+the same commit, with the reason stated.
+
 See [`CONTRIBUTING.md`](CONTRIBUTING.md) for setup, complete checks, and the
 pull request checklist.

CHANGELOG.md

@@ -9,7 +9,28 @@ tracked per class in [KNOWN_DIVERGENCES.md](KNOWN_DIVERGENCES.md).
 
 ## [Unreleased]
 
-No unreleased changes.
+### Fixed
+
+- `scripts/difftest.py` defaulted `--capa-cli` to `target/debug/capa`, which
+  is not the binary's name. A run that omitted the flag could not find it, and
+  on a tree that still held a stale artifact under the old name it would
+  difftest that instead -- and since the harness caches capa-x's side by
+  binary contents, the result would look clean and self-consistent while
+  measuring the wrong build. The default is now `target/debug/capa-x`.
+
+### Changed
+
+- The slow acceptance and robustness gates
+  (`capa-x/tests/cross_product_matrix.rs`, `aarch64_elf_fuzz.rs`,
+  `dotnet_dnfile_fuzz.rs`) are `#[ignore]`d: ~383 s of a ~402 s suite, none of
+  which an ordinary edit moves. `cargo test --workspace` is now ~20 s
+  locally. CI runs the full set with `--include-ignored`, so coverage is
+  unchanged.
+- `scripts/corpus-outer.expected.json` records the 200-sample outer corpus
+  baseline (98.55% rule-level agreement, 161/200 identical, 91 divergences,
+  0 errors). Without it the outer loop had no baseline to resolve and exited
+  nonzero on every run, so the pre-merge gate `AGENTS.md` documents could not
+  pass; it now reports regressions against known diffs.
 
 ## [1.0.0]
 

CONTRIBUTING.md

@@ -43,29 +43,62 @@ Python reference in `.venv`, and verifies every pin against `PINNED.md`.
 
 Use the cheapest loop that can observe the change:
 
-| Loop | Command | When |
-|---|---|---|
-| Inner | `cargo test -p capa-x --test features_parity` | Every change |
-| Mid | `python3 scripts/difftest.py --mode full --samples scripts/corpus-smoke.txt --capa-cli target/release/capa-x --jobs 6` | Every commit |
-| Outer | The mid command with `scripts/corpus-outer.txt` | Before merge or release |
+| Loop | Command | When | Cost |
+|---|---|---|---|
+| Inner | `cargo test -p capa-x --test features_parity` | Every change | ~7 s |
+| Mid | `python3 scripts/difftest.py --mode full --samples scripts/corpus-smoke.txt --capa-cli target/release/capa-x --jobs 6` | Every commit | ~40 s |
+| Outer | The mid command with `scripts/corpus-outer.txt` | Before merge or release | ~5 min |
 
 The inner loop is the transcribed upstream feature contract. The mid loop is
 the smoke regression guard. The outer loop measures rule-level agreement and
 the extra-rule count across the full corpus.
 
+The difftest costs above are what you pay *after a code change*, on a 10-core
+M1 Max; treat them as an order of magnitude, not a benchmark. Both sides are
+cached under `.cache/`: the Python reference by sample hash, capa-x's own
+output by binary contents. So a rebuilt binary invalidates only the capa-x
+side (mid ~40 s, outer ~5 min), and re-running either loop without rebuilding
+is 1-2 s. `.cache/` is disposable, but deleting it costs a one-time
+re-analysis of the corpus under the pinned Python reference, which is slow
+(minutes to hours) -- prefer `--no-rust-cache` when you want to distrust a
+capa-x result, rather than clearing the whole cache.
+
 `--jobs 1` is the semantic baseline for analysis output. Harness-level jobs
 only parallelize independent samples. Any byte difference between analysis
 with `--jobs 1` and another job count is a bug.
 
-Before submitting a pull request, run:
+### Before pushing
 
 ```bash
-cargo build && cargo test
-cargo fmt --check
+cargo fmt --all -- --check
 cargo clippy --workspace --all-targets -- -D warnings
+cargo test --workspace
+```
+
+`cargo test --workspace` is about 20 seconds. The slow acceptance and
+robustness gates are `#[ignore]`d so this stays an inner loop: the
+cross-product matrix and the two byte-flip fuzz suites are ~383 s of a ~402 s
+suite between them, and an ordinary edit does not move them. CI runs the full
+set with `--include-ignored` on all three platforms, so pushing without having
+run them locally is expected, not a shortcut.
+
+### Before merge or release
+
+Run what CI cannot run cheaply on every push, plus the ignored gates:
+
+```bash
+cargo test --workspace -- --include-ignored
 scripts/check_env.sh
+python3 scripts/difftest.py --mode full --samples scripts/corpus-outer.txt \
+  --capa-cli target/release/capa-x --jobs 6
 ```
 
+Both difftest corpora have a recorded per-sample baseline
+(`<corpus>.expected.json`), so the exit status reports *regressions* against
+known diffs rather than the raw presence of any diff. A deliberate change to
+what matches is re-recorded with `--write-expected`, in the same commit, with
+the reason in the pull request.
+
 Use the differential harness for behavioral changes:
 
 ```bash

capa-x/tests/aarch64_elf_fuzz.rs

@@ -53,7 +53,13 @@ fn fixture_bytes(name: &str) -> Vec<u8> {
     std::fs::read(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display()))
 }
 
+// 900 mutants through the full ELF pipeline, single-threaded: ~166 s, the
+// second largest cost in `cargo test --workspace`. It guards against rare
+// panics on malformed input, not against the behavior an ordinary edit moves,
+// so it is a pre-merge gate rather than an inner-loop test. CI runs it via
+// `cargo test --workspace -- --include-ignored`.
 #[test]
+#[ignore = "slow robustness gate; run with --include-ignored"]
 fn mutated_aarch64_elf_samples_never_panic() {
     let default_hook = panic::take_hook();
     panic::set_hook(Box::new(|_| {}));

capa-x/tests/cross_product_matrix.rs

@@ -23,6 +23,12 @@
 //! separate gate -- this only proves the Rust-side shape is
 //! self-consistent, matching `capa-x/tests/schema_roundtrip.rs`'s own
 //! stated scope.
+//!
+//! Every test here is `#[ignore]`d: the four of them together are ~197 s, the
+//! single largest cost in `cargo test --workspace`, because each reloads the
+//! 1,042-rule corpus and runs all 11 shapes (cell 1 also at four job counts).
+//! This is a pre-merge acceptance matrix, not a test an ordinary edit moves.
+//! CI runs it via `cargo test --workspace -- --include-ignored`.
 
 #![allow(clippy::unwrap_used, clippy::expect_used)]
 
@@ -181,6 +187,7 @@ fn normalized_json(doc: &ResultDocument) -> serde_json::Value {
 /// byte-identical after timestamp normalization, matching AGENTS.md's
 /// blanket rule for every backend.
 #[test]
+#[ignore = "slow acceptance gate; run with --include-ignored"]
 fn jobs_1_matches_default_for_every_shape() {
     let rules = ruleset();
     let mut failures = Vec::new();
@@ -226,6 +233,7 @@ fn jobs_1_matches_default_for_every_shape() {
 /// thread count (e.g. an unstable sort, an uninitialized-memory read, a
 /// HashMap iteration order leaking into output).
 #[test]
+#[ignore = "slow acceptance gate; run with --include-ignored"]
 fn repeated_output_is_identical_for_every_shape() {
     let rules = ruleset();
     let mut failures = Vec::new();
@@ -260,6 +268,7 @@ fn repeated_output_is_identical_for_every_shape() {
 /// produced document back into `ResultDocument` and re-serializes to the
 /// same structure. See the module doc for how this differs from J14.
 #[test]
+#[ignore = "slow acceptance gate; run with --include-ignored"]
 fn result_document_round_trips_for_every_shape() {
     let rules = ruleset();
     let mut failures = Vec::new();
@@ -340,6 +349,7 @@ impl Rng {
 const MUTANTS_PER_SHAPE: usize = 15;
 
 #[test]
+#[ignore = "slow acceptance gate; run with --include-ignored"]
 fn malformed_input_never_panics_through_the_full_pipeline() {
     let rules = ruleset();
     let default_hook = panic::take_hook();

capa-x/tests/dotnet_dnfile_fuzz.rs

@@ -60,7 +60,12 @@ fn corpus_dir() -> PathBuf {
         .join("tests/testfiles/dotnet")
 }
 
+// 2,400 mutants through `dotnet::load`, single-threaded: ~20 s. Same shape as
+// `aarch64_elf_fuzz.rs` -- a malformed-input panic guard, not a test an
+// ordinary edit moves. CI runs it via
+// `cargo test --workspace -- --include-ignored`.
 #[test]
+#[ignore = "slow robustness gate; run with --include-ignored"]
 fn mutated_dotnet_samples_never_panic() {
     // The vendored fork's own panic hook still prints backtraces on our
     // caught panics; keep test output readable and restore it afterward.