diff --git a/.github/actions/moon-run/action.yml b/.github/actions/moon-run/action.yml index 35b2982..9ba31fa 100644 --- a/.github/actions/moon-run/action.yml +++ b/.github/actions/moon-run/action.yml @@ -46,6 +46,3 @@ runs: - uses: appthrust/moon-ci-retrospect@v1 if: success() || failure() - - run: echo "${{ steps.raa.outputs.summary }}" >> $GITHUB_STEP_SUMMARY - shell: bash - if: always() diff --git a/.moon/tasks/rust.yml b/.moon/tasks/rust.yml index 102ef53..b3e1a42 100644 --- a/.moon/tasks/rust.yml +++ b/.moon/tasks/rust.yml @@ -10,22 +10,30 @@ fileGroups: - "tests/**/*" tasks: + + cargo: command: "cargo" + env: &env + CARGO_TERM_COLOR: "always" options: runInCI: false + debug-env: + script: printenv | grep -E '^(CARGO|MOON|PROTOC|PROTO|PATH)' + env: *env + build: &build command: "cargo build" inputs: - "@group(cargo)" - "@group(sources)" - env: &env - CARGO_TERM_COLOR: "always" + env: *env build-release: <<: *build command: "cargo build --release" + env: *env options: runInCI: false @@ -51,7 +59,7 @@ tasks: - "@group(cargo)" - "@group(sources)" - "@group(tests)" - - "/clippy.toml" + - "/.clippy.toml" env: *env lint-fix: @@ -61,7 +69,7 @@ tasks: runInCI: false test: - command: "cargo nextest run --no-fail-fast -p solidus-$project" + command: "cargo nextest run --no-fail-fast -p $project" inputs: - "@group(cargo)" - "@group(sources)" @@ -83,3 +91,5 @@ tasks: inputs: - "@group(cargo)" env: *env + + diff --git a/riff-cli/.goreleaser.yaml b/cli/.goreleaser.yaml similarity index 100% rename from riff-cli/.goreleaser.yaml rename to cli/.goreleaser.yaml diff --git a/riff-cli/Dockerfile b/cli/Dockerfile similarity index 100% rename from riff-cli/Dockerfile rename to cli/Dockerfile diff --git a/riff-cli/moon.yml b/cli/moon.yml similarity index 100% rename from riff-cli/moon.yml rename to cli/moon.yml diff --git a/engine/fuzz/README.md b/engine/fuzz/README.md index 4256b48..7b69ab7 100644 --- a/engine/fuzz/README.md +++ b/engine/fuzz/README.md @@ -33,23 +33,17 @@ Prevention. │ (cargo fuzz tmin) (cargo fuzz cmin) │ │ │ │ │ │ ▼ ▼ │ -│ fuzz/regressions// fuzz/corpus// │ -└──────────────────────┼─────────────────────────────────────────────────┘ - │ (Minimized crashes) - ▼ -┌────────────────────────────────────────────────────────────────────────┐ -│ 3. Regression Prevention (cargo test) │ -│ │ -│ [ engine/build.rs ] ──► Reads fuzz/regressions/ at compile time │ -│ │ │ -│ ▼ │ -│ Generates fuzz_regression_.rs in $OUT_DIR │ -│ │ │ -│ ▼ │ -│ [ cargo test ] ───────► Executes minimized crashes as unit tests │ -│ every build, preventing regressions │ -└────────────────────────────────────────────────────────────────────────┘ -` `` +│ fuzz/regressions// fuzz/corpus//│ +└──────────────────────┤───────────────────────────────────┤─────────────┘ + │ (Minimized crashes) │ (Compacted) + ▼ ▼ +┌──────────────────────────────────────────────────────────────────────────┐ +│ 3. Persistence │ +│ │ +│ [ regressions/ ] ──► Committed to main (datatest-stable auto-discovers)│ +│ [ corpus/ ] ───────► Synced to orphan `corpus` branch + CI cache │ +└──────────────────────────────────────────────────────────────────────────┘ +``` ## How It Works @@ -89,6 +83,48 @@ Once a crash is minimized into `regressions/`, it becomes a permanent test with - These tests execute on every build, ensuring historical crashes can never silently regress. +## Corpus Management + +Corpus inputs are stored with two-tier persistence to avoid repository bloat while maintaining +shared progress across CI runs and local sessions. + +### CI: Actions Cache + Orphan Branch + +The `fuzz-cache` composite action manages both tiers via `mode: pull` and `mode: push` inputs: + +- `actions/cache@v4` provides fast, per-run restoration. Cache keys are isolated by main vs PR + (`fuzz-corpus-main-*` vs `fuzz-corpus-pr--*`). PR caches fall back to main if no + PR-specific cache exists. +- An orphan `corpus` branch provides durable, shared storage. On `pull`, the action fetches the + branch and seeds the corpus directory via `rsync`. On `push`, it syncs the updated corpus back + with retry logic. Branch operations are main-only (silently skipped for PRs). The branch is + self-initializing: created automatically on first push if it doesn't exist. + +Workflow sequence (main-fuzz.yml): +1. `fuzz-cache mode: pull` restores from CI cache, then seeds from `corpus` branch +2. `fuzz-saturate` runs all four engine/target variants +3. `fuzz-cache mode: push` syncs compacted corpus back to `corpus` branch +4. `fuzz-triage` minimizes crashes into `regressions/` (only if crashes found) + +### Local: Git Worktree + +For local sessions, a git worktree at `//.corpus/` provides access to the orphan branch +without switching branches: + +```bash +# One-time setup +git worktree add .corpus corpus + +# Typical local fuzzing session +moon run engine-fuzz:fuzz-corpus-pull # seed from CI's latest corpus +moon run engine-fuzz:fuzz-saturate # fuzz locally +moon run engine-fuzz:fuzz-corpus-push # triage, compact, push back +``` + +The `fuzz-corpus-push` task triages crashes, compacts the corpus, pulls any new CI discoveries +made during the session, and pushes the merged result to the `corpus` branch. + + ## Moon Task Graph Moon manages the parallelization of the full 2x2 engine/target matrix. @@ -97,23 +133,27 @@ Moon manages the parallelization of the full 2x2 engine/target matrix. fuzz ├── fuzz-saturate (parallel) │ ├── fuzz-structured-saturate -│ │ ├── libfuzz-structured-saturate (fuzz-run.sh ... 86400 180) -│ │ └── libafl-structured-saturate (fuzz-run.sh ... 86400 180 + FUZZ_FEATURES) +│ │ ├── libfuzz-structured-saturate (fuzz-run.sh ... 14400 2) +│ │ ├── libafl-structured-saturate (fuzz-run.sh ... 14400 2 + FUZZ_FEATURES) │ │ └── compact (fuzz-manage.sh compact) │ └── fuzz-unstructured-saturate │ ├── libfuzz-unstructured-saturate -│ └── libafl-unstructured-saturate +│ ├── libafl-unstructured-saturate │ └── compact └── fuzz-triage (sequential, after saturate) ├── triage parse_document_unstructured └── triage parse_document_structured -fuzz-ci (parallel, all four variants with -runs=0) -├── libfuzz-structured-ci -├── libfuzz-unstructured-ci -├── libafl-structured-ci -└── libafl-unstructured-ci -` `` +fuzz-triage (sequential, after gate signals crashes) +├── triage parse_document_unstructured +└── triage parse_document_structured + +fuzz (quick replay, libFuzzer only, -runs=0) +├── libfuzz-structured (fuzz-run.sh ... --replay) +└── libfuzz-unstructured (fuzz-run.sh ... --replay) + +fuzz-corpus-pull / fuzz-corpus-push (local worktree sync) +``` ### Commands @@ -129,7 +169,11 @@ moon run engine-fuzz:fuzz # Triage only: minimize new crash artifacts into regressions/ moon run engine-fuzz:fuzz-triage -` `` + +# Corpus sync (local, requires git worktree at .corpus/) +moon run engine-fuzz:fuzz-corpus-pull +moon run engine-fuzz:fuzz-corpus-push +``` ## CI Automation @@ -139,23 +183,25 @@ Two GitHub Actions workflows automate fuzzing in CI. ### Trunk Saturation (`main-fuzz.yml`) Runs weekly (Saturday 2 AM UTC) and on manual dispatch. Discovers all projects with a `fuzz-ci` -task via `moon-q-projects`, then runs `fuzz-saturate` across them in a matrix. If crashes are found, -it triages them automatically and opens a PR with the minimized regression files via the `pr-create` -composite action. +task via `moon-q-projects`, then runs `fuzz-saturate` across them in a matrix. Corpus is pulled +from cache and the orphan branch before fuzzing, and pushed back after. If crashes are found, it +triages them and opens a PR with the minimized regression files via the `pr-create` composite +action. ### PR Saturation (`pr-fuzz.yml`) -Triggered by adding the `fuzz` label to a PR, or via `workflow_dispatch` with a PR number. Runs the -same discover/saturate/triage cycle against the PR branch. Regression files are committed directly -to the PR branch (attributed to the last branch author). Results are reported as a commit status and -sticky PR comment via the `pr-set-status` composite action. +Triggered by adding the `fuzz` label to a PR, or via `workflow_dispatch` with a PR number. Runs +the same discover/saturate/triage cycle against the PR branch. Corpus is seeded from cache only +(falls back to main cache, no branch interaction). Regression files are committed directly to +the PR branch. Results are reported as a commit status and sticky PR comment via the +`pr-set-status` composite action. Both workflows use these shared composite actions: - `setup`: Checkout, toolchain installation, Cargo/Moon/pnpm caches - `moon-q-projects`: Discover fuzz-capable projects with metadata - `moon-run`: Execute moon tasks with retrospect reporting -- `fuzz-cache`: Per-project corpus caching with main/PR isolation +- `fuzz-cache`: Two-tier corpus persistence (CI cache + orphan branch) - `git-stage-artifacts`: Download and place matrix artifacts into project directories - `git-commit-push`: Commit and push with last-author attribution - `pr-create`: Idempotent PR creation (trunk workflow) @@ -165,10 +211,9 @@ Both workflows use these shared composite actions: ## State Management -- `fuzz/corpus//`: Gitignored. Persisted exclusively via GitHub Actions caching - (`fuzz-cache` action) to avoid repository bloat. -- `fuzz/artifacts//`: Gitignored. Temporary storage for raw crashes, minimized files, and - log output. -- `fuzz/regressions//`: Committed. Tiny, minimized crash files that serve as source of - truth for `cargo test`. - \ No newline at end of file +| Directory | Tracked | Persistence | +|---|---|---| +| `fuzz/corpus//` | Gitignored | CI cache (`actions/cache`) + orphan `corpus` branch. Main and PR caches isolated; branch ops main-only. | +| `fuzz/artifacts//` | Gitignored | Ephemeral. Raw crashes, minimized files, crash summaries, logs. | +| `fuzz/regressions//` | Committed | Minimized crash files. Source of truth for `cargo test` via `datatest-stable`. | +| `.corpus/` (repo root) | Gitignored | Local git worktree pointing at the orphan `corpus` branch. | diff --git a/slash-dart/moon.yml b/sdks/dart/moon.yml similarity index 100% rename from slash-dart/moon.yml rename to sdks/dart/moon.yml diff --git a/slash-elixir/moon.yml b/sdks/elixir/moon.yml similarity index 100% rename from slash-elixir/moon.yml rename to sdks/elixir/moon.yml diff --git a/slash-go/moon.yml b/sdks/go/moon.yml similarity index 100% rename from slash-go/moon.yml rename to sdks/go/moon.yml diff --git a/slash-haskell/moon.yml b/sdks/haskell/moon.yml similarity index 100% rename from slash-haskell/moon.yml rename to sdks/haskell/moon.yml diff --git a/slash-java/moon.yml b/sdks/java/moon.yml similarity index 100% rename from slash-java/moon.yml rename to sdks/java/moon.yml diff --git a/slash-javascript/moon.yml b/sdks/javascript/moon.yml similarity index 100% rename from slash-javascript/moon.yml rename to sdks/javascript/moon.yml diff --git a/slash-ocaml/moon.yml b/sdks/ocaml/moon.yml similarity index 100% rename from slash-ocaml/moon.yml rename to sdks/ocaml/moon.yml diff --git a/slash-php/moon.yml b/sdks/php/moon.yml similarity index 100% rename from slash-php/moon.yml rename to sdks/php/moon.yml diff --git a/slash-python/moon.yml b/sdks/python/moon.yml similarity index 100% rename from slash-python/moon.yml rename to sdks/python/moon.yml diff --git a/slash-ruby/moon.yml b/sdks/ruby/moon.yml similarity index 100% rename from slash-ruby/moon.yml rename to sdks/ruby/moon.yml diff --git a/slash-rust/moon.yml b/sdks/rust/moon.yml similarity index 100% rename from slash-rust/moon.yml rename to sdks/rust/moon.yml diff --git a/slash-web/moon.yml b/sdks/web/moon.yml similarity index 100% rename from slash-web/moon.yml rename to sdks/web/moon.yml diff --git a/slash-zig/moon.yml b/sdks/zig/moon.yml similarity index 100% rename from slash-zig/moon.yml rename to sdks/zig/moon.yml diff --git a/website/src/content/docs/soundcheck.md b/website/src/content/docs/soundcheck.md new file mode 100644 index 0000000..a1640a7 --- /dev/null +++ b/website/src/content/docs/soundcheck.md @@ -0,0 +1,176 @@ +--- +title: Soundcheck +description: How Solidus is tested. Three layers, property-based testing, structured fuzz testing, and five provable guarantees. +--- + +The parser is a total function. It accepts any UTF-8 input and always returns a valid result. It +cannot panic, cannot return an error, and cannot fail. The testing strategy is designed to prove +that guarantee holds under adversarial conditions. + +## Three-Layer Test Architecture + +Tests are organized by scope, not by type. + +**Layer 1: Unit tests** sit inside every module file, exercising individual functions with direct +calls. Each parsing stage (normalization, line joining, line classification, command accumulation, +text collection, finalization) has its own unit tests that verify behavior in isolation. These are +the atoms. + +**Layer 2: Integration tests** span multiple modules within the application layer. They exercise +cross-module composition through `parse_document`, including every parsing example from Appendix B +of the Syntax Specification. These are the molecules. + +**Layer 3: Cross-layer tests** cover the full public API surface. They validate structural +invariants that span the entire pipeline: ID sequencing, line-range consistency, deterministic +output, and roundtrip fidelity. + +All three layers run on every commit via `cargo nextest run`. Property tests (see below) are gated +behind a `tdd` feature flag so watch-mode iteration stays fast while the full suite runs before +every push. + +## Property-Based Testing + +Randomly generated inputs validate structural invariants that must hold for every possible document. +The engine uses [proptest](https://crates.io/crates/proptest) to generate inputs at scale, testing +properties like: + +- **Totality:** `parse_document` never panics on arbitrary input up to 500 characters. +- **Determinism:** parsing the same input twice always produces identical output. +- **ID sequencing:** command IDs are always `cmd-0`, `cmd-1`, ... in order. Text block IDs are + always `text-0`, `text-1`, ... in order. +- **Completeness:** every physical line in the input is accounted for by exactly one command or text + block. No line is lost or double-counted. +- **Text-only invariant:** input with no slash-prefixed lines produces zero commands and at least + one text block. +- **Fence body preservation:** fenced content passes through the state machine verbatim. No joining, + no escaping, no transformation. +- **Unclosed fence warning:** any fence reaching EOF without a closer always produces exactly one + `unclosed_fence` warning. + +When proptest finds a failing input, it automatically shrinks it to the minimal reproducing case and +writes it to a regression file. These files are committed to version control. Once a failure class +is found, it can never silently return. + +## Fuzz Testing + +Fuzz testing is where the parser meets genuine adversarial input. Two fuzz targets attack the parser +from complementary angles, and each target runs on two independent fuzzing engines. + +### Two Targets + +**`parse_document_unstructured`** feeds raw arbitrary bytes (filtered to valid UTF-8) into +`parse_document`. This is pure chaos: the fuzzer has no concept of slash commands, fences, or line +structure. It explores the full input space looking for panics and invariant violations. + +```rust +fuzz_target!(|data: &[u8]| { + if let Ok(input) = std::str::from_utf8(data) { + let result = parse_document(input); + assert!(!result.version.is_empty()); + // ... sequential IDs, valid argument modes, determinism + } +}); +``` + +**`parse_document_structured`** uses the `engine-fuzz-common` crate to generate syntactically +plausible documents via Rust's `Arbitrary` trait. A `FuzzDoc` is a sequence of up to 20 +`Fragment` variants (text lines, single-line commands, fenced commands, unclosed fences, joined +commands, invalid slashes, blanks) that are rendered into a concrete input string before parsing. +This approach biases the fuzzer toward structurally interesting inputs that exercise deep parser +states. + +```rust +fuzz_target!(|doc: FuzzDoc| { + let input = render_doc(&doc); + let result = parse_document(&input); + assert_ids_sequential(&result); + assert_argument_modes(&result); + assert_unclosed_fence_warning(&doc, &result); + // ... determinism +}); +``` + +Both harnesses assert the same invariants: version is populated, IDs are sequential, argument modes +are valid (`SingleLine` or `Fence`), and output is deterministic. The structured harness adds a +semantic check: if the last fragment is an `UnclosedFence`, the result must contain an +`unclosed_fence` warning. + +### Two Engines + +Each target compiles against two fuzzing engines via Cargo features: + +| Engine | Feature flag | Strength | +|---|---|---| +| libFuzzer | `libfuzzer` | Industry-standard coverage-guided fuzzer from LLVM | +| LibAFL | `libafl` | Rust-native fuzzer with advanced scheduling and mutation strategies | + +A compile-time guard prevents enabling both simultaneously. The 2x2 matrix (2 targets x 2 engines) +provides four independent attack vectors against the parser. + +### Shared Assertion Library + +The `engine-fuzz-common` crate is shared between the fuzz harnesses and the regression test +harnesses. It contains: + +- **Types:** `FuzzDoc`, `Fragment`, `CmdName`, `Payload`, `Header`, `FenceLang`, `FenceBody`, + `InvalidSlashKind` with bounded `Arbitrary` implementations. +- **Renderer:** `render_doc` converts a `FuzzDoc` into a concrete input string, sanitizing + newlines, escaping backticks, and stripping trailing join markers between fragments. +- **Assertions:** `assert_ids_sequential`, `assert_argument_modes`, `assert_unclosed_fence_warning` + validate structural invariants shared by harnesses and regression tests. + +### Regression Tests + +Crash inputs discovered by fuzzing are minimized and saved to `fuzz/regressions/`. Two +`datatest-stable` harnesses auto-discover regression files and replay them as standard +`cargo test` cases: + +- `fuzz_regression_parse_document_unstructured` replays raw bytes from + `fuzz/regressions/parse_document_unstructured/`. +- `fuzz_regression_parse_document_structured` deserializes `FuzzDoc` via `Arbitrary` from + `fuzz/regressions/parse_document_structured/`. + +Both replay harnesses assert the same invariants as the live fuzz targets, plus determinism (parse +twice, compare results). To add a regression: drop a file in the appropriate directory and run +tests. No test registration, no boilerplate. The `datatest-stable` harness discovers it +automatically. + +### CI Automation + +Two GitHub Actions workflows manage fuzz runs: + +**`main-fuzz.yml` (weekly saturation)** runs every Saturday at 2:00 AM UTC. It discovers all +projects with a `fuzz-ci` moon task, pulls the cached corpus, runs each target for up to 4 hours +with 4 parallel jobs, then pushes the grown corpus back to cache. If crashes are found, the +workflow triages and minimizes them, uploads artifacts, and opens a pull request to commit the new +regression files. + +**`pr-fuzz.yml` (label-triggered)** activates when a `fuzz` label is added to a pull request (or +via manual dispatch). It runs the same saturation flow against the PR branch for up to 5 hours per +target. On completion, it commits any regression files directly to the PR branch and posts a status +comment: green for clean, red with file counts and artifact links for crashes. + +Both workflows use the `fuzz-cache` composite action for two-tier corpus persistence: CI cache for +fast restore, with the corpus growing monotonically across runs. Paths are resolved from moon +project metadata (`corpus-path`, `regressions-path`). + +## What We Prove + +These aren't aspirations. They're properties enforced by the architecture and verified by the test +suite. + +**Deterministic.** Identical input always produces identical output. The engine uses no randomness, +no HashMap iteration order, and no floating point arithmetic. + +**Total.** Every input produces a valid `ParseResult`. There is no `Result<_, E>` return type. There +is no error path. Empty input returns empty results. Malformed input returns partial results with +warnings. + +**Safe.** No `unsafe` code anywhere in the engine. The only runtime dependency is `thiserror` for +error type derivation. + +**Faithful.** Parse, format, parse again yields a structurally equivalent result. This is the +roundtrip fidelity invariant defined in the Syntax Specification (Section 8.3). + +**Opaque.** The parser never interprets, tokenizes, or validates argument content. Your payload is +your business. The parser's job is to find the boundaries and hand you the content verbatim.