diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 622d7f9b..5f044101 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -14,7 +14,7 @@ "category": "security", "source": "./plugins/hawkscan", "homepage": "https://github.com/stackhawk/agent-skills/tree/main/plugins/hawkscan", - "version": "2.2.0" + "version": "2.4.0" }, { "name": "stackhawk-api", @@ -22,7 +22,7 @@ "category": "security", "source": "./plugins/api", "homepage": "https://github.com/stackhawk/agent-skills/tree/main/plugins/api", - "version": "2.2.0" + "version": "2.4.0" }, { "name": "hawkscan-ci", @@ -30,7 +30,7 @@ "category": "security", "source": "./plugins/hawkscan-ci", "homepage": "https://github.com/stackhawk/agent-skills/tree/main/plugins/hawkscan-ci", - "version": "2.2.0" + "version": "2.4.0" }, { "name": "stackhawk-data-seed", @@ -38,7 +38,7 @@ "category": "security", "source": "./plugins/stackhawk-data-seed", "homepage": "https://github.com/stackhawk/agent-skills/tree/main/plugins/stackhawk-data-seed", - "version": "2.2.0" + "version": "2.4.0" }, { "name": "wingman", @@ -46,7 +46,7 @@ "category": "security", "source": "./plugins/wingman", "homepage": "https://github.com/stackhawk/agent-skills/tree/main/plugins/wingman", - "version": "2.2.0" + "version": "2.4.0" }, { "name": "stackhawk-optimize", @@ -54,7 +54,7 @@ "category": "security", "source": "./plugins/optimize", "homepage": "https://github.com/stackhawk/agent-skills/tree/main/plugins/optimize", - "version": "2.2.0" + "version": "2.4.0" } ] } diff --git a/.claude/skills/skill-authoring/SKILL.md b/.claude/skills/skill-authoring/SKILL.md index 582cd20a..a4f1ca31 100644 --- a/.claude/skills/skill-authoring/SKILL.md +++ b/.claude/skills/skill-authoring/SKILL.md @@ -1,6 +1,6 @@ --- name: skill-authoring -version: 2.2.0 +version: 2.4.0 description: > Guides authoring and maintaining agent skills in this repository — enforcing Anthropic best practices, bumping versions, regenerating Cursor rules, and @@ -106,6 +106,32 @@ immediately before the first `##` section. Format: Shorter files may omit the ToC. +### Examples vs. placeholders (avoid literal-match traps) + +Agents copy **runnable code blocks verbatim** far more readily than they adapt them. A +concrete app-specific value baked into a command is therefore a trap: the agent runs it +literally against a different codebase and silently does the wrong thing (curls a route that +doesn't exist, greps a directory that isn't there). + +The rule that prevents this: + +- **Runnable code blocks use ``s that fail loudly if pasted as-is.** Prefer + `curl "$HOST"` over `curl "$HOST/authors"` — a literal paste of the placeholder + form breaks visibly, while a literal paste of `/authors` hits the wrong route silently. Put + the concrete value in an adjacent comment as a clearly-labeled `e.g.` +- **Concrete examples live in prose or comments, marked `e.g.`** There they aid comprehension + without being executable, so literal-matching them is harmless. +- **Literal values are fine only when they are genuinely universal** — e.g. probing the real, + fixed served-spec paths (`/openapi.json`, `/v3/api-docs`) is correct precisely because those + are the same across every app of that framework. If a value changes per codebase, it is a + placeholder, not a literal. +- **Command tables (grep/derivation recipes) are illustrative starting points.** Say so once + near the table, and point them at the repo's real source root and filenames rather than a + literal `src/` or `openapi.yaml`. + +The test: *"if an agent runs this exact text against a completely different repo, does it fail +loudly or mislead silently?"* Silent-mislead means parameterize it. + --- ## Creating a New Plugin diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 8e582044..dd44546b 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "stackhawk", - "version": "2.2.0", + "version": "2.4.0", "description": "Dynamic application and API security testing (DAST) powered by HawkScan. Configure scans, interpret findings, and query the StackHawk platform API.", "author": { "name": "StackHawk", diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index fc98d120..603d7d01 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -17,7 +17,7 @@ on: required: true default: "all" type: choice - options: [all, claude-code, codex, agy, cursor] + options: [all, claude-code, codex, cursor] # agy disabled (OAuth-only, can't auth in CI) rubric: description: "Also run the qualitative rubric grader (extra ANTHROPIC_API_KEY cost)" required: false @@ -256,105 +256,109 @@ jobs: retention-days: 30 # ── Antigravity (agy) — replaces Gemini ─────────────────────────────────── - eval-agy: - name: agy / ${{ matrix.skill }} / ${{ matrix.model }} - runs-on: ubuntu-latest - needs: validate-config - if: | - inputs.platform == 'all' || - inputs.platform == 'agy' - strategy: - fail-fast: false - matrix: - skill: [hawkscan, api, stackhawk-data-seed, hawkscan-ci, optimize] - model: [default] - - steps: - - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v5 - - name: Install Claude Code CLI (native, rubric grader) - run: | - curl -fsSL https://claude.ai/install.sh | bash - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Install agy CLI - run: | - # /cli/install.sh is the real bootstrapper; /install-cli returns the - # site's HTML landing page (piping that into bash is what broke before). - curl -fsSL https://antigravity.google/cli/install.sh | bash - echo "$HOME/.local/bin" >> "$GITHUB_PATH" # installer drops `agy` here - continue-on-error: true # don't abort the job — evals records any launch failure - - - name: Verify agy CLI - run: agy --version - continue-on-error: true # if unavailable, the eval run captures it as a per-prompt error - - - name: Install StackHawk plugins - env: - # NOTE: agy ignores API-key env vars and forces interactive Google OAuth - # (verified empirically; upstream antigravity-cli#78 open). This mapping is - # kept for if/when key auth lands; headless CI needs the OAuth-token - # injection below. - ANTIGRAVITY_API_KEY: ${{ secrets.AGY_API_KEY }} - run: | - echo y | agy plugin install plugins/hawkscan - echo y | agy plugin install plugins/api - echo y | agy plugin install plugins/stackhawk-data-seed - echo y | agy plugin install plugins/hawkscan-ci - echo y | agy plugin install plugins/optimize - continue-on-error: true # depends on agy CLI; best-effort so evals still runs - - # HawkScan 6 ships as a single self-contained native binary (no JDK) that - # provides BOTH scanning (`hawk`) and the platform/API CLI (`hawk op …`). - # Install it from the download manifest (source of truth for version + URL) - # into /usr/local/bin (already on PATH). One install covers the hawkscan - # skill's CLI path AND the api skill + hawkscan Step-1 dedup checks. - - name: Install hawk CLI (combined binary, manifest-driven) - run: | - set -euo pipefail - MANIFEST=https://download.stackhawk.com/hawkdocs/hawk.manifest.json - url="$(curl -fsSL "$MANIFEST" | jq -r '.latest.assets[] - | select(.asset.group == "linux-x64" and (.url | endswith("/hawk"))) | .url')" - [ -n "$url" ] || { echo "ERROR: no linux-x64 hawk binary in manifest"; exit 1; } - echo "Installing hawk from $url" - curl -fsSL "$url" -o /tmp/hawk - chmod +x /tmp/hawk - sudo mv /tmp/hawk /usr/local/bin/hawk - continue-on-error: true # absence is captured per-prompt in the eval traces - - name: Verify hawk CLI - run: | - hawk version - hawk op --help - continue-on-error: true # absence is captured per-prompt in the eval traces - - - name: Run ${{ matrix.skill }} evals - env: - # agy ignores API-key env vars and forces OAuth (upstream #78); kept for - # if/when key auth lands. Headless CI needs OAuth-token injection (TODO). - ANTIGRAVITY_API_KEY: ${{ secrets.AGY_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} # rubric grader (claude) - HAWK_API_KEY: ${{ secrets.HAWK_API_KEY }} - HAWK_FORMAT: json - run: | - MODEL_ARGS=() - if [ "${{ matrix.model }}" != "default" ]; then MODEL_ARGS=(--model "${{ matrix.model }}"); fi - RUBRIC=""; if [ "${{ inputs.rubric }}" = "true" ]; then RUBRIC="--rubric"; fi - uv run evals --harness agy --skill ${{ matrix.skill }} "${MODEL_ARGS[@]}" $RUBRIC - # No continue-on-error: the CLI now decides job color (exit 1 only on a - # PLUMBING FAILURE — 0 prompts ran cleanly). agy can't auth in headless CI - # (OAuth-only, upstream #78), so it exits 1 here and the job goes RED, which - # is correct: red = "couldn't run, needs work." The digest still renders agy's - # 🚫 cell because Upload results + the report job are both `if: always()`. - - - name: Upload results - if: always() - uses: actions/upload-artifact@v7 - with: - name: eval-agy-${{ matrix.skill }}-${{ matrix.model }} - path: evals/harnesses/agy/results/${{ matrix.skill }}/ - retention-days: 30 - - # ── Cursor ──────────────────────────────────────────────────────────────── + # --- eval-agy DISABLED: agy is OAuth-only and cannot authenticate in headless CI. + # Re-enable by uncommenting the job below, re-adding `agy` to the platform options, + # and adding `eval-agy` back to the report job's `needs:` list. See git history. + +# eval-agy: +# name: agy / ${{ matrix.skill }} / ${{ matrix.model }} +# runs-on: ubuntu-latest +# needs: validate-config +# # agy is OAuth-only and cannot authenticate in headless CI (always red), so it is +# # EXCLUDED from `all` — run it explicitly with platform: agy only when needed. +# if: inputs.platform == 'agy' +# strategy: +# fail-fast: false +# matrix: +# skill: [hawkscan, api, stackhawk-data-seed, hawkscan-ci, optimize] +# model: [default] +# +# steps: +# - uses: actions/checkout@v7 +# - uses: astral-sh/setup-uv@v5 +# - name: Install Claude Code CLI (native, rubric grader) +# run: | +# curl -fsSL https://claude.ai/install.sh | bash +# echo "$HOME/.local/bin" >> "$GITHUB_PATH" +# +# - name: Install agy CLI +# run: | +# # /cli/install.sh is the real bootstrapper; /install-cli returns the +# # site's HTML landing page (piping that into bash is what broke before). +# curl -fsSL https://antigravity.google/cli/install.sh | bash +# echo "$HOME/.local/bin" >> "$GITHUB_PATH" # installer drops `agy` here +# continue-on-error: true # don't abort the job — evals records any launch failure +# +# - name: Verify agy CLI +# run: agy --version +# continue-on-error: true # if unavailable, the eval run captures it as a per-prompt error +# +# - name: Install StackHawk plugins +# env: +# # NOTE: agy ignores API-key env vars and forces interactive Google OAuth +# # (verified empirically; upstream antigravity-cli#78 open). This mapping is +# # kept for if/when key auth lands; headless CI needs the OAuth-token +# # injection below. +# ANTIGRAVITY_API_KEY: ${{ secrets.AGY_API_KEY }} +# run: | +# echo y | agy plugin install plugins/hawkscan +# echo y | agy plugin install plugins/api +# echo y | agy plugin install plugins/stackhawk-data-seed +# echo y | agy plugin install plugins/hawkscan-ci +# echo y | agy plugin install plugins/optimize +# continue-on-error: true # depends on agy CLI; best-effort so evals still runs +# +# # HawkScan 6 ships as a single self-contained native binary (no JDK) that +# # provides BOTH scanning (`hawk`) and the platform/API CLI (`hawk op …`). +# # Install it from the download manifest (source of truth for version + URL) +# # into /usr/local/bin (already on PATH). One install covers the hawkscan +# # skill's CLI path AND the api skill + hawkscan Step-1 dedup checks. +# - name: Install hawk CLI (combined binary, manifest-driven) +# run: | +# set -euo pipefail +# MANIFEST=https://download.stackhawk.com/hawkdocs/hawk.manifest.json +# url="$(curl -fsSL "$MANIFEST" | jq -r '.latest.assets[] +# | select(.asset.group == "linux-x64" and (.url | endswith("/hawk"))) | .url')" +# [ -n "$url" ] || { echo "ERROR: no linux-x64 hawk binary in manifest"; exit 1; } +# echo "Installing hawk from $url" +# curl -fsSL "$url" -o /tmp/hawk +# chmod +x /tmp/hawk +# sudo mv /tmp/hawk /usr/local/bin/hawk +# continue-on-error: true # absence is captured per-prompt in the eval traces +# - name: Verify hawk CLI +# run: | +# hawk version +# hawk op --help +# continue-on-error: true # absence is captured per-prompt in the eval traces +# +# - name: Run ${{ matrix.skill }} evals +# env: +# # agy ignores API-key env vars and forces OAuth (upstream #78); kept for +# # if/when key auth lands. Headless CI needs OAuth-token injection (TODO). +# ANTIGRAVITY_API_KEY: ${{ secrets.AGY_API_KEY }} +# ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} # rubric grader (claude) +# HAWK_API_KEY: ${{ secrets.HAWK_API_KEY }} +# HAWK_FORMAT: json +# run: | +# MODEL_ARGS=() +# if [ "${{ matrix.model }}" != "default" ]; then MODEL_ARGS=(--model "${{ matrix.model }}"); fi +# RUBRIC=""; if [ "${{ inputs.rubric }}" = "true" ]; then RUBRIC="--rubric"; fi +# uv run evals --harness agy --skill ${{ matrix.skill }} "${MODEL_ARGS[@]}" $RUBRIC +# # No continue-on-error: the CLI now decides job color (exit 1 only on a +# # PLUMBING FAILURE — 0 prompts ran cleanly). agy can't auth in headless CI +# # (OAuth-only, upstream #78), so it exits 1 here and the job goes RED, which +# # is correct: red = "couldn't run, needs work." The digest still renders agy's +# # 🚫 cell because Upload results + the report job are both `if: always()`. +# +# - name: Upload results +# if: always() +# uses: actions/upload-artifact@v7 +# with: +# name: eval-agy-${{ matrix.skill }}-${{ matrix.model }} +# path: evals/harnesses/agy/results/${{ matrix.skill }}/ +# retention-days: 30 +# +# # ── Cursor ──────────────────────────────────────────────────────────────── eval-cursor: name: cursor / ${{ matrix.skill }} / ${{ matrix.model }} runs-on: ubuntu-latest @@ -438,7 +442,7 @@ jobs: # ── PR comment ──────────────────────────────────────────────────────────── report: name: Eval report (run summary + PR comment) - needs: [validate-config, eval-claude-code, eval-codex, eval-agy, eval-cursor] + needs: [validate-config, eval-claude-code, eval-codex, eval-cursor] if: always() runs-on: ubuntu-latest permissions: diff --git a/VERSION b/VERSION index e3a4f193..9183195a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2.0 \ No newline at end of file +2.4.0 \ No newline at end of file diff --git a/cursor/.cursor/rules/stackhawk-api-hawk-op.mdc b/cursor/.cursor/rules/stackhawk-api-hawk-op.mdc index 725d76b4..96a89135 100644 --- a/cursor/.cursor/rules/stackhawk-api-hawk-op.mdc +++ b/cursor/.cursor/rules/stackhawk-api-hawk-op.mdc @@ -53,8 +53,7 @@ hawk op status # Confirm auth ``` `hawk init` writes credentials to `~/.hawk/hawk.properties`. -`hawk op` reads that file plus the `HAWK_API_KEY` env var. The legacy -standalone-binary config file is no longer used. +`hawk op` reads that file plus the `HAWK_API_KEY` env var. **CI/CD only:** For pipeline use, set `HAWK_API_KEY` directly as a secret. Org ID and output format can be set via env vars: @@ -109,8 +108,7 @@ extraction, no token handling. ### Latest scan for an app (overview + alerts) ```bash -hawk op scan get --app "" -hawk op scan get --app-id +hawk op scan get --app "" # --app takes a name or a UUID ``` Wraps: `/scan/{orgId}` + `/scan/{scanId}/alerts`. Returns a scan header, alert @@ -301,7 +299,7 @@ call — useful when building a reporting pipeline. - **Don't forget `--detail full` when you need the remediation/HTTP message payload.** The default `hawk op scan get` output is the overview — it won't include per-URI evidence. -- **Don't combine `--app` (name) and `--app-id` (UUID) in the same command.** - Pick one. `--app` resolves by name (case-insensitive exact match); `--app-id` is exact by UUID. - If a name matches **more than one** app, `hawk op` errors and tells you to use `--app-id` — - fall back to `--app-id ` (grab the UUID from `hawk op app list`). +- **`--app` takes a name or a UUID.** A UUID resolves exactly; a name resolves by + case-insensitive exact match. If a name matches **more than one** app, `hawk op` errors and + lists the matches — disambiguate by passing that app's UUID to `--app` (grab it from + `hawk op app list`). diff --git a/cursor/.cursor/rules/stackhawk-api.mdc b/cursor/.cursor/rules/stackhawk-api.mdc index 02a45825..6bbb128e 100644 --- a/cursor/.cursor/rules/stackhawk-api.mdc +++ b/cursor/.cursor/rules/stackhawk-api.mdc @@ -33,8 +33,7 @@ Before making any calls, check what's available: - valid org + JWT → proceed. - installed but unconfigured → `hawk init --browser` (interactive; the combined binary has no `hawk op init` — `hawk init` writes `~/.hawk/hawk.properties`, which - `hawk op` reads along with the `HAWK_API_KEY` env var; the legacy - standalone-binary config file is no longer used). + `hawk op` reads along with the `HAWK_API_KEY` env var). - not installed → instruct the user to install the `hawk` CLI (docs: [docs.stackhawk.com](https://docs.stackhawk.com)) and stop; do **not** fall back to curl. diff --git a/cursor/.cursor/rules/stackhawk-hawkscan-ci.mdc b/cursor/.cursor/rules/stackhawk-hawkscan-ci.mdc index 7882599a..52cf2fbe 100644 --- a/cursor/.cursor/rules/stackhawk-hawkscan-ci.mdc +++ b/cursor/.cursor/rules/stackhawk-hawkscan-ci.mdc @@ -1,6 +1,6 @@ --- description: > - Use when the user wants to configure HawkScan in their CI/CD pipeline — triggers on "set up hawkscan in CI", "add stackhawk to my pipeline", "scan in CI", "configure github actions / gitlab / jenkins / circleci for hawkscan", "wire hawkscan into ci/cd". Provider-agnostic: detects the CI system, edits the pipeline file in place to add a HawkScan job, prompts for HAWK_API_KEY storage (CI-native secrets store or external secrets manager), wires commit-SHA + branch traceability. Defers all local-scan concerns (stackhawk.yml, auth, findings, triage) to stackhawk-hawkscan. + Use when the user wants to WIRE HawkScan into a CI/CD pipeline config file — triggers on "set up hawkscan in CI", "add stackhawk to my pipeline", "scan in CI", "configure github actions / gitlab / jenkins / circleci for hawkscan", "wire hawkscan into ci/cd". Provider-agnostic: detects the CI system, edits the pipeline file in place to add a HawkScan job, prompts for HAWK_API_KEY storage (CI-native secrets store or external secrets manager), wires commit-SHA + branch traceability. Defers all local-scan concerns (stackhawk.yml, auth, findings, triage) to stackhawk-hawkscan. If no local stackhawk.yml exists yet, still trigger and route local-config work to stackhawk-hawkscan. Do NOT trigger for documentation-only changes, informational/research questions about CI, or running a local scan (that is stackhawk-hawkscan) — this skill only edits CI pipeline config. globs: - ".github/workflows/**" - ".gitlab-ci.yml" diff --git a/cursor/.cursor/rules/stackhawk-hawkscan-model.mdc b/cursor/.cursor/rules/stackhawk-hawkscan-model.mdc index 08ffbb09..0e1d5736 100644 --- a/cursor/.cursor/rules/stackhawk-hawkscan-model.mdc +++ b/cursor/.cursor/rules/stackhawk-hawkscan-model.mdc @@ -407,6 +407,6 @@ lookups. Relevant commands and their documentation: |-------------------------------|------------------------------------------------------------| | List apps | `hawk op app list --format json` | | List envs for an app | `hawk op env list --app --format json` | -| Get scan findings with triage | `hawk op scan get --app --detail full --format json` | +| Get scan findings with triage | `hawk op scan get --app --detail full --format json` | These reads require the combined `hawk` CLI; the `api` skill covers setup (`hawk init --browser` or the `HAWK_API_KEY` env var). diff --git a/cursor/.cursor/rules/stackhawk-hawkscan-openapi-specs.mdc b/cursor/.cursor/rules/stackhawk-hawkscan-openapi-specs.mdc new file mode 100644 index 00000000..1bc098e7 --- /dev/null +++ b/cursor/.cursor/rules/stackhawk-hawkscan-openapi-specs.mdc @@ -0,0 +1,184 @@ +--- +description: > + HawkScan OpenAPI spec accuracy reference: framework-generic procedure for getting an accurate OpenAPI spec into openApiConf for a REST surface — prefer a spec the running app serves, suggest the code/build change that makes the framework generate one, verify it resolves against the app (base/context path), and derive one by hand only as a last resort. Use whenever wiring or fixing openApiConf, or when a scan's spec paths return 404s. +globs: +alwaysApply: false +--- +# OpenAPI Spec Accuracy Reference + +A REST scan is only as good as the spec behind it. An **accurate** spec is one whose +operations match the app's real routes *and* resolve against the scanned host — same base +path, same context path. An inaccurate spec is worse than no spec: HawkScan faithfully +requests every path it declares, so wrong paths become a wall of 404s that bury real +coverage and generate recurring false-positive noise on every rescan. + +This reference is framework-generic. Use it whenever a REST surface needs a spec wired into +`openApiConf` — discovery (SKILL.md Step 1a / `scan-planning.md`) points here, and the +quality gate points here when it reports a `spec-not-wired` or coverage gap. + +## Contents +- [The one rule: an unverified spec is a hypothesis](#the-one-rule-an-unverified-spec-is-a-hypothesis) +- [Get an accurate spec (preference order)](#get-an-accurate-spec-preference-order) +- [Framework spec-generation matrix](#framework-spec-generation-matrix) +- [Suggesting the code/build change](#suggesting-the-codebuild-change) +- [Base path and context path](#base-path-and-context-path) +- [Mandatory: verify the spec resolves](#mandatory-verify-the-spec-resolves) +- [Deriving a spec by hand (no tooling)](#deriving-a-spec-by-hand-no-tooling) + +--- + +## The one rule: an unverified spec is a hypothesis + +A checked-in `openapi.yaml`, a spec you generated, and a spec the app serves are all +*claims* about the API until you prove they resolve against the running app. Never wire a +spec into `openApiConf` and scan without the verification in "Mandatory: verify the spec +resolves" below. A checked-in file is the least trustworthy source — it drifts from the code, +can be stale, hand-maintained, or (as seen in the wild) emptied by a commit — so treat its +presence as a starting hypothesis, not a done task. + +## Get an accurate spec (preference order) + +Prefer a spec the framework produces from the *running app* over any static artifact. Work +down this list; stop at the first that yields a spec that passes verification. + +1. **The running app already serves one.** Probe the common served endpoints once the app is + up (see the matrix for framework-specific paths): + ```bash + for p in /openapi.json /openapi.yaml /v3/api-docs /v3/api-docs.yaml \ + /swagger.json /swagger/v1/swagger.json /api-docs /q/openapi; do + curl -sf -o /dev/null -w "%{http_code} $p\n" "$APP_BASE_URL$p" + done + ``` + A 200 here is the best source — it reflects real routing, including base/context path. + Wire it live with `openApiConf.path: ` so the scan always reads the current + spec. +2. **The framework can generate/serve one with a small change, but it isn't enabled.** This is + the ideal outcome — see "Suggesting the code/build change". Add the tooling, rebuild, + then fetch as in step 1. +3. **Only a static spec is checked in.** Verify it's current (path count matches the route + grep from discovery; paths resolve — see verification). If it's stale, prefer regenerating + via step 1/2. If you must use it, wire `openApiConf.filePath: ` and verify. +4. **A spec is published *outside* the repo.** Mature projects often maintain an accurate spec + somewhere other than the app repo — check before giving up: a docs site (e.g. + `api-docs..org`, a ReDoc/Swagger-UI page), a dedicated `/api-docs` or + `-openapi` repo, a spec published as a release asset or package, or a link in the + README/docs. This is exactly Firefly III's situation — no spec in-repo, but a complete one + at `firefly-iii/api-docs`. Fetch it, pin the version to match the app, wire `filePath`, and + verify it resolves (an external spec can lag the running app). +5. **No spec anywhere and no tooling for this stack.** Derive one by hand from the route + inventory — see "Deriving a spec by hand". + +## Framework spec-generation matrix + +Examples, not an exhaustive list. The method generalizes: identify the framework from the +manifest, then determine its idiomatic OAS support. For anything not listed, **find out** — +`hawk config show`/docs won't help here, so search "`` openapi generation" or +check the framework's own docs before concluding none exists. + +| Stack | Idiomatic OAS tooling | Serves at | Notes | +|-------|----------------------|-----------|-------| +| FastAPI (Python) | built in | `/openapi.json` | Already served — no change needed; just probe. | +| NestJS (Node) | `@nestjs/swagger` | `/api-json` (configurable) | First-party. | +| Express / Fastify (Node) | `swagger-jsdoc` + `swagger-ui-express`; `@fastify/swagger` | configurable | JSDoc-annotation or plugin based. | +| Spring Boot (Java/Kotlin) | `springdoc-openapi-starter-webmvc-ui` | `/v3/api-docs` | Reflects `server.servlet.context-path` in its server URL. | +| Quarkus (Java) | `quarkus-smallrye-openapi` | `/q/openapi` | Often on by default in dev. | +| Django REST Framework | `drf-spectacular` | `/api/schema/` | `drf-yasg` is the older alternative. | +| Flask (Python) | `apiflask`, `flasgger`, `flask-smorest` | configurable | — | +| ASP.NET Core | `Swashbuckle.AspNetCore` or `NSwag` | `/swagger/v1/swagger.json` | Often scaffolded already. | +| Go | `swaggo/swag`, `danielgtaylor/huma` | configurable | swag generates from annotations. | +| Rails | `rswag` | configurable | Spec-from-request-specs. | +| Laravel (PHP) | `l5-swagger`, `scramble` | configurable | Scramble infers without annotations. | +| Rust | `utoipa`, `aide` | configurable | — | + +## Suggesting the code/build change + +When the framework can emit an accurate spec but doesn't yet (preference step 2), this is the +highest-value move: a served, framework-generated spec is accurate by construction and stays +current as the code changes. Explore and suggest a **concrete** change — not "consider adding +OpenAPI support": + +- Name the exact dependency and where it goes (the build file + coordinate/version). +- Show the minimal config/annotation to expose the endpoint, and state the served path. +- State the DAST value it unlocks ("wires N routes HawkScan otherwise can't see; replaces a + stale/hand-derived spec that would 404"). +- Scope it dev/test-only where the framework allows, so it needn't ship to production. + +Present this as a recommendation with the diff and let the user decide — the skill does not +apply unrequested code changes (see "Recommend code changes for gaps" in `scan-planning.md`). +Make the suggestion good enough that saying yes is a one-step change, then rebuild and fetch +the served spec. + +## Base path and context path + +The most common accuracy failure is a base-path mismatch. HawkScan builds each request as the +configured host **plus the spec's operation path**; a spec whose paths omit the app's base or +context path 404s on every operation. Detect the app's base path during discovery and make +the effective `host + spec-path` land on real routes: + +| Framework | Where the base/context path lives | +|-----------|-----------------------------------| +| Spring MVC | `server.servlet.context-path` (application.yml/properties) | +| Spring WebFlux | `spring.webflux.base-path` | +| Express | the mount prefix in `app.use('/api/v1', router)` | +| Django | the URL prefix in the root `urls.py` `include()` | +| ASP.NET | `UsePathBase(...)` / route prefix | +| Rails | `scope`/`namespace` in `config/routes.rb` | + +Two ways to make it resolve — pick one and be consistent: +- Put the base path in `app.host` (e.g. `http://localhost:5000/api/v1`) and keep spec paths, + `loginPath`, and `testPath` **relative**; or +- Keep `app.host` at the origin and ensure the spec's operation paths already include the base + path (a framework-served spec usually does). + +Do **not** split the difference — e.g. stripping the context path from `app.host` and +hardcoding it only into `loginPath`/`testPath`. That fixes auth but leaves every +`openApiConf` path unprefixed, which is exactly how a scan ends up 71% 404s while login still +works. If `hawk validate auth` misbehaves with a path component in `host`, resolve it without +desyncing the spec's base path — verify the outcome with the resolve check below. + +## Mandatory: verify the spec resolves + +Before the first real scan, prove the wired spec produces real requests — regardless of +source. This is the check that catches base-path mismatch, stale specs, and wrong hosts. + +1. **Count sanity.** Spec operation count should be in the same ballpark as the route grep + from discovery. A spec with far fewer/more paths than the code has is suspect. +2. **Resolve check.** Take 2–3 real operation paths **from the spec you're wiring** and + request each **as the scanner will build it** — effective `host` + operation path — against + the running app, both as the spec declares it and with the app's detected base/context path + prepended. Substitute your own values for the placeholders; do not run them literally: + ```bash + # = an operation path copied from the spec (e.g. /authors) + # = the app's detected base/context path, or empty if none (e.g. /api/v1) + curl -s -o /dev/null -w "%{http_code} %{url_effective}\n" "$SCAN_HOST" + curl -s -o /dev/null -w "%{http_code} %{url_effective}\n" "$SCAN_HOST" + ``` + If the spec-derived URL 404s while the base/context-path-prefixed variant returns a real + status (200/401/403), the base path is wrong — fix `app.host` or the spec before scanning. + A raw container 404 (e.g. Tomcat's default page) rather than an application response is a + strong tell that the request never reached the app's routing. +3. After the scan, confirm with `hawk op scan uris `: the scanned URIs should be the + spec's real routes, not un-prefixed shadows of them. + +## Deriving a spec by hand (no tooling) + +When the stack has no OAS generator (or adding one is out of scope), build a minimal spec from +the route inventory discovery already produced. Keep it small and correct rather than +complete: + +- One `paths` entry per real route; include path parameters as `{id}` style templates. +- Set `servers[].url` to the real base (including context path) — but remember HawkScan keys + off `host + operation path`, so still run the resolve check; don't rely on `servers` alone. +- Include request bodies/params only where they matter for reaching the endpoint. +- Wire via `openApiConf.filePath` and verify as above. + +A hand-derived spec is a maintenance liability — always prefer suggesting the framework +tooling (step 2) so the spec regenerates from code instead of drifting. + +**A hand-derived spec still beats `hawk.spider.seedPaths`.** seedPaths only lists URLs to +visit — no methods, no request bodies, no path/query parameters — so a seedPaths-only scan +can GET a handful of collection endpoints but never exercises POST/PUT/PATCH, parameterized +routes, or request-body validation. That is why a spec-less scan collapses to a fraction of +the real surface — on a typical REST API that can be well under 10% of the routes reached. +Only use seedPaths as a supplement to a spec, or as a genuine last resort when even a +hand-derived spec is impossible. diff --git a/cursor/.cursor/rules/stackhawk-hawkscan-scan-planning.mdc b/cursor/.cursor/rules/stackhawk-hawkscan-scan-planning.mdc new file mode 100644 index 00000000..1e37c612 --- /dev/null +++ b/cursor/.cursor/rules/stackhawk-hawkscan-scan-planning.mdc @@ -0,0 +1,217 @@ +--- +description: > + HawkScan scan-planning (discovery) reference: code-first discovery of an app's API surfaces before writing or editing stackhawk.yml — reading repo docs first, per-framework route inventory, recommending code changes for structural gaps, asking the user instead of guessing, and producing a per-surface stackhawk.yml config. Use for the first scan of a repo, a quality-gate structural gap, or a user-requested re-plan. +globs: +alwaysApply: false +--- +# Scan Planning Reference (Discovery) + +Discovery is how the hawkscan skill decides *what* to scan and *how* before it writes or +edits a single `stackhawk.yml`. Run it in three situations: + +- **First scan of a repo.** No `stackhawk.yml` exists yet (SKILL.md Step 1a / Step 2a) — + discovery produces the surface inventory that Step 2a's config generation consumes. +- **The quality gate looped back with a structural gap.** A post-scan coverage check found + routes the scan never reached, or a surface it didn't know existed. Re-run discovery + against that specific gap rather than starting over. +- **The user asks to re-plan the scan.** New API surface added, a monorepo grew a service, + an old assumption no longer holds. + +Discovery is stateless: there is no plan file. `stackhawk.yml` is the only durable artifact, +and anything the user tells you that isn't a config field belongs as a comment in +`stackhawk.yml` or in the repo's `CLAUDE.md`/`AGENTS.md` — see "Ask, don't guess" below. + +Work code-first, interactively: explore before asking, ask the user before guessing, and +never stall waiting for information you could get by reading three more files or asking one +direct question. + +## Contents +- [Discover the app's API surfaces](#discover-the-apps-api-surfaces) +- [Recommend code changes for gaps](#recommend-code-changes-for-gaps) +- [Ask, don't guess](#ask-dont-guess) +- [Configure per surface](#configure-per-surface) +- [Cross-checking](#cross-checking) + +--- + +## Discover the app's API surfaces + +### Pass 1 — read what the repo already says + +Most repos document themselves. Read these, in priority order, before exploring code — +treat what they say as authoritative and don't rediscover by grepping what a doc already +states: + +| Source | Typically documents | +|--------|----------------------| +| `AGENTS.md` | run/build/test commands, layout, conventions | +| `CLAUDE.md` | same, written for agents — often the richest source | +| `GEMINI.md`, `.github/copilot-instructions.md` | agent run/build guidance | +| `.cursor/rules/*` | project conventions and setup steps | +| `README*` | quickstart, run command, default host/port | +| `CONTRIBUTING*` | local dev setup, how to run services and tests | +| `docs/` setup / quickstart / architecture pages | deeper API and service-boundary detail | + +Harvest: run command + host/port, which API surfaces exist and where their source lives +(useful in a monorepo), and any documented spec location or dev/test credential. Read these +from the docs and config — discovery never starts the app to learn them. A +`docker-compose.yml` port mapping or an `.env`/`.env.example` `APP_URL` gives you the host +and port without running anything; starting the app is a scan-time step, not a discovery one. + +### Pass 2 — explore code per surface type + +For anything the docs didn't answer, find the surfaces the way a developer would: read +manifests, entry points, and route definitions. Identify **every** distinct API surface in +the repo — a service can have more than one (e.g. a REST API plus an internal gRPC service). +For each surface found, record its type and derive a route inventory using the matching +technique. The commands below are **illustrative starting points**, not fixed incantations: +point them at the repo's real source root and its actual spec/schema filenames — don't run +`src/`, `openapi.yaml`, or `schema.graphql` literally — and adjust the pattern to what the +code actually uses. + +| Surface | Detection | Route inventory / derivation | +|---------|-----------|-------------------------------| +| REST — Spring | `@RestController`, `@Controller` classes | `grep -rEo '@(Get\|Post\|Put\|Delete\|Patch\|Request)Mapping' src/` — count matches | +| REST — Express/Node | `app.get/post/put/delete`, `router.get/post/...` | `grep -rEo "\.(get\|post\|put\|delete\|patch)\(" src/` (or route files under `routes/`) | +| REST — Rails | `config/routes.rb` | count route-verb lines: `grep -cE '^\s*(get\|post\|put\|patch\|delete)\s' config/routes.rb` | +| REST — Django | `urls.py` | count `path(`/`re_path(` entries: `grep -rc 'path(' **/urls.py` | +| REST — Go (gin/mux/chi) | `router.GET(`, `r.HandleFunc(`, `mux.NewRouter()` | `grep -rEo '\.(GET\|POST\|PUT\|DELETE\|PATCH)\(' .` | +| REST — .NET | `[HttpGet]`, `[HttpPost]`, `[Route(...)]` attributes | `grep -rEo '\[Http(Get\|Post\|Put\|Delete\|Patch)\]' .` | +| REST — OpenAPI spec | checked-in `openapi.{json,yaml,yml}`, `swagger*.{json,yaml}`; or served at `/v3/api-docs`, `/swagger.json`, `/openapi.json` once the app is running | count `paths:` entries: `yq '.paths | keys | length' openapi.yaml` (or the JSON equivalent). **A found spec is a hypothesis, not a win — see `openapi-specs.md` for getting an *accurate* one.** | +| GraphQL | schema files (`*.graphql`, `*.gql`); server libs in manifests (`graphql`, `apollo-server`, `graphql-yoga`, `graphene`, etc.); a `/graphql` endpoint | count root fields: `grep -A50 'type Query' schema.graphql \| grep -cE '^\s+\w+'` (repeat for `type Mutation`) | +| gRPC | `.proto` files; `grpc`/`protobuf` deps in the manifest | count `rpc` methods per service: `grep -c '^\s*rpc ' *.proto` | +| JSON-RPC / SOAP | fixed-path JSON-RPC dispatcher; WSDL files (`*.wsdl`), SOAP endpoint patterns | count `/api-docs` repo, a + release asset) — mature OSS projects often do. The full framework-generic procedure — probe + for a served spec, the per-stack tooling matrix, how to write a concrete suggestion, the + external-spec check, and the base-path/resolve verification — is in `openapi-specs.md`. Do + that rather than wiring a stale checked-in file, or falling back to `seedPaths`, and hoping. +- **No test credential exists.** Recommend a test-only token issuer or seed script (a + dev-only endpoint or fixture that mints a valid session/JWT) gated behind an environment + check so it never ships to production. +- **GraphQL introspection disabled in the target environment.** Recommend enabling + introspection for the scan environment only, or checking in the schema file if it isn't + already. + +These are recommendations only. Never apply an unrequested code change to close a coverage +gap — present the recommendation, explain the DAST value it unlocks, and let the user decide +whether to make the change. + +## Ask, don't guess + +Code exploration sometimes can't account for the whole URL space: + +- A **monorepo slice** — the checked-out repo is one service among several, and its API is + fronted by a gateway or composed with other services not present locally. +- A **gateway fronting upstreams** — the visible code is a thin proxy; the real route + surface lives in services this repo doesn't contain. +- **Dynamic routing** — routes built from a database, a plugin registry, or a config file + not checked into source control. + +When code can't answer a surface question, ask the user directly — for the missing service's +repo, the gateway's route table, or the runtime detail that would resolve it. Don't guess a +plausible-looking answer and don't stall the whole discovery pass waiting on one open +question if other surfaces are ready to configure. + +Once the user answers, that answer has nowhere else to live — there is no plan file to record +it in. Persist it as a comment next to the relevant `stackhawk.yml` block (e.g. `# gateway +also fronts the billing-service repo at github.com/org/billing — see route table in +#eng-platform`) or as a line in the repo's `CLAUDE.md`/`AGENTS.md` if it's a fact about the +repo rather than about one scan. Either way, the goal is the same: the next agent to run +discovery on this repo should never have to ask the same question twice. + +## Configure per surface + +Write **one `stackhawk.yml` config per API surface** (REST, GraphQL, gRPC, SOAP, SPA, ...). +Order them by DAST value, highest first: the primary or backing API surface goes first, a +SPA frontend goes last (per the SPA guidance above, it typically finds only header/CSP +issues). This order is not cosmetic — state it explicitly in the summary you show the user, +so the rationale for what scans first is visible, not implicit. + +**Naming and platform mapping for multi-surface repos:** a single-surface repo uses plain +`stackhawk.yml`. With multiple surfaces, name each config `stackhawk-.yml` (e.g. +`stackhawk-rest.yml`, `stackhawk-graphql.yml`) and run each scan as +`hawk scan stackhawk-.yml`. All surfaces share one `applicationId`; give each +surface its own environment (e.g. `dev-rest`, `dev-graphql`). Findings compare scan-to-scan +within an environment, so per-surface envs keep each surface's history coherent instead of +interleaving a REST scan's baseline with a GraphQL scan's. + +For each surface: + +1. Wire the spec you found or confirmed in discovery: `openApiConf` for REST, + `graphqlConf` for GraphQL, `grpcConf` + the `.proto` for gRPC, `soapConf` + the WSDL for + SOAP. Use `hawk config show --text` for the exact field shape — don't + hand-write the block from memory. For REST, the spec must be *verified-accurate* before + it's wired: get it and prove it resolves per `openapi-specs.md` (a wrong spec silently + 404s every operation), and make sure `app.host` and the spec agree on the base/context + path. +2. Configure auth per Phase 1c in SKILL.md — one auth block per surface if surfaces have + different login mechanisms (common when a gateway and an upstream service use different + credential types). +3. Validate every surface's config before treating discovery as complete: + ```bash + hawk validate config + hawk validate api # when a spec is wired + hawk validate auth # when authentication: is present + ``` + +**Before the first scan of a fresh repo, present a one-screen summary and get the user's +confirmation.** The summary should state, per surface: the surface type, which config file +covers it, the scan order and why, the auth approach, the expected route count with how it +was derived (the same grep/spec-count evidence gathered above), and — for any REST surface +with a wired spec — the result of the mandatory resolve-check (e.g. "3/3 sample spec paths +resolved against host"; see `openapi-specs.md`). The resolve-check is the one thing that +catches a base-path mismatch before it wastes a whole scan, so its result belongs in the +summary rather than left to chance. Confirming this up front is cheap; discovering the scan +order, auth assumption, or a spec that 404s every path was wrong after a scan has already +run is not. + +## Cross-checking + +Don't configure `openApiConf`, `graphqlConf`, `grpcConf`, `soapConf`, or any other +spec-wiring block from memory alone — these shapes change across hawk releases. Two sources, +in order of preference: + +1. **`hawk config show
--text`** — the canonical, version-matched recipe for the + installed CLI. This is always correct for the hawk version actually in use and should be + the default source for field syntax. +2. **docs.stackhawk.com**, via WebFetch or WebSearch when available — useful for narrative + guidance `hawk config show` doesn't carry (why a shape works the way it does, worked + examples for less common combinations). Prefer it over recalling the shape from training + data whenever there's any doubt. + +If the two disagree, trust `hawk config show` — it reflects the CLI version actually +installed, not a documentation snapshot that may be ahead of or behind it. diff --git a/cursor/.cursor/rules/stackhawk-hawkscan-scan-quality.mdc b/cursor/.cursor/rules/stackhawk-hawkscan-scan-quality.mdc new file mode 100644 index 00000000..6d74d2a3 --- /dev/null +++ b/cursor/.cursor/rules/stackhawk-hawkscan-scan-quality.mdc @@ -0,0 +1,192 @@ +--- +description: > + HawkScan scan-quality (post-scan quality gate) reference: run after every scan and rescan, before findings become fix tasks. Derives a fresh coverage expectation each run (spec-wired or route-inventory), the four coverage checks, and a bounded, additive-only stackhawk.yml config iteration to close gaps — never blocks a real finding from being reported and fixed. +globs: +alwaysApply: false +--- +# Scan Quality Reference + +Run this gate after **every** scan — full scans and rescans alike — before findings are +parsed into fix tasks (SKILL.md calls it Step 4.5, sitting between Step 3's scan run and +Step 4's fix-task generation). It is a feedback loop into config tuning, **not a governor**: +gate state never blocks a finding from being reported and fixed. A thin scan that only +reached a fraction of the surface can still have found something real, and that finding +gets fixed now, regardless of what the gate says. The gate's job is narrower: notice when a +scan almost certainly missed API surface, and drive a bounded, additive config fix so the +*next* scan reaches more of it. + +The gate derives its expectation fresh every time it runs. There is no persisted plan or +prior-run state to compare against — recompute the expected surface from the repo and the +effective config each time, the same way discovery would if run again right now. + +**Multi-config execution.** When a repo has more than one `stackhawk.yml` surface (discovery +writes one config per surface, ordered by DAST value — primary or backing API first, a SPA +frontend last), scan and gate **sequentially**, one config at a time, in that same +value order. Gate immediately after each config's scan rather than batching every scan +first and gating afterward — an auth-wall caught and fixed on config 1 saves the identical +wasted scan on config 2. The iteration caps described below (2 interactive / 1 autonomous) +apply **per config**, not to the run as a whole. The autonomous post-code-change loop scans +only the config(s) whose surface the change touched — that scan must still cover that +surface's entire scope (the fix-ALL-findings rule from Step 4 is unchanged) — defaulting to +the primary surface's config when it's unclear which surface a change touched. A full sweep +across every config is an interactive operation only; never trigger one autonomously. + +## Contents +- [Derive the expectation (fresh, every scan)](#derive-the-expectation-fresh-every-scan) +- [The five checks](#the-five-checks) +- [On gaps: iterate the config](#on-gaps-iterate-the-config) +- [Reporting rules](#reporting-rules) +- [Degradation](#degradation) + +--- + +## Derive the expectation (fresh, every scan) + +Before you can tell whether a scan under-covered its target, you need something to compare +it against. Build that expectation the same way every time, right after the scan finishes: + +- **Spec wired** (`openApiConf`, `graphqlConf`, `grpcConf`, `soapConf`): the expectation is + the paths or operations the spec declares. `hawk op scan uris --format json` is + config-aware — it reads the effective scan config and automatically fetches + graphql-operations or jsonrpc-methods when `graphqlConf` or `jsonRpcConf` is present in + that config (force either with `--graphql` / `--jsonrpc` if auto-detection misses it). + That gives you the real scanned-URI set to diff against the spec's declared set. +- **No spec wired**: re-derive the expected route inventory using the same per-framework + route greps as the discovery reference (`scan-planning.md`) — the same detection + heuristics, the same counting technique, run fresh against the current code, not carried + over from whatever discovery counted last time. Code changes between scans; the + expectation should too. +- **Spec-less apps that keep coming up short**: if the grepped expectation and the scanned + URIs repeatedly diverge because there's nothing authoritative to wire, that's a signal to + recommend generating a spec — the same code-change recommendations discovery makes + (`scan-planning.md`'s "Recommend code changes for gaps" section: `springdoc-openapi` for + Spring, `swagger-jsdoc`/`@nestjs/swagger` for Node, `drf-yasg`/`drf-spectacular` for + Django REST Framework, `swashbuckle` for .NET). Surface the recommendation; don't apply it + unrequested. +- `hawk op scan uris` and `hawk op scan config` ship with the combined `hawk` CLI the + preflight already requires. If either subcommand errors or isn't recognized on the + installed CLI, fall back per the Degradation section below — detect it by the command + failing, don't ask the user which version they have. + +Because the no-spec expectation is re-derived with the **same** per-framework greps discovery +uses, the gate inherits discovery's blind spots: a surface neither discovery nor this +re-derivation detects (a second gRPC service, a dynamically-mounted router) won't be flagged +as unscanned — the gate can't miss-diff against a surface it never counted. This is why the +user-confirmed discovery summary (`scan-planning.md`) is the real backstop for surface +completeness, not the gate alone. + +## The five checks + +Run all five after every scan. Each has its own command, pass condition, and a stable +reason identifier used when reporting a gap — this gate's own reporting (Step 4.5 in +SKILL.md) uses these identifiers directly, so keep them exactly as written here across any +edits to this file: `coverage-gap`, `spec-not-wired`, `surface-unscanned`, +`auth-validate-failed`, `auth-wall`, `all-4xx`, `base-path-mismatch`, `env-unreachable`. + +| # | Check | What it measures | Command(s) | Pass condition | Reason identifier(s) | +|---|-------|-------------------|------------|-----------------|------------------------| +| 1 | Coverage | Scanned URIs vs. the expectation derived above | `hawk op scan uris --format json` diffed against the expectation list | **Never fails the gate.** Always computed and reported as evidence, e.g. "14 of 41 planned routes untouched," with the untouched routes listed | `coverage-gap` (evidence only) | +| 1b | Base-path resolve | Whether spec-derived paths actually resolve against the app, or systematically 404 because the config and spec disagree on the base/context path | In the scanned-URI list, look for the same route appearing both prefixed and un-prefixed (e.g. `/api/v1/authors` **and** bare `/authors`), or a cluster of spec paths all 404 while a prefixed variant succeeds; confirm with a curl of `host + spec-path` vs the prefixed variant | No systematic base-path 404 pattern — spec paths resolve to real routes. This **does** fail the gate (structural). Fix per `openapi-specs.md` "Base path and context path" | `base-path-mismatch` | +| 2 | Auth | Live auth validation before the scan; auth-wall signals during it | `hawk validate auth .yml` pre-scan (against the config being gated — repos with multiple per-surface configs validate each one); `hawk op scan metrics ` for auth-wall flags | `validate auth` exited 0, and no auth-wall flag on any surface that has `authentication:` configured | `auth-validate-failed`, `auth-wall` | +| 3 | Surface-completeness | Every surface discovery found has a config, that config actually ran, and any spec found for it is wired into that config | `hawk op scan config ` (effective config the scan ran with); fall back to reading the local `.yml` files | Every discovered surface maps to a config that exists, ran this scan, and has its spec wired if one exists | `surface-unscanned`, `spec-not-wired` | +| 4 | Health | Connection-failure spikes, timeout streaks, app unreachable mid-scan; all-4xx response patterns on a configured surface | `hawk op scan metrics ` for flags; `hawk op scan get ` for `url_count` and status summary | No connection-failure spike, no timeout streak, no mid-scan unreachability, and no configured surface returning all 4xx | `env-unreachable`, `all-4xx` | + +**Signal classes.** Every non-evidence gap above is either config-class or environment-class +— the class decides what you're allowed to do about it: + +- **Config-class** — `spec-not-wired`, `surface-unscanned`, `auth-validate-failed` (the fix + lives in the config's `authentication:` block), `auth-wall` while the app is demonstrably + up, `all-4xx` on a configured surface (this is usually auth-shaped: the surface is + reachable but every request is being rejected), and `base-path-mismatch` (the fix is + aligning `app.host` and the spec's base/context path — `openapi-specs.md`). These route + into the normal bounded config iteration described below. +- **Environment-class** — `env-unreachable`, covering connection-failure spikes, timeout + streaks, and the app going unreachable mid-scan. **Never touch the config for these.** + Verify the app is up using the same playbook as SKILL.md's Step 1c / Step 6 exit-1 + diagnosis, restart or wait if needed, then retry the scan once — that retry does not + consume a config-iteration. If the same environment-class signal recurs on the retry, + report it as an environment problem and stop; do not keep retrying and do not reach for a + config edit to work around it. + +Performance signals — slow paths, rate-limited endpoints, a scan that ran long — are not a +gate concern at all. They route to the `optimize` skill's existing suggestion path (SKILL.md +Step 4, surfaced once per session for full scans ≥ 20 minutes), never to this gate. + +## On gaps: iterate the config + +When one or more config-class gaps are open, loop back to SKILL.md's Step 2b (tune existing +`stackhawk.yml`) with this shape: + +- **Batch, then rescan once.** Collect every gate-named fix from this scan's gap set — wire + the missing spec, enable `hawk.spider.ajax`, fix the `authentication:` block, add a config + for an unscanned surface — into a single edit pass. Rescan once after applying all of + them together. Don't rescan after each individual fix; that wastes scans the gate exists + to save. +- **Only iterate when the config actually changed.** If a reported gap has no config-level + fix — a monorepo slice with no visibility into the missing service, dynamic routing that + needs a code change, a spec that genuinely doesn't exist yet — report it and move on. + Don't burn an iteration re-scanning a config you didn't touch. +- **Iteration cap: 2 interactive, 1 autonomous — per config, and counted in rescans, not + edits.** A batch of five config edits followed by one rescan is one iteration. This + matches the Autonomous Security Loop's existing guard rail of at most one fix-rescan cycle + when running unattended. +- **At the cap, stop.** Report the remaining structural gaps by reason identifier, the + coverage route-diff evidence (the untouched-route list from check 1), and concrete next + steps — including any code-change recommendations discovery would make for surfaces that + can't be closed by config alone. Never claim the scan is done, or the app is secure, while + gaps remain open at the cap. +- **A severe coverage gap with all four structural checks green** is not a config-iteration + target — it means the scan reached everything it was told to reach and still found little. + Route it to SKILL.md's Step 6 empty-data handling instead: suggest the + `stackhawk-data-seed` skill rather than continuing to edit the scan config. + +**Additive-only.** Every fix applied during iteration must add coverage: wire a spec, turn +on the Ajax Spider, correct an auth recipe, add a missing surface's config. Iteration must +**never** widen `excludePaths`, raise `failureThreshold`, add `excludePlugins`, or otherwise +weaken the scan so it happens to pass the gate. If the only way to close a gap is to reduce +what gets scanned, that is not a pass — report the honest gap instead of manufacturing a +clean one. + +Environment-class gaps never enter this loop at all — see the Signal classes note above: +verify the app, retry once for free, and stop if it recurs. + +## Reporting rules + +- **Findings are always reported and always fixable**, independent of gate state. The gate + never withholds, delays, or downgrades a finding — a scan that under-covered the surface + can still have found a real, exploitable bug, and that bug gets fixed on this pass. +- **Never say the scan is "done and secure"** (or any equivalent framing) while a + config-class or environment-class gap is still open. Say plainly what was scanned, what + wasn't, and why — the reason identifiers exist so this statement is specific, not vague. +- **The autonomous post-code-change loop reports gate gaps rather than treating exit 0 as + success.** It runs this gate after its scan, and if gaps remain once the autonomous + iteration cap (1) is reached, it reports them explicitly — reason identifiers, coverage + route-diff evidence, and next steps — instead of accepting a clean exit code at face value. +- **Performance-class signals are never gate gaps.** If health metrics point at slowness + rather than unreachability or all-4xx, that's an `optimize` suggestion, not a quality-gate + reason identifier. +- Keep every reported gap tied to one of the eight reason identifiers above so this gate's + own reporting stays specific rather than parsing prose — the identifiers must stay stable + across edits to this file. + +## Degradation + +`hawk op scan uris` and `hawk op scan config` ship with the combined `hawk` CLI. If either +subcommand errors or isn't recognized on the installed CLI, degrade silently and keep going — +detect it by the command failing, don't ask the user to check their version: + +- **Coverage (check 1)** degrades from a full scanned-vs-expected route diff to a single + before/after number: `url_count` from `hawk op scan get `. You lose the + untouched-route list; still report the number as evidence, just note it's coarser. +- **Surface-completeness (check 3)** degrades from reading the effective scan config via + `hawk op scan config` to local signals: read the per-surface `.yml` files directly to + confirm a surface's config exists and its spec block is present, and use the + `hawk validate config` output for each file to confirm the config is structurally sound. +- **Auth and health (checks 2 and 4)** are unaffected — `hawk op scan metrics` and + `hawk validate auth` remain the primary signal either way. + +`hawk op scan stats` does not exist yet (it's a follow-up ticket) but will eventually add +measured, per-path engine statistics rather than the grep- and `url_count`-based estimates +this reference relies on today. When it ships, prefer its measured signal over the inferred +approach described here. Until then, its absence is expected — degrade silently and don't +block or warn on it being missing. diff --git a/cursor/.cursor/rules/stackhawk-hawkscan.mdc b/cursor/.cursor/rules/stackhawk-hawkscan.mdc index 5f7df8f5..7ea262ca 100644 --- a/cursor/.cursor/rules/stackhawk-hawkscan.mdc +++ b/cursor/.cursor/rules/stackhawk-hawkscan.mdc @@ -23,12 +23,20 @@ This skill requires **hawk v6.0.0 or newer**. Verify: ```bash hawk version hawk config --help >/dev/null 2>&1 || echo "MISSING: hawk config — upgrade hawk to v6.0.0+" +hawk skills status ``` If hawk is older than `6.0.0` or `hawk config --help` fails, stop and tell the user to upgrade before proceeding. Do not inline auth recipes from memory — they live in `hawk config show` and are stale by design when hardcoded. +**`hawk skills status`** reports whether the installed StackHawk agent skills are current. +If it lists anything out of date, surface the exact upgrade command it prints (e.g. +`/plugin marketplace update stackhawk && /plugin update hawkscan`) and recommend the user +update before continuing — a stale skill may be missing fixes for the very issues you'll hit. +It's a recommendation, not a hard stop: proceed if the user prefers. (If the subcommand isn't +recognized, the installed hawk predates it — skip this check.) + See `references/installation.md` for upgrade and install instructions. --- @@ -40,19 +48,23 @@ The `api` skill wraps read-only StackHawk platform lookups via the `hawk` CLI (` | Purpose | Command | |-------------------------------|----------------------------------------------------------------------------------| | Check if App exists | `hawk op app list --format json` | -| Check if Env exists | `hawk op env list --app --format json` | -| Get findings with triage | `hawk op scan get --app --detail full --format json` | +| Check if Env exists | `hawk op env list --app --format json` | +| Get findings with triage | `hawk op scan get --app --detail full --format json` | | List ASM repos | `hawk op repo list --format json` | | Link app to ASM repo | `hawk op repo link --repo-id --app-id ` | -| Get tech flags | `hawk op app tech-flags get --app --format json` | -| Disable all tech flags | `hawk op app tech-flags disable-all --app --yes` | -| Set specific tech flags | `hawk op app tech-flags set --app Key=true` | +| Get tech flags | `hawk op app tech-flags get --app --format json` | +| Disable all tech flags | `hawk op app tech-flags disable-all --app --yes` | +| Set specific tech flags | `hawk op app tech-flags set --app Key=true` | | Triage a finding | `hawk op scan triage --scan --hash --status false-positive --note ""` | | Bulk triage from file | `hawk op scan triage --scan --from-file triage.yaml` | | Annotate w/o triage perm | `hawk op finding note --scan --hash --note "..."` | +| Get scanned URIs | `hawk op scan uris --format json` (skip if the subcommand errors) | +| Get effective scan config | `hawk op scan config ` (skip if the subcommand errors) | These reads require the combined `hawk` CLI; the `api` skill covers setup (`hawk init --browser` or the `HAWK_API_KEY` env var). +**Selecting an app:** `--app` takes an application **name or UUID** — pass whichever you have (e.g. the `applicationId` from `stackhawk.yml`, or the app's name). No separate flag is needed. Also parse `--format json` output defensively — a "skills out of date" banner may precede the JSON. + The `stackhawk-data-seed` skill sets up checked-in backend seed data via `hawk perch seed`. Hand off to it when authentication fails because the backend has no valid credential (**Phase 1c.6**), or after a scan when auth succeeded but endpoints returned empty data @@ -76,6 +88,20 @@ names deliberately. Findings have a lifecycle (NEW, FALSE_POSITIVE, RISK_ACCEPTE --- +## Progress output + +Keep the terminal informed so a running loop never looks silent. Prefix every status line with +`StackHawk | ` — **ASCII only, no emoji** (must render in Cursor, Windows `cmd`, and PowerShell; +keep the branded lines themselves plain ASCII). Emit one short line **entering each phase** — +discovery, config, scan, quality gate, findings, each fix batch, rescan, report — with what's +happening plus the key number (routes, coverage, findings). Before a scan/rescan, note it takes +a few minutes and let hawk's own progress stream through (don't suppress it); announce +completion after. Status output, **not** a prompt — never pause for input. E.g. +`StackHawk | Scanning http://localhost:8080 - a few minutes; progress streams below`, then +`StackHawk | 3 findings (2 High, 1 Medium) - fixing all`. Per-phase lines: `references/autonomous-loop.md`. + +--- + ## Phase 0: App Setup & Verification Run Phase 0 **once** when onboarding a new application (`stackhawk.yml` being created for @@ -116,32 +142,28 @@ enable only detected flags via `hawk op app tech-flags`; if none, skip. > 2. `CI=true` or `GITHUB_ACTIONS=true` → `CI` > 3. Git branch `main`/`master`/`production` → `Production`; `staging` → `Staging`; otherwise → `Development` -### Step 1a: Understand the Application +### Step 1a: Discover What to Scan -Before generating config, build a working understanding of the target: **how it runs -locally, how to reach its API, whether it's a SPA, and whether exercising it requires -auth.** Work in three passes; stop as soon as you can answer all four: +Before generating config, run code-first discovery: find every API surface in the repo +(REST, GraphQL, gRPC, SPA, ...), derive a route inventory per surface, and get the run +command, host/port, and auth shape needed to reach each one. Prefer the repo's own docs +over guessing, explore code to fill gaps, and ask the user directly for whatever remains +unresolved — never stall or invent. -1. **Read what the repo already documents.** Most repos describe how to run themselves — - prefer their own words over guessing, and treat them as authoritative. - → Source files and what to harvest: [`references/app-discovery.md`](references/app-discovery.md) +**Discovery is read-only and static.** Determine the run command, host, and port by +*reading* config and docs — `docker-compose.yml` port mappings, `.env`/`.env.example` +(e.g. `APP_URL`), the README — not by launching anything. You never need to start the app, +run a container, or bring up a server to discover *what* to scan; standing up the target is +a scan-time step (Step 1c), not part of discovery. -2. **Explore to fill the gaps.** For anything the docs don't answer, investigate the repo - the way a developer would when handed an unfamiliar codebase — read the run/build - manifests, entry points, and route definitions — until you can state, concretely: - - **Run command + host/port** — how to start it locally and where it listens. - - **API style + base path** — REST (note an OpenAPI/Swagger spec if one is served or - checked in), GraphQL, gRPC, or a plain HTML web app. - - **SPA?** — a client-rendered JS front end changes what we scan. - - **Auth?** — does reaching the real endpoints require a login, and what shape does it - appear to use? This is an *understanding* signal only; picking, obtaining, and - configuring the credential is Step 1c / Phase 1c. +→ Full discovery workflow — per-surface detection, route-inventory derivation, gap + recommendations, and the user-confirmed summary required before the first scan: + [`references/scan-planning.md`](references/scan-planning.md) +→ Docs-first source table (which files to read and what to harvest): + [`references/app-discovery.md`](references/app-discovery.md) -3. **Confirm the unknowns with the user.** Never stall or invent. Ask directly for - anything still unresolved — most often the start command, host/port, or the login. When - file access is limited, skip straight to asking for all four. - -Record the four answers — Step 1c and Step 2 consume them. +Record what discovery produced — surfaces, run command, host/port, auth shape — Step 1c and +Step 2 consume them. **SPA rule:** if the app is a client-rendered JS front end, never scan it without the Ajax Spider, and note that a separate backend API is usually the higher-value target. Full @@ -163,7 +185,8 @@ strategy, frontend-vs-backend scenarios, and config templates: ```bash hawk create app --name "" --env ``` - Resolve `` with the Env Name Algorithm above. Announce the created app ID and URL. + Resolve `` with the Env Name Algorithm above. Announce it (Progress output): + `StackHawk | Created app () - `. 6. **Env exists?** Determine env name via Env Name Algorithm. Run `hawk op env list --app --format json`. Reuse if exists; otherwise run `hawk op env create --app --env --host `. @@ -199,6 +222,16 @@ Do not proceed to Step 3 until validation passes. → API-type-specific config (OpenAPI, GraphQL, gRPC, seed paths, spider tuning): [`references/config-patterns.md`](references/config-patterns.md) +**REST surface? Get an accurate OpenAPI spec BEFORE the first scan — this is not optional.** +A REST scan with no spec (spider/`seedPaths` only) reaches a small fraction of the API; a +scan with a *wrong* spec 404s every path. Do not skip to a scan on the assumption a spec is +missing or "good enough" — **open and follow [`references/openapi-specs.md`](references/openapi-specs.md)** to +work its preference order (a spec the running app serves → a code/build change that generates +one → a spec published outside the repo → hand-derived), then run its resolve-check +(`host + spec-path` returns real routes, not 404s) before scanning. Reaching for +`hawk.spider.seedPaths` instead of a spec is a last resort, not a shortcut — read that file +first. Same idea for GraphQL/gRPC: wire the schema/proto, don't scan blind. + ### Phase 1c: Authentication Configuration Use `hawk config show` to fetch the canonical recipe for the app's auth pattern. @@ -260,7 +293,9 @@ After seeding, re-run `hawk validate auth stackhawk.yml` and continue. Review the config against the current app state: -- **Low path count?** Check in order: SPA/JS app → enable `hawk.spider.ajax: true`; API spec available → wire `openApiConf`/`graphqlConf`; specific known-deep paths → add `seedPaths`. Omit `seedPaths` unless there is a specific reason — they're rarely needed when Ajax Spider or a spec is configured. +- **Gate reported gaps?** Tune what the gate names (wire spec, enable ajax spider, fix + auth) — additive-only; see [`references/scan-quality.md`](references/scan-quality.md). +- **Low path count?** For a REST surface the fix is almost always an accurate spec — get one per `references/openapi-specs.md` (a wired, resolving `openApiConf` is worth far more than any spider tuning). Then: SPA/JS app → `hawk.spider.ajax: true`; GraphQL → wire `graphqlConf`. `hawk.spider.seedPaths` is a last resort (URLs only — no methods/bodies/params, so it can't reach POST/PUT or parameterized routes); prefer even a hand-derived spec over it, and omit it entirely once a spec is wired. - **Auth failing?** Verify `authentication` block; re-fetch the relevant recipe via `hawk config show
--text` (Phase 1c). - **Too noisy / too slow?** Add `app.excludePaths` or `app.includePaths`; tune `hawk.spider.maxDurationMinutes`. - **New API type added?** Add corresponding `graphqlConf`, `openApiConf`, etc. @@ -333,6 +368,34 @@ hawk rescan --scan-id --json-output # fast fix verification — r --- +## Step 4.5: Quality Gate + +Run this gate after **every** scan — full scans and rescans alike, any exit code — before +parsing findings into fix tasks. Exception: on exit 1 where no scan was actually started +(config parse failure, app unreachable pre-scan), there is no `scanId` to gate against — skip +straight to Step 6's diagnosis / the environment-class path instead. It is a feedback loop +into config tuning, **not a governor**: gate state never blocks a finding from being reported +and fixed; a thin scan that missed surface can still have found something real. + +Run the five checks and derive the expectation fresh each time, exactly as +`references/scan-quality.md` describes — coverage (evidence-only), base-path resolve, auth, +surface-completeness, and health. +On config-class gaps (`spec-not-wired`, `surface-unscanned`, `auth-validate-failed`, +`auth-wall`, `all-4xx`, `base-path-mismatch`), loop back to Step 2b with an additive-only fix, batch every gap +into one edit pass, and rescan once. Iteration cap: 2 interactive / 1 autonomous, per +config, counted in rescans. On environment-class gaps (`env-unreachable`), never edit +config — verify the app is up, retry once for free, and stop if it recurs. Multi-config +repos gate sequentially, one config at a time, immediately after that config's scan. + +At the cap, or once checks are clean, proceed to Step 4 regardless — findings are always +reported and fixable. Never say a scan is "done and secure" while a gate gap is open; state +plainly what was scanned, what wasn't, and why, using the reason identifiers. + +→ Full check definitions, commands, degradation when a subcommand is unavailable, and the + iteration/reporting rules: [`references/scan-quality.md`](references/scan-quality.md) + +--- + ## Step 4: Parse Findings and Generate Fix Tasks Use `--json-output` for structured results (requires Dev Release v5.3.41+). Suppresses all @@ -412,14 +475,16 @@ initialized; skill is active. **Skip for:** documentation-only changes; config edits without code changes; exploratory tasks; when the user explicitly says to skip. -**The loop** (full 7-step detail with guard rails): +**The loop** (full step-by-step detail with guard rails): → [`references/autonomous-loop.md`](references/autonomous-loop.md) **Guard rails summary:** - One scan at a time — never run `hawk scan` or `hawk rescan` with `&` or `nohup` - Max one fix-rescan cycle per task — if findings remain after fixing, report them -- Always announce: "Running security scan...", "Found N vulnerabilities, fixing...", "Rescanning..." +- Always narrate with the `StackHawk | ` prefix at every phase (see **Progress output**) — never let the loop run silent - Interruptible — stop immediately if the user says to +- Report gate gaps — never silently accept exit 0 from a thin scan +- Never claim "done and secure" while gate gaps are open --- @@ -428,7 +493,7 @@ initialized; skill is active. - **Don't scan before the app is running.** HawkScan will exit 1 with a connection error. - **Try `https://` first — HawkScan accepts self-signed certificates.** Only fall back to `http://` if the scan actually fails to connect with a TLS error. - **Don't hardcode API keys or credentials in `stackhawk.yml`.** Use env vars. -- **Low path count ≠ clean app.** Feed an API spec or auth config before concluding the app is secure. If auth succeeds but endpoints return empty data, seed the backend (Phase 1c.6). +- **Never accept an exit-0 scan that fails the quality gate.** Untouched planned routes, auth-walls, or an unscanned surface mean config iteration, not done — see [`references/scan-quality.md`](references/scan-quality.md). - **Don't ignore exit code 42.** It's a deliberate signal that findings crossed the threshold — treat it as a build failure. - **String interpolation mid-value doesn't work.** `host: "https://${HOST}/api"` will NOT interpolate. Use `host: ${FULL_HOST_URL}`. - **Never refer to the scanner as ZAP.** The product is HawkScan / StackHawk. The underlying engine is HSTE, not ZAP. Neither "ZAP" nor "OWASP ZAP" should appear in output or instructions. diff --git a/cursor/.cursor/rules/stackhawk-optimize-cli.mdc b/cursor/.cursor/rules/stackhawk-optimize-cli.mdc index 6fe07fd0..68b5f714 100644 --- a/cursor/.cursor/rules/stackhawk-optimize-cli.mdc +++ b/cursor/.cursor/rules/stackhawk-optimize-cli.mdc @@ -12,7 +12,7 @@ alwaysApply: false | Fetch a base policy as JSON | `hawk op policy get --name ` | | Create/upsert a trial or permanent policy | `hawk op policy create --file --name [--display-name ] [--dry-run]` | | Delete a policy | `hawk op policy delete --name --yes [--dry-run]` | -| (Optional) set app default | `hawk op policy assign --app |--app-id --name [--dry-run]` | +| (Optional) set app default | `hawk op policy assign --app --name [--dry-run]` | | Read canonical tech flags | `hawk op app tech-flags get --app --format json` | | Get per-path scan metrics + signal flags | `hawk op scan metrics [--sort heaviest\|slowest\|erroring\|most-requested] [--top N] [--method ] [--operations] --format json` | diff --git a/evals/cli.py b/evals/cli.py index e9a12754..5dbf01cf 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -46,6 +46,12 @@ def main() -> None: out_dir = RESULTS_ROOT / args.harness / "results" / args.skill out_dir.mkdir(parents=True, exist_ok=True) for p in prompts: + if p.target_repo is not None and args.harness != "claude-code": + # Discovery (target_repo) cells clone + read-only-guard a real repo, which + # only the claude-code harness supports; codex/cursor/agy short-circuit them. + # Skip entirely so an un-run cell renders as "not present" in the matrix, + # not a phantom blocking failure that tanks the score for those harnesses. + continue try: run = adapter.launch(p.prompt, args.skill, p.id, plugin_dirs, model=args.model, load_skill=True, @@ -70,6 +76,13 @@ def main() -> None: run.error = run.error or f"judge failed: {type(e).__name__}: {e}" res = grade_discovery(p, run, cfg.checks, judge_checks, platform=args.harness, skill=args.skill) + elif p.own_checks_only: + # Reasoning-only cell (e.g. post-scan gate): grade on its own + # applies_to checks + expected, not the global scan-flow checks + # (preflight/step1/scan) that a paper exercise never performs. + from evals.lib.grading import grade_discovery + res = grade_discovery(p, run, cfg.checks, [], + platform=args.harness, skill=args.skill) else: did = adapter.detect_trigger(run, args.skill) res = grade(p, run, cfg.checks, platform=args.harness, skill=args.skill, diff --git a/evals/harnesses/claude-code/discovery_guard.py b/evals/harnesses/claude-code/discovery_guard.py index 42b23ed7..28ab96c3 100644 --- a/evals/harnesses/claude-code/discovery_guard.py +++ b/evals/harnesses/claude-code/discovery_guard.py @@ -24,7 +24,14 @@ def main(): deny("hawk scan is out of scope for discovery") if re.search(r"\bgit\s+(commit|push)\b", low): deny("git commit/push not allowed") - if re.search(r"\b(docker|docker-compose|podman|nerdctl)\b", low) \ + # Only deny commands that actually START a container/app — not read-only + # inspection (`docker ps`, `docker images`, `docker compose config`, `docker + # inspect`), which the agent legitimately uses to read port mappings and + # config during discovery. Matching the bare word `docker` denied those too. + if re.search(r"\bdocker\s+(run|start|create|compose\s+up|compose\s+run)\b", low) \ + or re.search(r"\bdocker-compose\s+(up|run|start)\b", low) \ + or re.search(r"\b(podman|nerdctl)\s+(run|start|create)\b", low) \ + or re.search(r"\bpodman-compose\s+(up|run|start)\b", low) \ or re.search(r"\b(npm|pnpm|yarn)\s+(start|run\s+dev|run\s+serve|serve)\b", low) \ or re.search(r"\b(bootrun|runserver|uvicorn|gunicorn|hypercorn|nodemon)\b", low) \ or re.search(r"spring-boot:run|flask\s+run|mix\s+phx\.server|air\b|./gradlew\s+bootrun", low) \ diff --git a/evals/hawkscan/answer-keys/firefly-iii.json b/evals/hawkscan/answer-keys/firefly-iii.json index 74fb8911..42b62413 100644 --- a/evals/hawkscan/answer-keys/firefly-iii.json +++ b/evals/hawkscan/answer-keys/firefly-iii.json @@ -7,14 +7,14 @@ "evidence": "composer.json: \"php\": \">=8.5\", \"laravel/framework\": \"^13\"." }, "run_command": { - "value": "Official Docker image `fireflyiii/core` via docker compose against a database (MySQL/MariaDB/PostgreSQL/SQLite), served on container port 8080; or local dev with `php artisan serve` (Laravel dev server on 127.0.0.1:8000). A database is required.", + "value": "In-repo local dev: `composer install`, generate app + Passport keys, run migrations (a database is required — MySQL/MariaDB/PostgreSQL/SQLite), then `php artisan serve` (Laravel dev server on 127.0.0.1:8000). Production packaging ships as the separate `fireflyiii/core` Docker image (container port 8080); there is no docker-compose in this repo. Either identifying the in-repo `artisan serve` path OR the external Docker image is a correct answer.", "must_hit": false, - "evidence": "docker-compose examples; php artisan serve" + "evidence": "No docker-compose/Dockerfile in the repo; `php artisan serve` for local dev; `fireflyiii/core` image is built/published from separate packaging." }, "host": { - "value": "http://localhost:8080 (official Docker image) or http://127.0.0.1:8000 (php artisan serve). APP_URL defaults to http://localhost.", + "value": "http://127.0.0.1:8000 (`php artisan serve`, the in-repo local run) OR http://localhost:8080 (the separate `fireflyiii/core` Docker image). APP_URL defaults to http://localhost. Any of these is correct.", "must_hit": true, - "evidence": ".env.example APP_URL=http://localhost (line 301)." + "evidence": ".env.example APP_URL=http://localhost; Laravel dev server default 127.0.0.1:8000; fireflyiii/core image exposes 8080." }, "api_style": { "value": "REST. Base path /api/v1/. The OpenAPI/Swagger spec ships in the separate firefly-iii/api-docs repo, not served from this app.", diff --git a/evals/hawkscan/process-checks.json b/evals/hawkscan/process-checks.json index 80a12a34..d72cc3e4 100644 --- a/evals/hawkscan/process-checks.json +++ b/evals/hawkscan/process-checks.json @@ -343,11 +343,52 @@ { "id": "discovery_stayed_read_only", "phase": "app-discovery", - "description": "The read-only guard denied nothing (no writes/scans/app-start/remote push).", + "description": "Discovery took no HARD out-of-scope action — no file writes, no scans, no network egress, no remote push. A blocked-and-recovered app-start attempt is graded separately (discovery_no_app_start), not here.", "type": "guard_clean", + "guard_scope": "hard", "applies_to": ["firefly-iii", "wikijs", "memos", "dawarich"], "severity": "blocking", - "failure_guidance": "Discovery is observational; any guard denial means the agent acted outside scope." + "failure_guidance": "A write/scan/network-egress denial means discovery genuinely left its read-only lane — a real regression." + }, + { + "id": "discovery_no_app_start", + "phase": "app-discovery", + "description": "The agent did not try to START the app/container during discovery — host/port and run command should be read from config, not learned by launching anything.", + "type": "guard_clean", + "guard_scope": "app_start", + "applies_to": ["firefly-iii", "wikijs", "memos", "dawarich"], + "severity": "warning", + "failure_guidance": "Weaker models reflexively try `php artisan serve` / `go run` / `docker run` to learn the host; the guard blocks it and discovery recovers, so this is a soft foul (warning), not a blocking failure. Ideal discovery derives host/port from docker-compose/.env/README instead." + }, + { + "id": "gate_identifies_base_path_mismatch", + "phase": "step_4_5", + "description": "For the base-path-mismatch scenario, the agent names the base/context-path resolution gap (the gate's base-path-mismatch signal) as the cause of the systematic 404s.", + "type": "output_contains", + "applies_to": ["hw-21"], + "signals": ["base-path-mismatch", "base path", "base-path", "context path", "context-path"], + "severity": "blocking", + "failure_guidance": "A spec whose paths all 404 while a prefixed variant resolves is a base-path/context-path mismatch, not a missing spec or an environment problem. Naming it correctly is what routes to the right fix (align app.host and the spec base path)." + }, + { + "id": "gate_identifies_auth_wall", + "phase": "step_4_5", + "description": "For the auth-wall scenario, the agent names the auth-wall signal (authenticated paths returning a wall of 401/403) rather than misreading it as an environment problem or coverage-only.", + "type": "output_contains", + "applies_to": ["hw-22"], + "signals": ["auth-wall", "auth wall", "authentication wall"], + "severity": "blocking", + "failure_guidance": "A cluster of 401/403 across authenticated paths on a reachable app is a config-class auth-wall, fixed by correcting the authentication block — not an environment-class gap." + }, + { + "id": "gate_coverage_evidence_reported", + "phase": "step_4_5", + "description": "For the auth-wall scenario, the agent reports the coverage evidence (routes reached vs. expected) rather than ignoring the under-coverage.", + "type": "output_contains", + "applies_to": ["hw-22"], + "signals": ["22", "254", "coverage", "reached"], + "severity": "warning", + "failure_guidance": "Coverage is evidence-only but must still be reported (e.g. 22 of 254 routes reached) so the gap is specific." } ] } diff --git a/evals/hawkscan/prompts.yaml b/evals/hawkscan/prompts.yaml index 5e13f022..9c5dbb7d 100644 --- a/evals/hawkscan/prompts.yaml +++ b/evals/hawkscan/prompts.yaml @@ -105,6 +105,37 @@ invocation_type: negative prompt: What types of vulnerabilities does HawkScan detect? notes: Research question; no scan intent, no code change +- id: hw-21 + should_trigger: true + invocation_type: explicit + prompt: | + Using the hawkscan skill's post-scan quality gate (Step 4.5 / scan-quality.md), evaluate a scan that just completed. You do NOT have live API access in this exercise — reason over the data provided below; do not try to run `hawk op` commands. + + Scanned config (stackhawk.yml): + app: + applicationId: ${APP_ID} + env: Development + host: http://localhost:8080 + openApiConf: + path: /v3/api-docs + The wired spec declares 60 operations under paths like `/authors`, `/books`, `/orders`. + The completed scan's scanned-URI list shows 60 requests and EVERY one returned 404. A manual check shows the app actually serves those routes under `/api/v1` — `/api/v1/authors` returns 200/401 while bare `/authors` returns 404. + + Apply the quality gate: name the gate signal, say whether it is config-class or environment-class, and give the single correct fix. Do not declare the scan done or the app secure. + notes: Gate reasoning — base-path mismatch (spec paths 404 while prefixed variants resolve) + own_checks_only: true +- id: hw-22 + should_trigger: true + invocation_type: explicit + prompt: | + Using the hawkscan skill's post-scan quality gate (Step 4.5 / scan-quality.md), evaluate a completed scan. You do NOT have live API access in this exercise — reason over the data below; do not try to run `hawk op` commands. + + The scanned config has an `authentication:` block configured (OAuth2 bearer), and the app is confirmed up and reachable. + The scan's metrics flag an auth-wall — a large cluster of 401/403 responses across the authenticated paths. The scanned-URI list shows only 22 of an expected 254 routes were reached. + + Apply the quality gate: name the primary gate signal, say whether it is config-class or environment-class, and give the correct next step, noting the coverage evidence. Do not weaken the scan (e.g. widening excludePaths or raising failureThreshold) to make it pass, and do not claim the app is done or secure. + notes: Gate reasoning — auth-wall (config-class) + coverage-gap evidence + own_checks_only: true - id: firefly-iii should_trigger: true invocation_type: implicit diff --git a/evals/lib/grading.py b/evals/lib/grading.py index 0c37c329..fa23526c 100644 --- a/evals/lib/grading.py +++ b/evals/lib/grading.py @@ -28,6 +28,24 @@ def _haystack(run: ParsedRun) -> str: return " ".join([*run.bash_commands, run.output_text]).lower() +# Read-only guard denials come in two flavors during discovery. An app-START +# attempt (`php artisan serve`, `go run`, `docker run`) is BLOCKED by the guard and +# the agent recovers — a soft, model-personality foul (opus resists it, weaker +# models reflex it), so it shouldn't hard-fail a cell whose discovery still +# completed read-only. A write / scan / network-egress denial is a HARD foul: +# discovery genuinely left its read-only lane. A check picks which flavor it asserts +# via `guard_scope`: "hard" (writes/scans/egress only), "app_start", or "all" (default). +_APP_START_DENY = "starting the app/server/container" + + +def _scoped_denials(denials: list[str], scope: str) -> list[str]: + if scope == "hard": + return [d for d in denials if _APP_START_DENY not in d] + if scope == "app_start": + return [d for d in denials if _APP_START_DENY in d] + return list(denials) # "all" + + def run_process_checks(run: ParsedRun, checks: list[dict]) -> list[ProcessCheckResult]: haystack = _haystack(run) # A "don't RUN command X" anti-pattern is about actual execution, so it scans @@ -77,16 +95,17 @@ def run_process_checks(run: ParsedRun, checks: list[dict]) -> list[ProcessCheckR else: passed = anti_hit is None # no preference expressed; only anti-patterns matter elif ctype == "guard_clean": - # Passes iff the read-only guard denied nothing during the run. - passed = not run.guard_denials + # Passes iff the guard denied nothing in this check's scope (see + # _scoped_denials). Default scope "all" keeps prior behavior. + rel = _scoped_denials(run.guard_denials, check.get("guard_scope", "all")) + passed = not rel + # Surface WHAT was denied (in scope) so a failure is diagnosable from the + # artifact without re-running the (costly) cell. + if rel: + anti_hit = "; ".join(d.strip() for d in rel)[:500] else: passed = signal_hit is not None and (anti_hit is None if antis else True) - # Surface WHAT the read-only guard denied so a guard_clean failure is - # diagnosable from the artifact without re-running the (costly) cell. - if ctype == "guard_clean" and run.guard_denials: - anti_hit = "; ".join(d.strip() for d in run.guard_denials)[:500] - results.append(ProcessCheckResult( id=check["id"], passed=passed, severity=check.get("severity", "warning"), diff --git a/evals/lib/models.py b/evals/lib/models.py index c94a0a8b..fc83f836 100644 --- a/evals/lib/models.py +++ b/evals/lib/models.py @@ -47,6 +47,7 @@ class PromptConfig(BaseModel): expected: list[ExpectedCheck] = [] target_repo: TargetRepo | None = None # clone into the cell cwd (read-only run) answer_key: str | None = None # path, relative to the suite dir + own_checks_only: bool = False # grade only this cell's applies_to checks (reasoning cells) — skip global scan-flow checks class Verdict(str, Enum): diff --git a/evals/stackhawk-data-seed/process-checks.json b/evals/stackhawk-data-seed/process-checks.json index 74da6f85..b2783cb3 100644 --- a/evals/stackhawk-data-seed/process-checks.json +++ b/evals/stackhawk-data-seed/process-checks.json @@ -101,7 +101,8 @@ "example test org" ], "severity": "blocking", - "failure_guidance": "Phase 2 is a dialog gate. Emitting artifacts without confirming the proposal violates the skill's contract \u2014 the user may want different entities or counts." + "failure_guidance": "Phase 2 is a dialog gate. Emitting artifacts without confirming the proposal violates the skill's contract \u2014 the user may want different entities or counts.", + "tier": "extended" }, { "id": "phase3_emits_to_data_seed_dir", @@ -114,7 +115,8 @@ "data-seed/credentials.env.example" ], "severity": "warning", - "failure_guidance": "The output contract is data-seed/. Missing these files means downstream consumers (hawkscan reads .data-seed-credentials.env, the future Runner reads manifest.yaml) have nothing to consume." + "failure_guidance": "The output contract is data-seed/. Missing these files means downstream consumers (hawkscan reads .data-seed-credentials.env, the future Runner reads manifest.yaml) have nothing to consume.", + "tier": "extended" }, { "id": "phase3_no_legacy_bootstrap_dir", @@ -138,7 +140,8 @@ "data-seed/manifest.yaml" ], "severity": "warning", - "failure_guidance": "The manifest is the contract between the skill and the future Runner. No manifest = no replay = no value." + "failure_guidance": "The manifest is the contract between the skill and the future Runner. No manifest = no replay = no value.", + "tier": "extended" }, { "id": "phase3_manifest_self_validated", @@ -154,7 +157,8 @@ "no circular" ], "severity": "warning", - "failure_guidance": "A half-broken manifest is worse than a clear error. Self-validation must run before any file is written to disk." + "failure_guidance": "A half-broken manifest is worse than a clear error. Self-validation must run before any file is written to disk.", + "tier": "extended" }, { "id": "phase3_credentials_emitted", @@ -165,7 +169,8 @@ ".data-seed-credentials.env" ], "severity": "warning", - "failure_guidance": "The credentials file is the handoff to hawkscan. Without it the seed is unusable." + "failure_guidance": "The credentials file is the handoff to hawkscan. Without it the seed is unusable.", + "tier": "extended" }, { "id": "phase3_gitignore_updated", @@ -177,7 +182,8 @@ ".data-seed-credentials.env" ], "severity": "warning", - "failure_guidance": "If the credentials file is not gitignored, the suggested commit in Phase 4.2 may publish secrets to the repo." + "failure_guidance": "If the credentials file is not gitignored, the suggested commit in Phase 4.2 may publish secrets to the repo.", + "tier": "extended" }, { "id": "phase3_idempotent_seed_steps", @@ -213,7 +219,8 @@ "Invoke hawkscan" ], "severity": "warning", - "failure_guidance": "Without the summary, the user doesn't know what files exist, what to commit, or how to continue to the next step (hawkscan)." + "failure_guidance": "Without the summary, the user doesn't know what files exist, what to commit, or how to continue to the next step (hawkscan).", + "tier": "extended" }, { "id": "antipattern_no_stackhawk_yml_written", @@ -266,6 +273,68 @@ ], "severity": "warning", "failure_guidance": "Unlike hawkscan, this skill is explicitly NOT autonomous. Treating it as a per-commit safety net wastes the user's time and dirties their tree." + }, + { + "id": "ds17_minimal_seed_proposed", + "phase": "phase_2", + "description": "Given an inline repo digest, the agent proposes a MINIMUM seed (counts + entities, ~one user / one org) rather than an exhaustive dump.", + "type": "output_contains", + "applies_to": [ + "ds-17" + ], + "signals": [ + "minimum", + "minimal", + "one user", + "1 user", + "single user", + "one org", + "1 org", + "single org", + "one organization" + ], + "severity": "blocking", + "failure_guidance": "Phase 2's core is proposing the SMALLEST seed that unblocks an authenticated scan. No minimal-set framing means the agent didn't do the Phase-2 design." + }, + { + "id": "ds17_credential_identified", + "phase": "phase_2", + "description": "Agent identifies the one credential an authenticated HawkScan would use (the bearer/JWT token from the login flow).", + "type": "output_contains", + "applies_to": [ + "ds-17" + ], + "signals": [ + "bearer", + "jwt", + "token", + "api key", + "api_key", + "credential", + "login" + ], + "severity": "warning", + "failure_guidance": "The seed exists to let an authenticated scan in; the proposal should name the credential HawkScan authenticates with." + }, + { + "id": "ds17_confirmation_gate", + "phase": "phase_2", + "description": "Agent surfaces the proposal for confirmation before emitting anything (Phase 2 is a dialog gate).", + "type": "output_contains", + "applies_to": [ + "ds-17" + ], + "signals": [ + "confirm", + "ok to proceed", + "before i", + "review", + "sign off", + "does this look", + "let me know" + ], + "severity": "warning", + "failure_guidance": "Phase 2 must present the proposal for sign-off, not emit artifacts unprompted." } ] } diff --git a/evals/stackhawk-data-seed/prompts.yaml b/evals/stackhawk-data-seed/prompts.yaml index 57173411..419baf22 100644 --- a/evals/stackhawk-data-seed/prompts.yaml +++ b/evals/stackhawk-data-seed/prompts.yaml @@ -78,3 +78,17 @@ invocation_type: negative prompt: Start the docker-compose stack and run the integration tests notes: Environment startup — skill explicitly forbids running services +- id: ds-17 + should_trigger: true + invocation_type: explicit + own_checks_only: true + prompt: | + Using the stackhawk-data-seed skill's Phase 2 (minimum-seed design), propose the seed for this service. You do NOT have shell or CLI access in this exercise — reason over the digest below and output only the proposal (do not write files or run commands). + + Repo digest: + - PostgreSQL datastore. Tables: `users` (id, email, password_hash, org_id), `organizations` (id, name), `api_keys` (id, user_id, token_hash). + - Auth: POST /login takes {email, password} and returns a bearer JWT. Routes under /api/v1 require that bearer token. + - No fixtures/seed checked in; users and organizations are empty on a fresh database. + + Propose the MINIMUM seed an authenticated HawkScan needs to reach the API: which entities and how many, and the one credential HawkScan would authenticate with. Present it for confirmation before emitting anything. + notes: Phase-2 reasoning fixture — minimal-seed proposal from an inline digest (own_checks_only; no live repo / hawk perch needed) diff --git a/gemini-extension.json b/gemini-extension.json index 0054094a..7e2386ef 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,5 +1,5 @@ { "name": "stackhawk", - "version": "2.2.0", + "version": "2.4.0", "description": "StackHawk security scanning and API reporting skills. AUTONOMOUS: After completing any code change, invoke the hawkscan skill to scan, fix all reported vulnerabilities (not just your changes), and rescan to verify. Also invoke for any mention of hawkscan, stackhawk, DAST, security scan." } diff --git a/plugin.json b/plugin.json index c8aeee57..38d6368a 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "name": "stackhawk", - "version": "2.2.0", + "version": "2.4.0", "description": "StackHawk security scanning and API reporting skills. Includes hawkscan (DAST scanning) and api (security posture reporting).", "author": { "name": "StackHawk", diff --git a/plugins/api/.claude-plugin/plugin.json b/plugins/api/.claude-plugin/plugin.json index dc390d94..9fdb379c 100644 --- a/plugins/api/.claude-plugin/plugin.json +++ b/plugins/api/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "stackhawk-api", - "version": "2.2.0", + "version": "2.4.0", "description": "Query the StackHawk platform API for security posture reporting, findings analysis, and app management. Uses the combined hawk CLI (hawk op \u2026) for all platform queries.", "author": { "name": "StackHawk", diff --git a/plugins/api/.codex-plugin/plugin.json b/plugins/api/.codex-plugin/plugin.json index 65575443..50f77b1a 100644 --- a/plugins/api/.codex-plugin/plugin.json +++ b/plugins/api/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "stackhawk-api", - "version": "2.2.0", + "version": "2.4.0", "description": "Query the StackHawk platform API for security posture reporting, findings analysis, and app management. Uses the combined hawk CLI (hawk op \u2026) for all platform queries.", "author": { "name": "StackHawk", diff --git a/plugins/api/skills/api/SKILL.md b/plugins/api/skills/api/SKILL.md index 1a38465c..44637d42 100644 --- a/plugins/api/skills/api/SKILL.md +++ b/plugins/api/skills/api/SKILL.md @@ -1,6 +1,6 @@ --- name: api -version: 2.2.0 +version: 2.4.0 description: > Use this skill when a user or agent needs to query the StackHawk platform for security reporting, findings analysis, or app management. Triggers include: @@ -40,8 +40,7 @@ Before making any calls, check what's available: - valid org + JWT → proceed. - installed but unconfigured → `hawk init --browser` (interactive; the combined binary has no `hawk op init` — `hawk init` writes `~/.hawk/hawk.properties`, which - `hawk op` reads along with the `HAWK_API_KEY` env var; the legacy - standalone-binary config file is no longer used). + `hawk op` reads along with the `HAWK_API_KEY` env var). - not installed → instruct the user to install the `hawk` CLI (docs: [docs.stackhawk.com](https://docs.stackhawk.com)) and stop; do **not** fall back to curl. diff --git a/plugins/api/skills/api/references/hawk-op-shortcuts.md b/plugins/api/skills/api/references/hawk-op-shortcuts.md index 80ab438d..2550197f 100644 --- a/plugins/api/skills/api/references/hawk-op-shortcuts.md +++ b/plugins/api/skills/api/references/hawk-op-shortcuts.md @@ -47,8 +47,7 @@ hawk op status # Confirm auth ``` `hawk init` writes credentials to `~/.hawk/hawk.properties`. -`hawk op` reads that file plus the `HAWK_API_KEY` env var. The legacy -standalone-binary config file is no longer used. +`hawk op` reads that file plus the `HAWK_API_KEY` env var. **CI/CD only:** For pipeline use, set `HAWK_API_KEY` directly as a secret. Org ID and output format can be set via env vars: @@ -103,8 +102,7 @@ extraction, no token handling. ### Latest scan for an app (overview + alerts) ```bash -hawk op scan get --app "" -hawk op scan get --app-id +hawk op scan get --app "" # --app takes a name or a UUID ``` Wraps: `/scan/{orgId}` + `/scan/{scanId}/alerts`. Returns a scan header, alert @@ -295,7 +293,7 @@ call — useful when building a reporting pipeline. - **Don't forget `--detail full` when you need the remediation/HTTP message payload.** The default `hawk op scan get` output is the overview — it won't include per-URI evidence. -- **Don't combine `--app` (name) and `--app-id` (UUID) in the same command.** - Pick one. `--app` resolves by name (case-insensitive exact match); `--app-id` is exact by UUID. - If a name matches **more than one** app, `hawk op` errors and tells you to use `--app-id` — - fall back to `--app-id ` (grab the UUID from `hawk op app list`). +- **`--app` takes a name or a UUID.** A UUID resolves exactly; a name resolves by + case-insensitive exact match. If a name matches **more than one** app, `hawk op` errors and + lists the matches — disambiguate by passing that app's UUID to `--app` (grab it from + `hawk op app list`). diff --git a/plugins/hawkscan-ci/.claude-plugin/plugin.json b/plugins/hawkscan-ci/.claude-plugin/plugin.json index 72719a3b..63b28a6d 100644 --- a/plugins/hawkscan-ci/.claude-plugin/plugin.json +++ b/plugins/hawkscan-ci/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "hawkscan-ci", - "version": "2.2.0", + "version": "2.4.0", "description": "Configure HawkScan in your CI/CD pipeline. Provider-agnostic: detects GitHub Actions, GitLab CI, Jenkins, CircleCI, Azure Pipelines, Bitbucket, Buildkite, and other CI systems from repo files, edits the pipeline file in place to add a HawkScan job, and prompts for HAWK_API_KEY storage (CI-native secrets store or external secrets manager).", "author": { "name": "StackHawk", diff --git a/plugins/hawkscan-ci/.codex-plugin/plugin.json b/plugins/hawkscan-ci/.codex-plugin/plugin.json index ba893c21..b73cced8 100644 --- a/plugins/hawkscan-ci/.codex-plugin/plugin.json +++ b/plugins/hawkscan-ci/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "hawkscan-ci", - "version": "2.2.0", + "version": "2.4.0", "description": "Configure HawkScan in your CI/CD pipeline. Provider-agnostic: detects GitHub Actions, GitLab CI, Jenkins, CircleCI, Azure Pipelines, Bitbucket, Buildkite, and other CI systems from repo files, edits the pipeline file in place to add a HawkScan job, and prompts for HAWK_API_KEY storage (CI-native secrets store or external secrets manager).", "author": { "name": "StackHawk", diff --git a/plugins/hawkscan-ci/skills/hawkscan-ci/SKILL.md b/plugins/hawkscan-ci/skills/hawkscan-ci/SKILL.md index 8c44d497..90820aa4 100644 --- a/plugins/hawkscan-ci/skills/hawkscan-ci/SKILL.md +++ b/plugins/hawkscan-ci/skills/hawkscan-ci/SKILL.md @@ -1,19 +1,21 @@ --- name: hawkscan-ci -version: 2.2.0 +version: 2.4.0 description: > - Use when the user wants to configure HawkScan in their CI/CD pipeline — - triggers on "set up hawkscan in CI", "add stackhawk to my pipeline", - "scan in CI", "configure github actions / gitlab / jenkins / circleci - for hawkscan", "wire hawkscan into ci/cd", or any provider-named - variant. Provider-agnostic: detects the CI system from repo files, - edits the pipeline file in place to add a HawkScan job, prompts the - user to set HAWK_API_KEY in their CI's native secrets engine (or an - organizationally-approved external secrets manager), and wires - commit-SHA + branch traceability. Defers every local-scan concern - (stackhawk.yml, auth, findings, triage) to the hawkscan skill — - requires a working local scan path before activating. Explicit - trigger only; no autonomous code-change hook. + Use when the user wants to WIRE HawkScan into a CI/CD pipeline config + file — triggers on "set up hawkscan in CI", "add stackhawk to my + pipeline", "scan in CI", "configure github actions / gitlab / jenkins / + circleci for hawkscan", "wire hawkscan into ci/cd", or any provider-named + variant. Provider-agnostic: detects the CI system from repo files, edits + the pipeline file in place to add a HawkScan job, prompts for HAWK_API_KEY + storage in the CI's native secrets store (or an approved external + manager), and wires commit-SHA + branch traceability. If no local + stackhawk.yml exists yet, still trigger and route the local-config work to + the hawkscan skill. Do NOT trigger for: documentation-only changes (e.g. + editing a README about CI scanning); informational or research questions + ("what CI providers does HawkScan support?"); or running / performing a + scan — a local scan is the hawkscan skill, whereas this skill only edits + CI pipeline config. Explicit trigger only; no autonomous code-change hook. --- # HawkScan CI Skill diff --git a/plugins/hawkscan/.claude-plugin/plugin.json b/plugins/hawkscan/.claude-plugin/plugin.json index f59dfed1..56f2c42c 100644 --- a/plugins/hawkscan/.claude-plugin/plugin.json +++ b/plugins/hawkscan/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "hawkscan", - "version": "2.2.0", + "version": "2.4.0", "description": "Configure, run, and interpret HawkScan DAST results inside Claude Code. Generates stackhawk.yml configs, runs scans via CLI or Docker, and transforms security findings into prioritized fix tasks for your coding agent.", "author": { "name": "StackHawk", diff --git a/plugins/hawkscan/.codex-plugin/plugin.json b/plugins/hawkscan/.codex-plugin/plugin.json index 44ee20a7..20e6085c 100644 --- a/plugins/hawkscan/.codex-plugin/plugin.json +++ b/plugins/hawkscan/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "hawkscan", - "version": "2.2.0", + "version": "2.4.0", "description": "Configure, run, and interpret HawkScan DAST results. Generates stackhawk.yml configs, runs scans via CLI or Docker, and transforms security findings into prioritized fix tasks for your coding agent.", "author": { "name": "StackHawk", diff --git a/plugins/hawkscan/skills/hawkscan/SKILL.md b/plugins/hawkscan/skills/hawkscan/SKILL.md index ae949a23..bb90b53d 100644 --- a/plugins/hawkscan/skills/hawkscan/SKILL.md +++ b/plugins/hawkscan/skills/hawkscan/SKILL.md @@ -1,19 +1,21 @@ --- name: hawkscan -version: 2.2.0 +version: 2.4.0 description: > Runs the HawkScan DAST security loop — configure, scan, fix all reported - vulnerabilities (not just your changes), rescan to verify. Use when the user - asks to run or perform a security/DAST scan, to test an app or API for - vulnerabilities, or to verify a vulnerability is fixed; and AUTONOMOUSLY right - after you complete a code change (feature, bugfix, refactor) — "done" means - "done and secure," so run the loop without asking permission. Do NOT trigger - for: informational questions about what HawkScan is, detects, or how it works - (e.g. "what vulnerabilities does HawkScan find?"); editing stackhawk.yml or - other config without running a scan; querying existing findings, security - posture, untriaged counts, or scan history (use the stackhawk-api skill); - documentation-only changes; installing or setting up the CLI; or when the - user explicitly says to skip scanning. + vulnerabilities (not just your changes), rescan to verify. Performs + code-first discovery to configure high-value scans, and quality-gates every + scan against code-derived expectations before findings are parsed. Use when + the user asks to run or perform a security/DAST scan, to test an app or API + for vulnerabilities, or to verify a vulnerability is fixed; and AUTONOMOUSLY + right after you complete a code change (feature, bugfix, refactor) — "done" + means "done and secure," so run the loop without asking permission. Do NOT + trigger for: informational questions about what HawkScan is, detects, or how + it works (e.g. "what vulnerabilities does HawkScan find?"); editing + stackhawk.yml or other config without running a scan; querying existing + findings, security posture, untriaged counts, or scan history (use the + stackhawk-api skill); documentation-only changes; installing or setting up + the CLI; or when the user explicitly says to skip scanning. --- # HawkScan Skill @@ -32,12 +34,20 @@ This skill requires **hawk v6.0.0 or newer**. Verify: ```bash hawk version hawk config --help >/dev/null 2>&1 || echo "MISSING: hawk config — upgrade hawk to v6.0.0+" +hawk skills status ``` If hawk is older than `6.0.0` or `hawk config --help` fails, stop and tell the user to upgrade before proceeding. Do not inline auth recipes from memory — they live in `hawk config show` and are stale by design when hardcoded. +**`hawk skills status`** reports whether the installed StackHawk agent skills are current. +If it lists anything out of date, surface the exact upgrade command it prints (e.g. +`/plugin marketplace update stackhawk && /plugin update hawkscan`) and recommend the user +update before continuing — a stale skill may be missing fixes for the very issues you'll hit. +It's a recommendation, not a hard stop: proceed if the user prefers. (If the subcommand isn't +recognized, the installed hawk predates it — skip this check.) + See `references/installation.md` for upgrade and install instructions. --- @@ -49,19 +59,23 @@ The `api` skill wraps read-only StackHawk platform lookups via the `hawk` CLI (` | Purpose | Command | |-------------------------------|----------------------------------------------------------------------------------| | Check if App exists | `hawk op app list --format json` | -| Check if Env exists | `hawk op env list --app --format json` | -| Get findings with triage | `hawk op scan get --app --detail full --format json` | +| Check if Env exists | `hawk op env list --app --format json` | +| Get findings with triage | `hawk op scan get --app --detail full --format json` | | List ASM repos | `hawk op repo list --format json` | | Link app to ASM repo | `hawk op repo link --repo-id --app-id ` | -| Get tech flags | `hawk op app tech-flags get --app --format json` | -| Disable all tech flags | `hawk op app tech-flags disable-all --app --yes` | -| Set specific tech flags | `hawk op app tech-flags set --app Key=true` | +| Get tech flags | `hawk op app tech-flags get --app --format json` | +| Disable all tech flags | `hawk op app tech-flags disable-all --app --yes` | +| Set specific tech flags | `hawk op app tech-flags set --app Key=true` | | Triage a finding | `hawk op scan triage --scan --hash --status false-positive --note ""` | | Bulk triage from file | `hawk op scan triage --scan --from-file triage.yaml` | | Annotate w/o triage perm | `hawk op finding note --scan --hash --note "..."` | +| Get scanned URIs | `hawk op scan uris --format json` (skip if the subcommand errors) | +| Get effective scan config | `hawk op scan config ` (skip if the subcommand errors) | These reads require the combined `hawk` CLI; the `api` skill covers setup (`hawk init --browser` or the `HAWK_API_KEY` env var). +**Selecting an app:** `--app` takes an application **name or UUID** — pass whichever you have (e.g. the `applicationId` from `stackhawk.yml`, or the app's name). No separate flag is needed. Also parse `--format json` output defensively — a "skills out of date" banner may precede the JSON. + The `stackhawk-data-seed` skill sets up checked-in backend seed data via `hawk perch seed`. Hand off to it when authentication fails because the backend has no valid credential (**Phase 1c.6**), or after a scan when auth succeeded but endpoints returned empty data @@ -85,6 +99,20 @@ names deliberately. Findings have a lifecycle (NEW, FALSE_POSITIVE, RISK_ACCEPTE --- +## Progress output + +Keep the terminal informed so a running loop never looks silent. Prefix every status line with +`StackHawk | ` — **ASCII only, no emoji** (must render in Cursor, Windows `cmd`, and PowerShell; +keep the branded lines themselves plain ASCII). Emit one short line **entering each phase** — +discovery, config, scan, quality gate, findings, each fix batch, rescan, report — with what's +happening plus the key number (routes, coverage, findings). Before a scan/rescan, note it takes +a few minutes and let hawk's own progress stream through (don't suppress it); announce +completion after. Status output, **not** a prompt — never pause for input. E.g. +`StackHawk | Scanning http://localhost:8080 - a few minutes; progress streams below`, then +`StackHawk | 3 findings (2 High, 1 Medium) - fixing all`. Per-phase lines: `references/autonomous-loop.md`. + +--- + ## Phase 0: App Setup & Verification Run Phase 0 **once** when onboarding a new application (`stackhawk.yml` being created for @@ -125,32 +153,28 @@ enable only detected flags via `hawk op app tech-flags`; if none, skip. > 2. `CI=true` or `GITHUB_ACTIONS=true` → `CI` > 3. Git branch `main`/`master`/`production` → `Production`; `staging` → `Staging`; otherwise → `Development` -### Step 1a: Understand the Application +### Step 1a: Discover What to Scan -Before generating config, build a working understanding of the target: **how it runs -locally, how to reach its API, whether it's a SPA, and whether exercising it requires -auth.** Work in three passes; stop as soon as you can answer all four: +Before generating config, run code-first discovery: find every API surface in the repo +(REST, GraphQL, gRPC, SPA, ...), derive a route inventory per surface, and get the run +command, host/port, and auth shape needed to reach each one. Prefer the repo's own docs +over guessing, explore code to fill gaps, and ask the user directly for whatever remains +unresolved — never stall or invent. -1. **Read what the repo already documents.** Most repos describe how to run themselves — - prefer their own words over guessing, and treat them as authoritative. - → Source files and what to harvest: [`references/app-discovery.md`](references/app-discovery.md) +**Discovery is read-only and static.** Determine the run command, host, and port by +*reading* config and docs — `docker-compose.yml` port mappings, `.env`/`.env.example` +(e.g. `APP_URL`), the README — not by launching anything. You never need to start the app, +run a container, or bring up a server to discover *what* to scan; standing up the target is +a scan-time step (Step 1c), not part of discovery. -2. **Explore to fill the gaps.** For anything the docs don't answer, investigate the repo - the way a developer would when handed an unfamiliar codebase — read the run/build - manifests, entry points, and route definitions — until you can state, concretely: - - **Run command + host/port** — how to start it locally and where it listens. - - **API style + base path** — REST (note an OpenAPI/Swagger spec if one is served or - checked in), GraphQL, gRPC, or a plain HTML web app. - - **SPA?** — a client-rendered JS front end changes what we scan. - - **Auth?** — does reaching the real endpoints require a login, and what shape does it - appear to use? This is an *understanding* signal only; picking, obtaining, and - configuring the credential is Step 1c / Phase 1c. +→ Full discovery workflow — per-surface detection, route-inventory derivation, gap + recommendations, and the user-confirmed summary required before the first scan: + [`references/scan-planning.md`](references/scan-planning.md) +→ Docs-first source table (which files to read and what to harvest): + [`references/app-discovery.md`](references/app-discovery.md) -3. **Confirm the unknowns with the user.** Never stall or invent. Ask directly for - anything still unresolved — most often the start command, host/port, or the login. When - file access is limited, skip straight to asking for all four. - -Record the four answers — Step 1c and Step 2 consume them. +Record what discovery produced — surfaces, run command, host/port, auth shape — Step 1c and +Step 2 consume them. **SPA rule:** if the app is a client-rendered JS front end, never scan it without the Ajax Spider, and note that a separate backend API is usually the higher-value target. Full @@ -172,7 +196,8 @@ strategy, frontend-vs-backend scenarios, and config templates: ```bash hawk create app --name "" --env ``` - Resolve `` with the Env Name Algorithm above. Announce the created app ID and URL. + Resolve `` with the Env Name Algorithm above. Announce it (Progress output): + `StackHawk | Created app () - `. 6. **Env exists?** Determine env name via Env Name Algorithm. Run `hawk op env list --app --format json`. Reuse if exists; otherwise run `hawk op env create --app --env --host `. @@ -208,6 +233,16 @@ Do not proceed to Step 3 until validation passes. → API-type-specific config (OpenAPI, GraphQL, gRPC, seed paths, spider tuning): [`references/config-patterns.md`](references/config-patterns.md) +**REST surface? Get an accurate OpenAPI spec BEFORE the first scan — this is not optional.** +A REST scan with no spec (spider/`seedPaths` only) reaches a small fraction of the API; a +scan with a *wrong* spec 404s every path. Do not skip to a scan on the assumption a spec is +missing or "good enough" — **open and follow [`references/openapi-specs.md`](references/openapi-specs.md)** to +work its preference order (a spec the running app serves → a code/build change that generates +one → a spec published outside the repo → hand-derived), then run its resolve-check +(`host + spec-path` returns real routes, not 404s) before scanning. Reaching for +`hawk.spider.seedPaths` instead of a spec is a last resort, not a shortcut — read that file +first. Same idea for GraphQL/gRPC: wire the schema/proto, don't scan blind. + ### Phase 1c: Authentication Configuration Use `hawk config show` to fetch the canonical recipe for the app's auth pattern. @@ -269,7 +304,9 @@ After seeding, re-run `hawk validate auth stackhawk.yml` and continue. Review the config against the current app state: -- **Low path count?** Check in order: SPA/JS app → enable `hawk.spider.ajax: true`; API spec available → wire `openApiConf`/`graphqlConf`; specific known-deep paths → add `seedPaths`. Omit `seedPaths` unless there is a specific reason — they're rarely needed when Ajax Spider or a spec is configured. +- **Gate reported gaps?** Tune what the gate names (wire spec, enable ajax spider, fix + auth) — additive-only; see [`references/scan-quality.md`](references/scan-quality.md). +- **Low path count?** For a REST surface the fix is almost always an accurate spec — get one per `references/openapi-specs.md` (a wired, resolving `openApiConf` is worth far more than any spider tuning). Then: SPA/JS app → `hawk.spider.ajax: true`; GraphQL → wire `graphqlConf`. `hawk.spider.seedPaths` is a last resort (URLs only — no methods/bodies/params, so it can't reach POST/PUT or parameterized routes); prefer even a hand-derived spec over it, and omit it entirely once a spec is wired. - **Auth failing?** Verify `authentication` block; re-fetch the relevant recipe via `hawk config show
--text` (Phase 1c). - **Too noisy / too slow?** Add `app.excludePaths` or `app.includePaths`; tune `hawk.spider.maxDurationMinutes`. - **New API type added?** Add corresponding `graphqlConf`, `openApiConf`, etc. @@ -342,6 +379,34 @@ hawk rescan --scan-id --json-output # fast fix verification — r --- +## Step 4.5: Quality Gate + +Run this gate after **every** scan — full scans and rescans alike, any exit code — before +parsing findings into fix tasks. Exception: on exit 1 where no scan was actually started +(config parse failure, app unreachable pre-scan), there is no `scanId` to gate against — skip +straight to Step 6's diagnosis / the environment-class path instead. It is a feedback loop +into config tuning, **not a governor**: gate state never blocks a finding from being reported +and fixed; a thin scan that missed surface can still have found something real. + +Run the five checks and derive the expectation fresh each time, exactly as +`references/scan-quality.md` describes — coverage (evidence-only), base-path resolve, auth, +surface-completeness, and health. +On config-class gaps (`spec-not-wired`, `surface-unscanned`, `auth-validate-failed`, +`auth-wall`, `all-4xx`, `base-path-mismatch`), loop back to Step 2b with an additive-only fix, batch every gap +into one edit pass, and rescan once. Iteration cap: 2 interactive / 1 autonomous, per +config, counted in rescans. On environment-class gaps (`env-unreachable`), never edit +config — verify the app is up, retry once for free, and stop if it recurs. Multi-config +repos gate sequentially, one config at a time, immediately after that config's scan. + +At the cap, or once checks are clean, proceed to Step 4 regardless — findings are always +reported and fixable. Never say a scan is "done and secure" while a gate gap is open; state +plainly what was scanned, what wasn't, and why, using the reason identifiers. + +→ Full check definitions, commands, degradation when a subcommand is unavailable, and the + iteration/reporting rules: [`references/scan-quality.md`](references/scan-quality.md) + +--- + ## Step 4: Parse Findings and Generate Fix Tasks Use `--json-output` for structured results (requires Dev Release v5.3.41+). Suppresses all @@ -421,14 +486,16 @@ initialized; skill is active. **Skip for:** documentation-only changes; config edits without code changes; exploratory tasks; when the user explicitly says to skip. -**The loop** (full 7-step detail with guard rails): +**The loop** (full step-by-step detail with guard rails): → [`references/autonomous-loop.md`](references/autonomous-loop.md) **Guard rails summary:** - One scan at a time — never run `hawk scan` or `hawk rescan` with `&` or `nohup` - Max one fix-rescan cycle per task — if findings remain after fixing, report them -- Always announce: "Running security scan...", "Found N vulnerabilities, fixing...", "Rescanning..." +- Always narrate with the `StackHawk | ` prefix at every phase (see **Progress output**) — never let the loop run silent - Interruptible — stop immediately if the user says to +- Report gate gaps — never silently accept exit 0 from a thin scan +- Never claim "done and secure" while gate gaps are open --- @@ -437,7 +504,7 @@ initialized; skill is active. - **Don't scan before the app is running.** HawkScan will exit 1 with a connection error. - **Try `https://` first — HawkScan accepts self-signed certificates.** Only fall back to `http://` if the scan actually fails to connect with a TLS error. - **Don't hardcode API keys or credentials in `stackhawk.yml`.** Use env vars. -- **Low path count ≠ clean app.** Feed an API spec or auth config before concluding the app is secure. If auth succeeds but endpoints return empty data, seed the backend (Phase 1c.6). +- **Never accept an exit-0 scan that fails the quality gate.** Untouched planned routes, auth-walls, or an unscanned surface mean config iteration, not done — see [`references/scan-quality.md`](references/scan-quality.md). - **Don't ignore exit code 42.** It's a deliberate signal that findings crossed the threshold — treat it as a build failure. - **String interpolation mid-value doesn't work.** `host: "https://${HOST}/api"` will NOT interpolate. Use `host: ${FULL_HOST_URL}`. - **Never refer to the scanner as ZAP.** The product is HawkScan / StackHawk. The underlying engine is HSTE, not ZAP. Neither "ZAP" nor "OWASP ZAP" should appear in output or instructions. diff --git a/plugins/hawkscan/skills/hawkscan/references/app-discovery.md b/plugins/hawkscan/skills/hawkscan/references/app-discovery.md index fb47a00f..70f985e8 100644 --- a/plugins/hawkscan/skills/hawkscan/references/app-discovery.md +++ b/plugins/hawkscan/skills/hawkscan/references/app-discovery.md @@ -1,9 +1,9 @@ -# App Discovery Reference +# App Discovery — Docs-First Source Table -How to understand a target application before configuring a scan — what it is, how to run -it locally, how to reach its API, and whether it needs auth. Used by Step 1a in SKILL.md. -Prefer the repo's own documentation; explore only to fill gaps; ask the user for whatever -remains. Do not treat any of this as a fixed checklist of commands. +The full discovery workflow lives in `references/scan-planning.md` — see Step 1a in SKILL.md +(per-surface detection, route-inventory derivation, gap recommendations, and the +user-confirmed summary required before the first scan). This file holds the docs-first source +table below, which SKILL.md links to directly and which stands on its own. ## Read the repo's own docs first @@ -23,28 +23,3 @@ order, and treat what they say as authoritative: Harvest the **run/start command**, the **local host + port**, the **API style** (REST/OpenAPI, GraphQL, gRPC, or a plain web app), and any **documented dev/test login or seed data**. If a file answers a question, don't rediscover it by exploring. - -## The four answers discovery must produce - -From docs, exploration, or the user — be able to state all four before generating config: - -1. **Run command + host/port** — the exact way to start the app locally and the URL it - listens on. Feeds Step 1c ("App running?") and the `host:` in Step 2. -2. **API style + base path** — REST (an OpenAPI/Swagger spec, served or checked in, - enables `openApiConf`), GraphQL (`graphqlConf`), gRPC (`grpcConf`), or an HTML web app. - Determines the config shape in Step 2. -3. **SPA or not** — a client-rendered JS front end (React, Vue, Angular, Svelte, Next, - Nuxt, and the like) requires the Ajax Spider and changes target selection. See the SPA - rule in Step 1a of SKILL.md. -4. **Auth needed?** — does reaching the real endpoints require a login, and what shape does - it appear to use (form/session, bearer token, OAuth, custom)? Understand this so you - scan authenticated when required and don't waste a run against a login wall. **Stop - there** — picking, configuring, obtaining, or seeding the credential is owned by Step 1c - / Phase 1c in SKILL.md and the `stackhawk-data-seed` skill. Do not duplicate that here. - -## When you can't determine something - -Don't stall and don't invent. After reading the docs and a reasonable look through the -repo, ask the user directly for whatever is unresolved — most often the exact start -command, the host/port, or how to log in locally. When file access is limited, skip -exploration and ask up front for all four answers. diff --git a/plugins/hawkscan/skills/hawkscan/references/autonomous-loop.md b/plugins/hawkscan/skills/hawkscan/references/autonomous-loop.md index bff60692..c5e9a827 100644 --- a/plugins/hawkscan/skills/hawkscan/references/autonomous-loop.md +++ b/plugins/hawkscan/skills/hawkscan/references/autonomous-loop.md @@ -8,11 +8,13 @@ ## The Loop steps -After completing a code change, announce and execute these steps in order. +After completing a code change, announce and execute these steps in order. Narrate every +phase with the `StackHawk | ` prefix (SKILL.md **Progress output**) so the loop is never +silent — ASCII only, no emoji. **1. Announce** -Say: "Implementation complete. Running security scan against the application." +Say: `StackHawk | Implementation complete - running a security scan against the app` **2. Configure** @@ -42,20 +44,39 @@ timeout 30 hawk validate config stackhawk.yml || echo "Validate timed out — en **4. Scan** +Announce before launching: `StackHawk | Scanning - this takes a few minutes; scan progress streams below`. Run it synchronously and let hawk's own progress print (don't suppress it): + ```bash hawk scan --json-output ``` -Capture `scan.id` from the JSON output — you'll need it for rescan. +Capture `scan.id` from the JSON output — you'll need it for rescan. Then announce completion: +`StackHawk | Scan complete - URLs scanned (scan )`. + +**5. Quality gate** + +Before triaging findings, run the quality gate (`scan-quality.md`'s five checks — coverage, +base-path resolve, auth, surface-completeness, health) against this scan. On config-class gaps +(`spec-not-wired`, `surface-unscanned`, `auth-validate-failed`, `auth-wall`, `all-4xx`, +`base-path-mismatch`), loop +back to config tuning with a single additive-only batched fix and rescan once — the +autonomous cap here is **1** config-fix iteration (interactive workflows get 2; this +unattended loop gets 1). On environment-class gaps (`env-unreachable`: connection failures, +timeouts, app unreachable mid-scan), never edit the config: verify the app is up, retry the +scan once for free, and if the same environment-class signal recurs on the retry, report it +as an environment problem and stop — do not retry again or reach for a config edit. At the +cap, or once the gate is clean, proceed to the next step regardless — findings are always +reported and fixable independent of gate state. Announce the gate result: +`StackHawk | Quality gate: coverage /, auth , `. -**5. If findings exist** +**6. If findings exist** Run the Step 5 triage filter first (per-path `status` field): - Skip `FALSE_POSITIVE` / `RISK_ACCEPTED` paths entirely - Prioritize `ASSIGNED` paths before `NEW` of the same severity - Fix `NEW` paths in severity order -Announce: "Found [N] actionable vulnerabilities (+ [M] skipped due to prior triage). Fixing all actionable ones." +Announce: `StackHawk | [N] actionable findings (+ [M] skipped via prior triage) - fixing all` **Fix ALL findings — not just ones related to your recent changes.** DAST scans the entire running application. Pre-existing vulnerabilities are just as exploitable as new ones. @@ -63,9 +84,10 @@ Fix in severity order: High → Medium → Low; within same severity: injection Commit format: `fix: resolve [CWE-XXX] [vulnerability type] found by HawkScan` -**6. Rescan** +**7. Rescan** -Rescan is the default for all fix-verify cycles: +Rescan is the default for all fix-verify cycles. Announce before launching: +`StackHawk | Rescanning to verify fixes - a few minutes; progress streams below`. ```bash hawk rescan --scan-id --json-output @@ -87,12 +109,20 @@ Decision table for when you're tempted to skip rescan: | "Rescan might miss something" | Rescan re-runs exactly the plugins that fired on the parent scan. It's more targeted, not less. | | "The fix added new endpoints that need scanning" | This IS a full-scan condition — use full scan. | -**7. Report** +**8. Quality gate (post-rescan)** + +Run the same gate again against the rescan, per the rule that it runs after every scan and +rescan alike. The autonomous cap still applies per config across this whole task (scan + +rescan together), not per gate invocation — don't treat the rescan as a fresh budget for a +second config-fix iteration. + +**9. Report** -- If clean: "Rescan complete. Zero new findings. All security issues have been resolved." -- If findings remain: "Rescan found [N] remaining issues that require manual review:" and list them. +- If clean **and no gate gaps remain open**: `StackHawk | Rescan clean - 0 new findings; all security issues resolved on the scanned surface` +- If clean **but gate gaps remain open** (cap reached with config-class gaps still unresolved, or an environment-class gap that recurred): report the gaps by reason identifier, the coverage evidence, and the remaining next steps. Do not say "all security issues have been resolved" or otherwise imply the app is done and secure while a gap is open. +- If findings remain: `StackHawk | Rescan: [N] issues remain and need manual review` — then list them. - If findings were filtered by triage state: "Skipped [X] findings already triaged as RISK_ACCEPTED / FALSE_POSITIVE." -- If findings were marked FALSE_POSITIVE in Step 5: "Marked [N] findings as false positive — review at https://app.stackhawk.com/scans/\". +- If findings were marked FALSE_POSITIVE in step 6: "Marked [N] findings as false positive — review at https://app.stackhawk.com/scans/\". --- @@ -100,6 +130,8 @@ Decision table for when you're tempted to skip rescan: - **One scan at a time per app/env.** Never dispatch a second `hawk scan` or `hawk rescan` while one is still running. Run scan commands synchronously — never with `&` or `nohup`. Wait for the exit code. - **Max one fix-rescan cycle per task.** If the rescan still has findings after fixing, report the remaining issues rather than looping indefinitely. The user can ask for a follow-up. -- **Always announce what you're doing.** The developer should see "Running security scan...", "Found N vulnerabilities, fixing...", "Rescanning to verify..." in the output. +- **Always announce what you're doing** with the `StackHawk | ` prefix at every phase (SKILL.md **Progress output**) — the developer should see scan start, gate result, findings, and rescan stream by, never a silent gap. ASCII only, no emoji. - **Interruptible.** If the user interrupts or says to stop, stop immediately. -- **Don't block on scan failures.** If `hawk scan` exits with code 1 (config error, app unreachable), report the error and suggest fixes rather than retrying in a loop. +- **Don't block on scan failures.** If `hawk scan` exits with code 1 (config error, app unreachable), report the error rather than retrying in a loop — with one exception: an environment-class failure (connection refused, timeout, unreachable) gets exactly one free retry after verifying the app is up. If the same environment-class signal recurs on that retry, report it and stop; don't retry a second time. +- **Report gate gaps — never silently accept exit 0 from a thin scan.** +- **Never claim "done and secure" while gate gaps are open.** diff --git a/plugins/hawkscan/skills/hawkscan/references/openapi-specs.md b/plugins/hawkscan/skills/hawkscan/references/openapi-specs.md new file mode 100644 index 00000000..539064dd --- /dev/null +++ b/plugins/hawkscan/skills/hawkscan/references/openapi-specs.md @@ -0,0 +1,178 @@ +# OpenAPI Spec Accuracy Reference + +A REST scan is only as good as the spec behind it. An **accurate** spec is one whose +operations match the app's real routes *and* resolve against the scanned host — same base +path, same context path. An inaccurate spec is worse than no spec: HawkScan faithfully +requests every path it declares, so wrong paths become a wall of 404s that bury real +coverage and generate recurring false-positive noise on every rescan. + +This reference is framework-generic. Use it whenever a REST surface needs a spec wired into +`openApiConf` — discovery (SKILL.md Step 1a / `scan-planning.md`) points here, and the +quality gate points here when it reports a `spec-not-wired` or coverage gap. + +## Contents +- [The one rule: an unverified spec is a hypothesis](#the-one-rule-an-unverified-spec-is-a-hypothesis) +- [Get an accurate spec (preference order)](#get-an-accurate-spec-preference-order) +- [Framework spec-generation matrix](#framework-spec-generation-matrix) +- [Suggesting the code/build change](#suggesting-the-codebuild-change) +- [Base path and context path](#base-path-and-context-path) +- [Mandatory: verify the spec resolves](#mandatory-verify-the-spec-resolves) +- [Deriving a spec by hand (no tooling)](#deriving-a-spec-by-hand-no-tooling) + +--- + +## The one rule: an unverified spec is a hypothesis + +A checked-in `openapi.yaml`, a spec you generated, and a spec the app serves are all +*claims* about the API until you prove they resolve against the running app. Never wire a +spec into `openApiConf` and scan without the verification in "Mandatory: verify the spec +resolves" below. A checked-in file is the least trustworthy source — it drifts from the code, +can be stale, hand-maintained, or (as seen in the wild) emptied by a commit — so treat its +presence as a starting hypothesis, not a done task. + +## Get an accurate spec (preference order) + +Prefer a spec the framework produces from the *running app* over any static artifact. Work +down this list; stop at the first that yields a spec that passes verification. + +1. **The running app already serves one.** Probe the common served endpoints once the app is + up (see the matrix for framework-specific paths): + ```bash + for p in /openapi.json /openapi.yaml /v3/api-docs /v3/api-docs.yaml \ + /swagger.json /swagger/v1/swagger.json /api-docs /q/openapi; do + curl -sf -o /dev/null -w "%{http_code} $p\n" "$APP_BASE_URL$p" + done + ``` + A 200 here is the best source — it reflects real routing, including base/context path. + Wire it live with `openApiConf.path: ` so the scan always reads the current + spec. +2. **The framework can generate/serve one with a small change, but it isn't enabled.** This is + the ideal outcome — see "Suggesting the code/build change". Add the tooling, rebuild, + then fetch as in step 1. +3. **Only a static spec is checked in.** Verify it's current (path count matches the route + grep from discovery; paths resolve — see verification). If it's stale, prefer regenerating + via step 1/2. If you must use it, wire `openApiConf.filePath: ` and verify. +4. **A spec is published *outside* the repo.** Mature projects often maintain an accurate spec + somewhere other than the app repo — check before giving up: a docs site (e.g. + `api-docs..org`, a ReDoc/Swagger-UI page), a dedicated `/api-docs` or + `-openapi` repo, a spec published as a release asset or package, or a link in the + README/docs. This is exactly Firefly III's situation — no spec in-repo, but a complete one + at `firefly-iii/api-docs`. Fetch it, pin the version to match the app, wire `filePath`, and + verify it resolves (an external spec can lag the running app). +5. **No spec anywhere and no tooling for this stack.** Derive one by hand from the route + inventory — see "Deriving a spec by hand". + +## Framework spec-generation matrix + +Examples, not an exhaustive list. The method generalizes: identify the framework from the +manifest, then determine its idiomatic OAS support. For anything not listed, **find out** — +`hawk config show`/docs won't help here, so search "`` openapi generation" or +check the framework's own docs before concluding none exists. + +| Stack | Idiomatic OAS tooling | Serves at | Notes | +|-------|----------------------|-----------|-------| +| FastAPI (Python) | built in | `/openapi.json` | Already served — no change needed; just probe. | +| NestJS (Node) | `@nestjs/swagger` | `/api-json` (configurable) | First-party. | +| Express / Fastify (Node) | `swagger-jsdoc` + `swagger-ui-express`; `@fastify/swagger` | configurable | JSDoc-annotation or plugin based. | +| Spring Boot (Java/Kotlin) | `springdoc-openapi-starter-webmvc-ui` | `/v3/api-docs` | Reflects `server.servlet.context-path` in its server URL. | +| Quarkus (Java) | `quarkus-smallrye-openapi` | `/q/openapi` | Often on by default in dev. | +| Django REST Framework | `drf-spectacular` | `/api/schema/` | `drf-yasg` is the older alternative. | +| Flask (Python) | `apiflask`, `flasgger`, `flask-smorest` | configurable | — | +| ASP.NET Core | `Swashbuckle.AspNetCore` or `NSwag` | `/swagger/v1/swagger.json` | Often scaffolded already. | +| Go | `swaggo/swag`, `danielgtaylor/huma` | configurable | swag generates from annotations. | +| Rails | `rswag` | configurable | Spec-from-request-specs. | +| Laravel (PHP) | `l5-swagger`, `scramble` | configurable | Scramble infers without annotations. | +| Rust | `utoipa`, `aide` | configurable | — | + +## Suggesting the code/build change + +When the framework can emit an accurate spec but doesn't yet (preference step 2), this is the +highest-value move: a served, framework-generated spec is accurate by construction and stays +current as the code changes. Explore and suggest a **concrete** change — not "consider adding +OpenAPI support": + +- Name the exact dependency and where it goes (the build file + coordinate/version). +- Show the minimal config/annotation to expose the endpoint, and state the served path. +- State the DAST value it unlocks ("wires N routes HawkScan otherwise can't see; replaces a + stale/hand-derived spec that would 404"). +- Scope it dev/test-only where the framework allows, so it needn't ship to production. + +Present this as a recommendation with the diff and let the user decide — the skill does not +apply unrequested code changes (see "Recommend code changes for gaps" in `scan-planning.md`). +Make the suggestion good enough that saying yes is a one-step change, then rebuild and fetch +the served spec. + +## Base path and context path + +The most common accuracy failure is a base-path mismatch. HawkScan builds each request as the +configured host **plus the spec's operation path**; a spec whose paths omit the app's base or +context path 404s on every operation. Detect the app's base path during discovery and make +the effective `host + spec-path` land on real routes: + +| Framework | Where the base/context path lives | +|-----------|-----------------------------------| +| Spring MVC | `server.servlet.context-path` (application.yml/properties) | +| Spring WebFlux | `spring.webflux.base-path` | +| Express | the mount prefix in `app.use('/api/v1', router)` | +| Django | the URL prefix in the root `urls.py` `include()` | +| ASP.NET | `UsePathBase(...)` / route prefix | +| Rails | `scope`/`namespace` in `config/routes.rb` | + +Two ways to make it resolve — pick one and be consistent: +- Put the base path in `app.host` (e.g. `http://localhost:5000/api/v1`) and keep spec paths, + `loginPath`, and `testPath` **relative**; or +- Keep `app.host` at the origin and ensure the spec's operation paths already include the base + path (a framework-served spec usually does). + +Do **not** split the difference — e.g. stripping the context path from `app.host` and +hardcoding it only into `loginPath`/`testPath`. That fixes auth but leaves every +`openApiConf` path unprefixed, which is exactly how a scan ends up 71% 404s while login still +works. If `hawk validate auth` misbehaves with a path component in `host`, resolve it without +desyncing the spec's base path — verify the outcome with the resolve check below. + +## Mandatory: verify the spec resolves + +Before the first real scan, prove the wired spec produces real requests — regardless of +source. This is the check that catches base-path mismatch, stale specs, and wrong hosts. + +1. **Count sanity.** Spec operation count should be in the same ballpark as the route grep + from discovery. A spec with far fewer/more paths than the code has is suspect. +2. **Resolve check.** Take 2–3 real operation paths **from the spec you're wiring** and + request each **as the scanner will build it** — effective `host` + operation path — against + the running app, both as the spec declares it and with the app's detected base/context path + prepended. Substitute your own values for the placeholders; do not run them literally: + ```bash + # = an operation path copied from the spec (e.g. /authors) + # = the app's detected base/context path, or empty if none (e.g. /api/v1) + curl -s -o /dev/null -w "%{http_code} %{url_effective}\n" "$SCAN_HOST" + curl -s -o /dev/null -w "%{http_code} %{url_effective}\n" "$SCAN_HOST" + ``` + If the spec-derived URL 404s while the base/context-path-prefixed variant returns a real + status (200/401/403), the base path is wrong — fix `app.host` or the spec before scanning. + A raw container 404 (e.g. Tomcat's default page) rather than an application response is a + strong tell that the request never reached the app's routing. +3. After the scan, confirm with `hawk op scan uris `: the scanned URIs should be the + spec's real routes, not un-prefixed shadows of them. + +## Deriving a spec by hand (no tooling) + +When the stack has no OAS generator (or adding one is out of scope), build a minimal spec from +the route inventory discovery already produced. Keep it small and correct rather than +complete: + +- One `paths` entry per real route; include path parameters as `{id}` style templates. +- Set `servers[].url` to the real base (including context path) — but remember HawkScan keys + off `host + operation path`, so still run the resolve check; don't rely on `servers` alone. +- Include request bodies/params only where they matter for reaching the endpoint. +- Wire via `openApiConf.filePath` and verify as above. + +A hand-derived spec is a maintenance liability — always prefer suggesting the framework +tooling (step 2) so the spec regenerates from code instead of drifting. + +**A hand-derived spec still beats `hawk.spider.seedPaths`.** seedPaths only lists URLs to +visit — no methods, no request bodies, no path/query parameters — so a seedPaths-only scan +can GET a handful of collection endpoints but never exercises POST/PUT/PATCH, parameterized +routes, or request-body validation. That is why a spec-less scan collapses to a fraction of +the real surface — on a typical REST API that can be well under 10% of the routes reached. +Only use seedPaths as a supplement to a spec, or as a genuine last resort when even a +hand-derived spec is impossible. diff --git a/plugins/hawkscan/skills/hawkscan/references/platform-model.md b/plugins/hawkscan/skills/hawkscan/references/platform-model.md index 95697597..23bc81f3 100644 --- a/plugins/hawkscan/skills/hawkscan/references/platform-model.md +++ b/plugins/hawkscan/skills/hawkscan/references/platform-model.md @@ -401,6 +401,6 @@ lookups. Relevant commands and their documentation: |-------------------------------|------------------------------------------------------------| | List apps | `hawk op app list --format json` | | List envs for an app | `hawk op env list --app --format json` | -| Get scan findings with triage | `hawk op scan get --app --detail full --format json` | +| Get scan findings with triage | `hawk op scan get --app --detail full --format json` | These reads require the combined `hawk` CLI; the `api` skill covers setup (`hawk init --browser` or the `HAWK_API_KEY` env var). diff --git a/plugins/hawkscan/skills/hawkscan/references/scan-planning.md b/plugins/hawkscan/skills/hawkscan/references/scan-planning.md new file mode 100644 index 00000000..4579cd46 --- /dev/null +++ b/plugins/hawkscan/skills/hawkscan/references/scan-planning.md @@ -0,0 +1,211 @@ +# Scan Planning Reference (Discovery) + +Discovery is how the hawkscan skill decides *what* to scan and *how* before it writes or +edits a single `stackhawk.yml`. Run it in three situations: + +- **First scan of a repo.** No `stackhawk.yml` exists yet (SKILL.md Step 1a / Step 2a) — + discovery produces the surface inventory that Step 2a's config generation consumes. +- **The quality gate looped back with a structural gap.** A post-scan coverage check found + routes the scan never reached, or a surface it didn't know existed. Re-run discovery + against that specific gap rather than starting over. +- **The user asks to re-plan the scan.** New API surface added, a monorepo grew a service, + an old assumption no longer holds. + +Discovery is stateless: there is no plan file. `stackhawk.yml` is the only durable artifact, +and anything the user tells you that isn't a config field belongs as a comment in +`stackhawk.yml` or in the repo's `CLAUDE.md`/`AGENTS.md` — see "Ask, don't guess" below. + +Work code-first, interactively: explore before asking, ask the user before guessing, and +never stall waiting for information you could get by reading three more files or asking one +direct question. + +## Contents +- [Discover the app's API surfaces](#discover-the-apps-api-surfaces) +- [Recommend code changes for gaps](#recommend-code-changes-for-gaps) +- [Ask, don't guess](#ask-dont-guess) +- [Configure per surface](#configure-per-surface) +- [Cross-checking](#cross-checking) + +--- + +## Discover the app's API surfaces + +### Pass 1 — read what the repo already says + +Most repos document themselves. Read these, in priority order, before exploring code — +treat what they say as authoritative and don't rediscover by grepping what a doc already +states: + +| Source | Typically documents | +|--------|----------------------| +| `AGENTS.md` | run/build/test commands, layout, conventions | +| `CLAUDE.md` | same, written for agents — often the richest source | +| `GEMINI.md`, `.github/copilot-instructions.md` | agent run/build guidance | +| `.cursor/rules/*` | project conventions and setup steps | +| `README*` | quickstart, run command, default host/port | +| `CONTRIBUTING*` | local dev setup, how to run services and tests | +| `docs/` setup / quickstart / architecture pages | deeper API and service-boundary detail | + +Harvest: run command + host/port, which API surfaces exist and where their source lives +(useful in a monorepo), and any documented spec location or dev/test credential. Read these +from the docs and config — discovery never starts the app to learn them. A +`docker-compose.yml` port mapping or an `.env`/`.env.example` `APP_URL` gives you the host +and port without running anything; starting the app is a scan-time step, not a discovery one. + +### Pass 2 — explore code per surface type + +For anything the docs didn't answer, find the surfaces the way a developer would: read +manifests, entry points, and route definitions. Identify **every** distinct API surface in +the repo — a service can have more than one (e.g. a REST API plus an internal gRPC service). +For each surface found, record its type and derive a route inventory using the matching +technique. The commands below are **illustrative starting points**, not fixed incantations: +point them at the repo's real source root and its actual spec/schema filenames — don't run +`src/`, `openapi.yaml`, or `schema.graphql` literally — and adjust the pattern to what the +code actually uses. + +| Surface | Detection | Route inventory / derivation | +|---------|-----------|-------------------------------| +| REST — Spring | `@RestController`, `@Controller` classes | `grep -rEo '@(Get\|Post\|Put\|Delete\|Patch\|Request)Mapping' src/` — count matches | +| REST — Express/Node | `app.get/post/put/delete`, `router.get/post/...` | `grep -rEo "\.(get\|post\|put\|delete\|patch)\(" src/` (or route files under `routes/`) | +| REST — Rails | `config/routes.rb` | count route-verb lines: `grep -cE '^\s*(get\|post\|put\|patch\|delete)\s' config/routes.rb` | +| REST — Django | `urls.py` | count `path(`/`re_path(` entries: `grep -rc 'path(' **/urls.py` | +| REST — Go (gin/mux/chi) | `router.GET(`, `r.HandleFunc(`, `mux.NewRouter()` | `grep -rEo '\.(GET\|POST\|PUT\|DELETE\|PATCH)\(' .` | +| REST — .NET | `[HttpGet]`, `[HttpPost]`, `[Route(...)]` attributes | `grep -rEo '\[Http(Get\|Post\|Put\|Delete\|Patch)\]' .` | +| REST — OpenAPI spec | checked-in `openapi.{json,yaml,yml}`, `swagger*.{json,yaml}`; or served at `/v3/api-docs`, `/swagger.json`, `/openapi.json` once the app is running | count `paths:` entries: `yq '.paths | keys | length' openapi.yaml` (or the JSON equivalent). **A found spec is a hypothesis, not a win — see `openapi-specs.md` for getting an *accurate* one.** | +| GraphQL | schema files (`*.graphql`, `*.gql`); server libs in manifests (`graphql`, `apollo-server`, `graphql-yoga`, `graphene`, etc.); a `/graphql` endpoint | count root fields: `grep -A50 'type Query' schema.graphql \| grep -cE '^\s+\w+'` (repeat for `type Mutation`) | +| gRPC | `.proto` files; `grpc`/`protobuf` deps in the manifest | count `rpc` methods per service: `grep -c '^\s*rpc ' *.proto` | +| JSON-RPC / SOAP | fixed-path JSON-RPC dispatcher; WSDL files (`*.wsdl`), SOAP endpoint patterns | count `/api-docs` repo, a + release asset) — mature OSS projects often do. The full framework-generic procedure — probe + for a served spec, the per-stack tooling matrix, how to write a concrete suggestion, the + external-spec check, and the base-path/resolve verification — is in `openapi-specs.md`. Do + that rather than wiring a stale checked-in file, or falling back to `seedPaths`, and hoping. +- **No test credential exists.** Recommend a test-only token issuer or seed script (a + dev-only endpoint or fixture that mints a valid session/JWT) gated behind an environment + check so it never ships to production. +- **GraphQL introspection disabled in the target environment.** Recommend enabling + introspection for the scan environment only, or checking in the schema file if it isn't + already. + +These are recommendations only. Never apply an unrequested code change to close a coverage +gap — present the recommendation, explain the DAST value it unlocks, and let the user decide +whether to make the change. + +## Ask, don't guess + +Code exploration sometimes can't account for the whole URL space: + +- A **monorepo slice** — the checked-out repo is one service among several, and its API is + fronted by a gateway or composed with other services not present locally. +- A **gateway fronting upstreams** — the visible code is a thin proxy; the real route + surface lives in services this repo doesn't contain. +- **Dynamic routing** — routes built from a database, a plugin registry, or a config file + not checked into source control. + +When code can't answer a surface question, ask the user directly — for the missing service's +repo, the gateway's route table, or the runtime detail that would resolve it. Don't guess a +plausible-looking answer and don't stall the whole discovery pass waiting on one open +question if other surfaces are ready to configure. + +Once the user answers, that answer has nowhere else to live — there is no plan file to record +it in. Persist it as a comment next to the relevant `stackhawk.yml` block (e.g. `# gateway +also fronts the billing-service repo at github.com/org/billing — see route table in +#eng-platform`) or as a line in the repo's `CLAUDE.md`/`AGENTS.md` if it's a fact about the +repo rather than about one scan. Either way, the goal is the same: the next agent to run +discovery on this repo should never have to ask the same question twice. + +## Configure per surface + +Write **one `stackhawk.yml` config per API surface** (REST, GraphQL, gRPC, SOAP, SPA, ...). +Order them by DAST value, highest first: the primary or backing API surface goes first, a +SPA frontend goes last (per the SPA guidance above, it typically finds only header/CSP +issues). This order is not cosmetic — state it explicitly in the summary you show the user, +so the rationale for what scans first is visible, not implicit. + +**Naming and platform mapping for multi-surface repos:** a single-surface repo uses plain +`stackhawk.yml`. With multiple surfaces, name each config `stackhawk-.yml` (e.g. +`stackhawk-rest.yml`, `stackhawk-graphql.yml`) and run each scan as +`hawk scan stackhawk-.yml`. All surfaces share one `applicationId`; give each +surface its own environment (e.g. `dev-rest`, `dev-graphql`). Findings compare scan-to-scan +within an environment, so per-surface envs keep each surface's history coherent instead of +interleaving a REST scan's baseline with a GraphQL scan's. + +For each surface: + +1. Wire the spec you found or confirmed in discovery: `openApiConf` for REST, + `graphqlConf` for GraphQL, `grpcConf` + the `.proto` for gRPC, `soapConf` + the WSDL for + SOAP. Use `hawk config show --text` for the exact field shape — don't + hand-write the block from memory. For REST, the spec must be *verified-accurate* before + it's wired: get it and prove it resolves per `openapi-specs.md` (a wrong spec silently + 404s every operation), and make sure `app.host` and the spec agree on the base/context + path. +2. Configure auth per Phase 1c in SKILL.md — one auth block per surface if surfaces have + different login mechanisms (common when a gateway and an upstream service use different + credential types). +3. Validate every surface's config before treating discovery as complete: + ```bash + hawk validate config + hawk validate api # when a spec is wired + hawk validate auth # when authentication: is present + ``` + +**Before the first scan of a fresh repo, present a one-screen summary and get the user's +confirmation.** The summary should state, per surface: the surface type, which config file +covers it, the scan order and why, the auth approach, the expected route count with how it +was derived (the same grep/spec-count evidence gathered above), and — for any REST surface +with a wired spec — the result of the mandatory resolve-check (e.g. "3/3 sample spec paths +resolved against host"; see `openapi-specs.md`). The resolve-check is the one thing that +catches a base-path mismatch before it wastes a whole scan, so its result belongs in the +summary rather than left to chance. Confirming this up front is cheap; discovering the scan +order, auth assumption, or a spec that 404s every path was wrong after a scan has already +run is not. + +## Cross-checking + +Don't configure `openApiConf`, `graphqlConf`, `grpcConf`, `soapConf`, or any other +spec-wiring block from memory alone — these shapes change across hawk releases. Two sources, +in order of preference: + +1. **`hawk config show
--text`** — the canonical, version-matched recipe for the + installed CLI. This is always correct for the hawk version actually in use and should be + the default source for field syntax. +2. **docs.stackhawk.com**, via WebFetch or WebSearch when available — useful for narrative + guidance `hawk config show` doesn't carry (why a shape works the way it does, worked + examples for less common combinations). Prefer it over recalling the shape from training + data whenever there's any doubt. + +If the two disagree, trust `hawk config show` — it reflects the CLI version actually +installed, not a documentation snapshot that may be ahead of or behind it. diff --git a/plugins/hawkscan/skills/hawkscan/references/scan-quality.md b/plugins/hawkscan/skills/hawkscan/references/scan-quality.md new file mode 100644 index 00000000..24c17118 --- /dev/null +++ b/plugins/hawkscan/skills/hawkscan/references/scan-quality.md @@ -0,0 +1,186 @@ +# Scan Quality Reference + +Run this gate after **every** scan — full scans and rescans alike — before findings are +parsed into fix tasks (SKILL.md calls it Step 4.5, sitting between Step 3's scan run and +Step 4's fix-task generation). It is a feedback loop into config tuning, **not a governor**: +gate state never blocks a finding from being reported and fixed. A thin scan that only +reached a fraction of the surface can still have found something real, and that finding +gets fixed now, regardless of what the gate says. The gate's job is narrower: notice when a +scan almost certainly missed API surface, and drive a bounded, additive config fix so the +*next* scan reaches more of it. + +The gate derives its expectation fresh every time it runs. There is no persisted plan or +prior-run state to compare against — recompute the expected surface from the repo and the +effective config each time, the same way discovery would if run again right now. + +**Multi-config execution.** When a repo has more than one `stackhawk.yml` surface (discovery +writes one config per surface, ordered by DAST value — primary or backing API first, a SPA +frontend last), scan and gate **sequentially**, one config at a time, in that same +value order. Gate immediately after each config's scan rather than batching every scan +first and gating afterward — an auth-wall caught and fixed on config 1 saves the identical +wasted scan on config 2. The iteration caps described below (2 interactive / 1 autonomous) +apply **per config**, not to the run as a whole. The autonomous post-code-change loop scans +only the config(s) whose surface the change touched — that scan must still cover that +surface's entire scope (the fix-ALL-findings rule from Step 4 is unchanged) — defaulting to +the primary surface's config when it's unclear which surface a change touched. A full sweep +across every config is an interactive operation only; never trigger one autonomously. + +## Contents +- [Derive the expectation (fresh, every scan)](#derive-the-expectation-fresh-every-scan) +- [The five checks](#the-five-checks) +- [On gaps: iterate the config](#on-gaps-iterate-the-config) +- [Reporting rules](#reporting-rules) +- [Degradation](#degradation) + +--- + +## Derive the expectation (fresh, every scan) + +Before you can tell whether a scan under-covered its target, you need something to compare +it against. Build that expectation the same way every time, right after the scan finishes: + +- **Spec wired** (`openApiConf`, `graphqlConf`, `grpcConf`, `soapConf`): the expectation is + the paths or operations the spec declares. `hawk op scan uris --format json` is + config-aware — it reads the effective scan config and automatically fetches + graphql-operations or jsonrpc-methods when `graphqlConf` or `jsonRpcConf` is present in + that config (force either with `--graphql` / `--jsonrpc` if auto-detection misses it). + That gives you the real scanned-URI set to diff against the spec's declared set. +- **No spec wired**: re-derive the expected route inventory using the same per-framework + route greps as the discovery reference (`scan-planning.md`) — the same detection + heuristics, the same counting technique, run fresh against the current code, not carried + over from whatever discovery counted last time. Code changes between scans; the + expectation should too. +- **Spec-less apps that keep coming up short**: if the grepped expectation and the scanned + URIs repeatedly diverge because there's nothing authoritative to wire, that's a signal to + recommend generating a spec — the same code-change recommendations discovery makes + (`scan-planning.md`'s "Recommend code changes for gaps" section: `springdoc-openapi` for + Spring, `swagger-jsdoc`/`@nestjs/swagger` for Node, `drf-yasg`/`drf-spectacular` for + Django REST Framework, `swashbuckle` for .NET). Surface the recommendation; don't apply it + unrequested. +- `hawk op scan uris` and `hawk op scan config` ship with the combined `hawk` CLI the + preflight already requires. If either subcommand errors or isn't recognized on the + installed CLI, fall back per the Degradation section below — detect it by the command + failing, don't ask the user which version they have. + +Because the no-spec expectation is re-derived with the **same** per-framework greps discovery +uses, the gate inherits discovery's blind spots: a surface neither discovery nor this +re-derivation detects (a second gRPC service, a dynamically-mounted router) won't be flagged +as unscanned — the gate can't miss-diff against a surface it never counted. This is why the +user-confirmed discovery summary (`scan-planning.md`) is the real backstop for surface +completeness, not the gate alone. + +## The five checks + +Run all five after every scan. Each has its own command, pass condition, and a stable +reason identifier used when reporting a gap — this gate's own reporting (Step 4.5 in +SKILL.md) uses these identifiers directly, so keep them exactly as written here across any +edits to this file: `coverage-gap`, `spec-not-wired`, `surface-unscanned`, +`auth-validate-failed`, `auth-wall`, `all-4xx`, `base-path-mismatch`, `env-unreachable`. + +| # | Check | What it measures | Command(s) | Pass condition | Reason identifier(s) | +|---|-------|-------------------|------------|-----------------|------------------------| +| 1 | Coverage | Scanned URIs vs. the expectation derived above | `hawk op scan uris --format json` diffed against the expectation list | **Never fails the gate.** Always computed and reported as evidence, e.g. "14 of 41 planned routes untouched," with the untouched routes listed | `coverage-gap` (evidence only) | +| 1b | Base-path resolve | Whether spec-derived paths actually resolve against the app, or systematically 404 because the config and spec disagree on the base/context path | In the scanned-URI list, look for the same route appearing both prefixed and un-prefixed (e.g. `/api/v1/authors` **and** bare `/authors`), or a cluster of spec paths all 404 while a prefixed variant succeeds; confirm with a curl of `host + spec-path` vs the prefixed variant | No systematic base-path 404 pattern — spec paths resolve to real routes. This **does** fail the gate (structural). Fix per `openapi-specs.md` "Base path and context path" | `base-path-mismatch` | +| 2 | Auth | Live auth validation before the scan; auth-wall signals during it | `hawk validate auth .yml` pre-scan (against the config being gated — repos with multiple per-surface configs validate each one); `hawk op scan metrics ` for auth-wall flags | `validate auth` exited 0, and no auth-wall flag on any surface that has `authentication:` configured | `auth-validate-failed`, `auth-wall` | +| 3 | Surface-completeness | Every surface discovery found has a config, that config actually ran, and any spec found for it is wired into that config | `hawk op scan config ` (effective config the scan ran with); fall back to reading the local `.yml` files | Every discovered surface maps to a config that exists, ran this scan, and has its spec wired if one exists | `surface-unscanned`, `spec-not-wired` | +| 4 | Health | Connection-failure spikes, timeout streaks, app unreachable mid-scan; all-4xx response patterns on a configured surface | `hawk op scan metrics ` for flags; `hawk op scan get ` for `url_count` and status summary | No connection-failure spike, no timeout streak, no mid-scan unreachability, and no configured surface returning all 4xx | `env-unreachable`, `all-4xx` | + +**Signal classes.** Every non-evidence gap above is either config-class or environment-class +— the class decides what you're allowed to do about it: + +- **Config-class** — `spec-not-wired`, `surface-unscanned`, `auth-validate-failed` (the fix + lives in the config's `authentication:` block), `auth-wall` while the app is demonstrably + up, `all-4xx` on a configured surface (this is usually auth-shaped: the surface is + reachable but every request is being rejected), and `base-path-mismatch` (the fix is + aligning `app.host` and the spec's base/context path — `openapi-specs.md`). These route + into the normal bounded config iteration described below. +- **Environment-class** — `env-unreachable`, covering connection-failure spikes, timeout + streaks, and the app going unreachable mid-scan. **Never touch the config for these.** + Verify the app is up using the same playbook as SKILL.md's Step 1c / Step 6 exit-1 + diagnosis, restart or wait if needed, then retry the scan once — that retry does not + consume a config-iteration. If the same environment-class signal recurs on the retry, + report it as an environment problem and stop; do not keep retrying and do not reach for a + config edit to work around it. + +Performance signals — slow paths, rate-limited endpoints, a scan that ran long — are not a +gate concern at all. They route to the `optimize` skill's existing suggestion path (SKILL.md +Step 4, surfaced once per session for full scans ≥ 20 minutes), never to this gate. + +## On gaps: iterate the config + +When one or more config-class gaps are open, loop back to SKILL.md's Step 2b (tune existing +`stackhawk.yml`) with this shape: + +- **Batch, then rescan once.** Collect every gate-named fix from this scan's gap set — wire + the missing spec, enable `hawk.spider.ajax`, fix the `authentication:` block, add a config + for an unscanned surface — into a single edit pass. Rescan once after applying all of + them together. Don't rescan after each individual fix; that wastes scans the gate exists + to save. +- **Only iterate when the config actually changed.** If a reported gap has no config-level + fix — a monorepo slice with no visibility into the missing service, dynamic routing that + needs a code change, a spec that genuinely doesn't exist yet — report it and move on. + Don't burn an iteration re-scanning a config you didn't touch. +- **Iteration cap: 2 interactive, 1 autonomous — per config, and counted in rescans, not + edits.** A batch of five config edits followed by one rescan is one iteration. This + matches the Autonomous Security Loop's existing guard rail of at most one fix-rescan cycle + when running unattended. +- **At the cap, stop.** Report the remaining structural gaps by reason identifier, the + coverage route-diff evidence (the untouched-route list from check 1), and concrete next + steps — including any code-change recommendations discovery would make for surfaces that + can't be closed by config alone. Never claim the scan is done, or the app is secure, while + gaps remain open at the cap. +- **A severe coverage gap with all four structural checks green** is not a config-iteration + target — it means the scan reached everything it was told to reach and still found little. + Route it to SKILL.md's Step 6 empty-data handling instead: suggest the + `stackhawk-data-seed` skill rather than continuing to edit the scan config. + +**Additive-only.** Every fix applied during iteration must add coverage: wire a spec, turn +on the Ajax Spider, correct an auth recipe, add a missing surface's config. Iteration must +**never** widen `excludePaths`, raise `failureThreshold`, add `excludePlugins`, or otherwise +weaken the scan so it happens to pass the gate. If the only way to close a gap is to reduce +what gets scanned, that is not a pass — report the honest gap instead of manufacturing a +clean one. + +Environment-class gaps never enter this loop at all — see the Signal classes note above: +verify the app, retry once for free, and stop if it recurs. + +## Reporting rules + +- **Findings are always reported and always fixable**, independent of gate state. The gate + never withholds, delays, or downgrades a finding — a scan that under-covered the surface + can still have found a real, exploitable bug, and that bug gets fixed on this pass. +- **Never say the scan is "done and secure"** (or any equivalent framing) while a + config-class or environment-class gap is still open. Say plainly what was scanned, what + wasn't, and why — the reason identifiers exist so this statement is specific, not vague. +- **The autonomous post-code-change loop reports gate gaps rather than treating exit 0 as + success.** It runs this gate after its scan, and if gaps remain once the autonomous + iteration cap (1) is reached, it reports them explicitly — reason identifiers, coverage + route-diff evidence, and next steps — instead of accepting a clean exit code at face value. +- **Performance-class signals are never gate gaps.** If health metrics point at slowness + rather than unreachability or all-4xx, that's an `optimize` suggestion, not a quality-gate + reason identifier. +- Keep every reported gap tied to one of the eight reason identifiers above so this gate's + own reporting stays specific rather than parsing prose — the identifiers must stay stable + across edits to this file. + +## Degradation + +`hawk op scan uris` and `hawk op scan config` ship with the combined `hawk` CLI. If either +subcommand errors or isn't recognized on the installed CLI, degrade silently and keep going — +detect it by the command failing, don't ask the user to check their version: + +- **Coverage (check 1)** degrades from a full scanned-vs-expected route diff to a single + before/after number: `url_count` from `hawk op scan get `. You lose the + untouched-route list; still report the number as evidence, just note it's coarser. +- **Surface-completeness (check 3)** degrades from reading the effective scan config via + `hawk op scan config` to local signals: read the per-surface `.yml` files directly to + confirm a surface's config exists and its spec block is present, and use the + `hawk validate config` output for each file to confirm the config is structurally sound. +- **Auth and health (checks 2 and 4)** are unaffected — `hawk op scan metrics` and + `hawk validate auth` remain the primary signal either way. + +`hawk op scan stats` does not exist yet (it's a follow-up ticket) but will eventually add +measured, per-path engine statistics rather than the grep- and `url_count`-based estimates +this reference relies on today. When it ships, prefer its measured signal over the inferred +approach described here. Until then, its absence is expected — degrade silently and don't +block or warn on it being missing. diff --git a/plugins/hawkscan/skills/hawkscan/references/tech-flags.md b/plugins/hawkscan/skills/hawkscan/references/tech-flags.md index 8fd28e63..5a7db52c 100644 --- a/plugins/hawkscan/skills/hawkscan/references/tech-flags.md +++ b/plugins/hawkscan/skills/hawkscan/references/tech-flags.md @@ -43,7 +43,7 @@ Flag names are **dot-namespaced** (e.g., `Language.Java.Spring`), **case-sensiti hawk op app tech-flags get --app --format json ``` -`--app ` accepts the application name. The `hawk op` commands also accept `--app-id ` if you prefer to use the application ID directly. +`--app ` accepts either the application name or its UUID — pass whichever you have. Returns a JSON object with all available flags and their current true/false state: diff --git a/plugins/optimize/.claude-plugin/plugin.json b/plugins/optimize/.claude-plugin/plugin.json index 0836d856..ead71432 100644 --- a/plugins/optimize/.claude-plugin/plugin.json +++ b/plugins/optimize/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "stackhawk-optimize", - "version": "2.2.0", + "version": "2.4.0", "description": "Analyze a codebase and produce an optimal HawkScan setup (tech flags, scan-policy plugin selection, stackhawk.yml corrections), apply it as a non-destructive trial, run one trial scan, and promote or discard.", "author": { "name": "StackHawk", diff --git a/plugins/optimize/.codex-plugin/plugin.json b/plugins/optimize/.codex-plugin/plugin.json index 75bbf996..e7cc0148 100644 --- a/plugins/optimize/.codex-plugin/plugin.json +++ b/plugins/optimize/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "stackhawk-optimize", - "version": "2.2.0", + "version": "2.4.0", "description": "Analyze a codebase and produce an optimal HawkScan setup (tech flags, scan-policy plugin selection, stackhawk.yml corrections), apply it as a non-destructive trial, run one trial scan, and promote or discard.", "author": { "name": "StackHawk", diff --git a/plugins/optimize/skills/optimize/SKILL.md b/plugins/optimize/skills/optimize/SKILL.md index 175cf8c7..22e0556e 100644 --- a/plugins/optimize/skills/optimize/SKILL.md +++ b/plugins/optimize/skills/optimize/SKILL.md @@ -1,6 +1,6 @@ --- name: optimize -version: 2.2.0 +version: 2.4.0 description: > Analyze a codebase and produce an optimal HawkScan setup — tech flags, scan-policy plugin selection, and stackhawk.yml corrections — then apply it as a non-destructive diff --git a/plugins/optimize/skills/optimize/references/cli-contract.md b/plugins/optimize/skills/optimize/references/cli-contract.md index 17751924..9f92115e 100644 --- a/plugins/optimize/skills/optimize/references/cli-contract.md +++ b/plugins/optimize/skills/optimize/references/cli-contract.md @@ -6,7 +6,7 @@ | Fetch a base policy as JSON | `hawk op policy get --name ` | | Create/upsert a trial or permanent policy | `hawk op policy create --file --name [--display-name ] [--dry-run]` | | Delete a policy | `hawk op policy delete --name --yes [--dry-run]` | -| (Optional) set app default | `hawk op policy assign --app |--app-id --name [--dry-run]` | +| (Optional) set app default | `hawk op policy assign --app --name [--dry-run]` | | Read canonical tech flags | `hawk op app tech-flags get --app --format json` | | Get per-path scan metrics + signal flags | `hawk op scan metrics [--sort heaviest\|slowest\|erroring\|most-requested] [--top N] [--method ] [--operations] --format json` | diff --git a/plugins/stackhawk-data-seed/.claude-plugin/plugin.json b/plugins/stackhawk-data-seed/.claude-plugin/plugin.json index cd8afda4..b710b96b 100644 --- a/plugins/stackhawk-data-seed/.claude-plugin/plugin.json +++ b/plugins/stackhawk-data-seed/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "stackhawk-data-seed", - "version": "2.2.0", + "version": "2.4.0", "description": "Read a target repo and emit checked-in seed artifacts so HawkScan has data to scan. Generates SQL / HTTP / gRPC / Mongo / shell steps across the target repo and any upstream services it depends on, plus a manifest and a credentials handoff file that hawkscan consumes.", "author": { "name": "StackHawk", diff --git a/plugins/stackhawk-data-seed/.codex-plugin/plugin.json b/plugins/stackhawk-data-seed/.codex-plugin/plugin.json index 5c761014..f8500265 100644 --- a/plugins/stackhawk-data-seed/.codex-plugin/plugin.json +++ b/plugins/stackhawk-data-seed/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "stackhawk-data-seed", - "version": "2.2.0", + "version": "2.4.0", "description": "Read a target repo and emit checked-in seed artifacts so HawkScan has data to scan. Generates SQL / HTTP / gRPC / Mongo / shell steps across the target repo and any upstream services it depends on, plus a manifest and a credentials handoff file that hawkscan consumes.", "author": { "name": "StackHawk", diff --git a/plugins/stackhawk-data-seed/skills/stackhawk-data-seed/SKILL.md b/plugins/stackhawk-data-seed/skills/stackhawk-data-seed/SKILL.md index 0ae24501..fce1406c 100644 --- a/plugins/stackhawk-data-seed/skills/stackhawk-data-seed/SKILL.md +++ b/plugins/stackhawk-data-seed/skills/stackhawk-data-seed/SKILL.md @@ -1,6 +1,6 @@ --- name: stackhawk-data-seed -version: 2.2.0 +version: 2.4.0 description: > Set up checked-in seed data so authenticated HawkScan can reach non-trivial paths. Drives the `hawk perch seed` preflight, designs the minimum seed diff --git a/plugins/wingman/.claude-plugin/plugin.json b/plugins/wingman/.claude-plugin/plugin.json index 9d0eb79e..c406b15a 100644 --- a/plugins/wingman/.claude-plugin/plugin.json +++ b/plugins/wingman/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "wingman", - "version": "2.2.0", + "version": "2.4.0", "description": "StackHawk umbrella plugin \u2014 installs the default StackHawk skill set (hawkscan, api, data-seed, optimize) in one command for the combined hawk CLI.", "author": { "name": "StackHawk", diff --git a/plugins/wingman/.codex-plugin/plugin.json b/plugins/wingman/.codex-plugin/plugin.json index 9d0eb79e..c406b15a 100644 --- a/plugins/wingman/.codex-plugin/plugin.json +++ b/plugins/wingman/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "wingman", - "version": "2.2.0", + "version": "2.4.0", "description": "StackHawk umbrella plugin \u2014 installs the default StackHawk skill set (hawkscan, api, data-seed, optimize) in one command for the combined hawk CLI.", "author": { "name": "StackHawk", diff --git a/scripts/generate-cursor-rules.sh b/scripts/generate-cursor-rules.sh index 60afcc67..23229617 100755 --- a/scripts/generate-cursor-rules.sh +++ b/scripts/generate-cursor-rules.sh @@ -25,10 +25,13 @@ MAPPINGS=( "hawkscan/skills/hawkscan/references/installation.md|stackhawk-hawkscan-install|HawkScan installation: Homebrew (macOS), manifest-driven binary download, package installers (.pkg/.msi), verification (hawk version). Self-contained binary — no separate Java install needed.||false" "hawkscan/skills/hawkscan/references/false-positives.md|stackhawk-hawkscan-false-positives|HawkScan false positives guide: identifying false positives, deciding fix vs suppress, excludePaths and failureThreshold config, excluding scan plugins, reporting accepted risk.||false" "hawkscan/skills/hawkscan/references/platform-model.md|stackhawk-hawkscan-model|HawkScan StackHawk platform model: Org → App → Env → Scan hierarchy, what goes in stackhawk.yml vs what doesn't, finding triage state machine (New/Accepted/False Positive/Reopened) and agent behavior per state, tags for commit traceability (_STACKHAWK_GIT_COMMIT_SHA, _STACKHAWK_GIT_BRANCH), technology flags (platform UI only — no API yet), hawk rescan --scan-id for fast fix verification in the agentic loop, create vs reuse decision trees for Apps and Envs.||false" + "hawkscan/skills/hawkscan/references/scan-planning.md|stackhawk-hawkscan-scan-planning|HawkScan scan-planning (discovery) reference: code-first discovery of an app's API surfaces before writing or editing stackhawk.yml — reading repo docs first, per-framework route inventory, recommending code changes for structural gaps, asking the user instead of guessing, and producing a per-surface stackhawk.yml config. Use for the first scan of a repo, a quality-gate structural gap, or a user-requested re-plan.||false" + "hawkscan/skills/hawkscan/references/openapi-specs.md|stackhawk-hawkscan-openapi-specs|HawkScan OpenAPI spec accuracy reference: framework-generic procedure for getting an accurate OpenAPI spec into openApiConf for a REST surface — prefer a spec the running app serves, suggest the code/build change that makes the framework generate one, verify it resolves against the app (base/context path), and derive one by hand only as a last resort. Use whenever wiring or fixing openApiConf, or when a scan's spec paths return 404s.||false" + "hawkscan/skills/hawkscan/references/scan-quality.md|stackhawk-hawkscan-scan-quality|HawkScan scan-quality (post-scan quality gate) reference: run after every scan and rescan, before findings become fix tasks. Derives a fresh coverage expectation each run (spec-wired or route-inventory), the four coverage checks, and a bounded, additive-only stackhawk.yml config iteration to close gaps — never blocks a real finding from being reported and fixed.||false" "api/skills/api/SKILL.md|stackhawk-api|Use when querying the StackHawk platform for security reporting, findings analysis, or app management. Triggers include \"stackhawk api\", \"security posture\", \"findings report\", \"show me findings\", \"untriaged findings\", \"which apps\", \"scan history\", \"security dashboard\", \"triage\", \"what needs attention\". Uses the combined hawk CLI (hawk op …) for all platform queries. Do NOT use for running scans (use the stackhawk-hawkscan rule for \"scan my app\", \"hawkscan\", \"stackhawk.yml\", \"DAST\") or for fixing/remediating code or vulnerabilities — this skill only reads and reports platform data.||false" "api/skills/api/references/hawk-op-shortcuts.md|stackhawk-api-hawk-op|hawk op shortcuts for the StackHawk API: user intent → single hawk op command mapping for scan drill-down, app/user/team/policy/repo/oas/config/secret/audit listing, hosted scan control, env management. All platform queries go through hawk op — no raw REST fallback.||false" "api/skills/api/references/reporting-recipes.md|stackhawk-api-recipes|StackHawk API reporting recipes: org security posture summary, app deep dive (scan -> alerts -> findings), stale apps detection, scan diff (what changed since last scan). Pre-built compositions using hawk op commands. Prefer hawk op shortcuts (stackhawk-api-hawk-op) for the deep-dive chain; use these recipes for the per-env untriaged posture view.||false" - "hawkscan-ci/skills/hawkscan-ci/SKILL.md|stackhawk-hawkscan-ci|Use when the user wants to configure HawkScan in their CI/CD pipeline — triggers on \"set up hawkscan in CI\", \"add stackhawk to my pipeline\", \"scan in CI\", \"configure github actions / gitlab / jenkins / circleci for hawkscan\", \"wire hawkscan into ci/cd\". Provider-agnostic: detects the CI system, edits the pipeline file in place to add a HawkScan job, prompts for HAWK_API_KEY storage (CI-native secrets store or external secrets manager), wires commit-SHA + branch traceability. Defers all local-scan concerns (stackhawk.yml, auth, findings, triage) to stackhawk-hawkscan.|.github/workflows/**,.gitlab-ci.yml,Jenkinsfile*,.circleci/config.yml,azure-pipelines*.yml,bitbucket-pipelines.yml,.buildkite/pipeline.yml,.travis.yml,appspec.yml,buildspec.yml|false" + "hawkscan-ci/skills/hawkscan-ci/SKILL.md|stackhawk-hawkscan-ci|Use when the user wants to WIRE HawkScan into a CI/CD pipeline config file — triggers on \"set up hawkscan in CI\", \"add stackhawk to my pipeline\", \"scan in CI\", \"configure github actions / gitlab / jenkins / circleci for hawkscan\", \"wire hawkscan into ci/cd\". Provider-agnostic: detects the CI system, edits the pipeline file in place to add a HawkScan job, prompts for HAWK_API_KEY storage (CI-native secrets store or external secrets manager), wires commit-SHA + branch traceability. Defers all local-scan concerns (stackhawk.yml, auth, findings, triage) to stackhawk-hawkscan. If no local stackhawk.yml exists yet, still trigger and route local-config work to stackhawk-hawkscan. Do NOT trigger for documentation-only changes, informational/research questions about CI, or running a local scan (that is stackhawk-hawkscan) — this skill only edits CI pipeline config.|.github/workflows/**,.gitlab-ci.yml,Jenkinsfile*,.circleci/config.yml,azure-pipelines*.yml,bitbucket-pipelines.yml,.buildkite/pipeline.yml,.travis.yml,appspec.yml,buildspec.yml|false" "hawkscan-ci/skills/hawkscan-ci/references/execution-shapes.md|stackhawk-hawkscan-ci-execution-shapes|HawkScan CI execution shapes reference: native action (stackhawk/hawkscan-action) vs Docker image (stackhawk/hawkscan) vs CLI download; pinning strategy; per-provider quick-reference recipes for GitHub Actions, GitLab CI, Jenkins, CircleCI, Azure Pipelines, Bitbucket, Buildkite, Travis, AWS CodeBuild; SARIF / Code Scanning notes.||false" "hawkscan-ci/skills/hawkscan-ci/references/app-startup-patterns.md|stackhawk-hawkscan-ci-app-startup|HawkScan CI app-startup patterns reference: Docker Compose + wait-for-it, GitLab services keyword, GitHub Actions services map, build-then-run (Node, JVM), run-a-built-image, scan-existing-host (ephemeral env / preview deployment), networking gotchas (--network host vs host.docker.internal), health-check endpoint conventions.||false" "hawkscan-ci/skills/hawkscan-ci/references/failure-semantics.md|stackhawk-hawkscan-ci-failure|HawkScan CI failure semantics reference: exit codes 0/1/42, failureThreshold tuning, block-on-42 vs warn-only vs scheduled-baseline modes, retry strategy (don't retry 42), caching strategy (cache CLI/image, never findings), scheduled-vs-PR-trigger tradeoffs.||false" diff --git a/tests/lib/test_cli_resilience.py b/tests/lib/test_cli_resilience.py index 406b3282..9d6fefda 100644 --- a/tests/lib/test_cli_resilience.py +++ b/tests/lib/test_cli_resilience.py @@ -36,7 +36,7 @@ def test_main_survives_launch_crash(monkeypatch, tmp_path): assert (out / "cell.json").exists() assert (out / "summary.json").exists() cell = json.loads((out / "cell.json").read_text()) - assert len(cell["results"]) == 24 # all hawkscan prompts graded + assert len(cell["results"]) == 26 # all hawkscan prompts graded # positive prompts failed with a harness note; at least one note mentions the crash assert any("command not found" in r.get("note", "") for r in cell["results"]) diff --git a/tests/lib/test_discovery_guard.py b/tests/lib/test_discovery_guard.py index d8afd70e..8ed24a2d 100644 --- a/tests/lib/test_discovery_guard.py +++ b/tests/lib/test_discovery_guard.py @@ -194,6 +194,20 @@ def test_denies_go_and_cargo_run(self): event = {"tool_name": "Bash", "tool_input": {"command": c}} self.assertEqual(run_guard(event).returncode, 2, c) + def test_denies_container_start(self): + for c in ("docker compose up -d", "docker-compose up", "docker run -p 8080:8080 img", + "podman run img", "nerdctl start c", "podman-compose up"): + event = {"tool_name": "Bash", "tool_input": {"command": c}} + self.assertEqual(run_guard(event).returncode, 2, c) + + def test_allows_container_readonly_inspection(self): + # Reading config/port mappings during discovery is read-only — must NOT be + # denied just because the word "docker" appears (regression: ENG-708). + for c in ("docker ps", "docker images", "docker compose config", + "docker compose config --services", "docker inspect firefly", "docker compose ls"): + event = {"tool_name": "Bash", "tool_input": {"command": c}} + self.assertEqual(run_guard(event).returncode, 0, c) + # --- extended mutation vectors --- def test_denies_sed_in_place(self): event = {"tool_name": "Bash", "tool_input": {"command": "sed -i 's/a/b/' f.txt"}} diff --git a/tests/lib/test_guard_clean_check.py b/tests/lib/test_guard_clean_check.py index 0a4bb8d8..9038609d 100644 --- a/tests/lib/test_guard_clean_check.py +++ b/tests/lib/test_guard_clean_check.py @@ -36,5 +36,40 @@ def test_pass_has_no_anti_found(self): self.assertIsNone(res[0].anti_found) +APP_START = "Denied (read-only discovery): starting the app/server/container is out of scope for read-only discovery" +WRITE = "Denied (read-only discovery): Write would modify files" + + +class GuardScopeClassification(unittest.TestCase): + """A blocked-and-recovered app-START attempt is a soft foul; a write/scan/egress + denial is a hard foul. `guard_scope` lets a check assert one flavor.""" + + def _check(self, scope): + return [{"id": "c", "type": "guard_clean", "severity": "blocking", "guard_scope": scope}] + + def test_hard_scope_ignores_app_start(self): + run = ParsedRun(output_text="done", guard_denials=[APP_START]) + self.assertTrue(run_process_checks(run, self._check("hard"))[0].passed) + + def test_hard_scope_fails_on_write(self): + run = ParsedRun(output_text="done", guard_denials=[WRITE]) + self.assertFalse(run_process_checks(run, self._check("hard"))[0].passed) + + def test_hard_scope_fails_on_mixed(self): + run = ParsedRun(output_text="done", guard_denials=[APP_START, WRITE]) + res = run_process_checks(run, self._check("hard"))[0] + self.assertFalse(res.passed) + self.assertIn("Write would modify files", res.anti_found) + self.assertNotIn("starting the app", res.anti_found) # app-start not surfaced under hard scope + + def test_app_start_scope_flags_app_start(self): + run = ParsedRun(output_text="done", guard_denials=[APP_START]) + self.assertFalse(run_process_checks(run, self._check("app_start"))[0].passed) + + def test_app_start_scope_ignores_write(self): + run = ParsedRun(output_text="done", guard_denials=[WRITE]) + self.assertTrue(run_process_checks(run, self._check("app_start"))[0].passed) + + if __name__ == "__main__": unittest.main()