diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f212a7..c50f648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,39 @@ ## Unreleased +- `safe install` gains a project mode: with no package named and a manifest in + the current directory (`package.json`, `requirements.txt`, `pyproject.toml`, + `Cargo.toml`, `composer.json`, `go.mod`), it bulk-audits what the project + already depends on instead of printing a usage error. `--project` forces the + mode. It runs `safe audit scan --deps-only --project .` and prints a + one-screen summary — audited manifests, package count, findings by severity, + verdict, and the top critical/high findings with package and advisory id. + The mode audits only and never runs a package manager. Critical findings + refuse with exit 104 even under `--yes` (`--yes` accepts WARNs only); a WARN + verdict prompts interactively and refuses with 102 in a non-interactive + shell; a clean verdict exits 0 quietly there. A scan that fails or leaves no + readable result document fails closed with exit 100 rather than reporting a + clean project. Decisions are recorded in the safe-run audit log as + `install:project`. + +- Lockfile-keyed scan cache for `--deps-only` scans: when the dependency + evidence hashes to a set already scanned within 24 hours, the recorded result + is replayed (`[safe audit] scan cache hit ()`) with the same verdict and + exit code instead of re-running osv-scanner, syft, and grype — a repeat scan + of an unchanged tree drops from tens of seconds to well under one. Entries + live in `~/.local/share/safe/audit/scan-cache/`, keyed on machine, target, + mode, and each evidence file's own hash, so touching any lockfile or manifest + forces a real scan. `safe audit scan --no-cache` bypasses the lookup and + `SAFE_AUDIT_SCAN_CACHE_TTL_SECONDS` overrides the TTL. The cache can only + skip work, never invent a verdict: missing, expired, corrupt, or + unrecognizable entries fall through to a real scan, evidence-free scans are + never cached, and source/`--full` scans are excluded entirely (they stage + arbitrary files the evidence hash does not capture). + +- The wrapper project-scan preflight now runs `safe audit scan --deps-only`, so + a bare `npm ci` / `pnpm install` in an unchanged tree costs a cache hit + instead of a full scanner run. + - Gate `mise` backend installs: `mise install`/`up`/`use`/`exec` previously installed registry packages (`npm:*`, `pipx:*`, `cargo:*`, `go:*` backends, lifecycle scripts included) with no audit at all. A `mise` PATH wrapper now diff --git a/bin/safe b/bin/safe index e135645..667ecb2 100755 --- a/bin/safe +++ b/bin/safe @@ -905,11 +905,18 @@ quote_command() { cmd_install_usage() { cat <<'EOF' usage: + safe install [--project] [--yes] safe install [-g|--global] [--yes] [...] safe install --manager npm|pnpm|yarn|bun|composer -g [--yes] [--trust-host] [...] safe install --host [--yes] [package-manager flags...] [...] safe install --sandbox [safe run install flags...] [...] +With no package and a manifest in the current directory (or with `--project`), +safe bulk-audits the project's dependency evidence and reports; it audits only +and never runs a package manager. Critical findings refuse with exit 104 even +under `--yes`; a WARN verdict needs `--yes` or an interactive accept (exit 102 +in a non-interactive shell). + Host installs are audited with `safe audit check` before the package manager runs. `--yes` skips the final confirmation prompt after a successful audit. `--trust-host` adds exact npm versions to safe run host-allow after install. @@ -1436,8 +1443,140 @@ safe_install_confirm_trust() { [[ "$reply" == "y" || "$reply" == "Y" || "$reply" == "yes" || "$reply" == "YES" ]] } +# Manifests that make a directory a project worth bulk-auditing. Order is the +# display order in the project-mode summary. +SAFE_INSTALL_PROJECT_MANIFESTS=(package.json requirements.txt pyproject.toml Cargo.toml composer.json go.mod) + +safe_install_project_manifests() { + local manifest + for manifest in "${SAFE_INSTALL_PROJECT_MANIFESTS[@]}"; do + [[ -f "$manifest" ]] && printf '%s\n' "$manifest" + done + return 0 +} + +safe_install_project_manifest_present() { + local manifest + for manifest in "${SAFE_INSTALL_PROJECT_MANIFESTS[@]}"; do + [[ -f "$manifest" ]] && return 0 + done + return 1 +} + +safe_install_project_confirm() { + local reply + + printf 'safe: accept these audit results for this project? [y/N] ' >&2 + if ! IFS= read -r reply &1)" + scan_rc=$? + set -e + + # The structured verdict comes from the result document the scan wrote; the + # rendered text is a human summary, not a contract. No readable result means + # we cannot claim the project is clean -> fail closed. + local result_path="" + result_path="$(printf '%s\n' "$scan_output" | sed -n 's/^Details: //p' | tail -n 1)" + if [[ "$scan_rc" -ne 0 || -z "$result_path" || ! -r "$result_path" ]]; then + printf '%s\n' "$scan_output" >&2 + safe_install_gate_log "project" "$(pwd)" "REFUSED_SCAN_UNREADABLE" "exit=${scan_rc}" + refuse 100 "BLOCKED project audit — safe audit scan produced no readable result (exit ${scan_rc}); ask the operator; details: safe explain" + fi + + local verdict packages critical high medium low + verdict="$(jq -r '.verdict // "UNKNOWN"' "$result_path" 2>/dev/null || printf 'UNKNOWN')" + packages="$(jq -r '.summary.packages_total // 0' "$result_path" 2>/dev/null || printf '0')" + critical="$(jq -r '.cve_scan.critical // 0' "$result_path" 2>/dev/null || printf '0')" + high="$(jq -r '.cve_scan.high // 0' "$result_path" 2>/dev/null || printf '0')" + medium="$(jq -r '.cve_scan.medium // 0' "$result_path" 2>/dev/null || printf '0')" + low="$(jq -r '.cve_scan.low // 0' "$result_path" 2>/dev/null || printf '0')" + + # An unparseable verdict is not a clean one. + if [[ ! "$critical" =~ ^[0-9]+$ || "$verdict" == "UNKNOWN" ]]; then + safe_install_gate_log "project" "$(pwd)" "REFUSED_SCAN_UNPARSEABLE" + refuse 100 "BLOCKED project audit — safe audit result at ${result_path} is not readable as a verdict; ask the operator; details: safe explain" + fi + + # One-screen summary. The scan's own render is not echoed: it says the same + # thing at ten times the length. + printf 'safe install: project audit — %s\n' "$(pwd)" + printf ' manifests: %s\n' "${manifests[*]}" + printf ' packages: %s\n' "$packages" + printf ' findings: critical %s, high %s, medium %s, low %s\n' "$critical" "$high" "$medium" "$low" + printf ' verdict: %s\n' "$verdict" + local cache_note + cache_note="$(printf '%s\n' "$scan_output" | grep -F '[safe audit] scan cache hit' | tail -n 1 || true)" + [[ -n "$cache_note" ]] && printf ' %s\n' "${cache_note#\[safe audit\] }" + if [[ "$critical" -gt 0 ]] || [[ "$high" -gt 0 ]]; then + printf ' top findings:\n' + jq -r ' + [.cve_scan.findings[]? | select((.severity // "") | ascii_downcase | test("critical|high"))] + | sort_by(if ((.severity // "") | ascii_downcase | test("critical")) then 0 else 1 end) + | .[:5][] + | " - [" + (.severity // "unknown") + "] " + (.package // "unknown") + "@" + (.version // "unknown") + " " + (.id // "unknown") + " (" + (.source // "?") + ")" + ' "$result_path" 2>/dev/null || true + fi + printf ' details: %s\n' "$result_path" + + # Critical findings are the project-scale equivalent of a BLOCK verdict: + # --yes accepts WARNs, never these. + if [[ "$critical" -gt 0 ]]; then + safe_install_gate_log "project" "$(pwd)" "REFUSED_BLOCK" "critical=${critical}" + refuse 104 "BLOCKED project audit — ${critical} critical finding(s) in this project's dependencies; to allow: operator review — safe audit scan --deps-only --project .; details: safe explain" + fi + + if [[ "$verdict" == "GO" ]]; then + safe_install_gate_log "project" "$(pwd)" "PROCEED" "verdict=GO" + if [[ "$yes" == "1" || ! -t 0 || ! -t 1 ]]; then + return 0 + fi + safe_install_project_confirm || { err "project audit not accepted"; return 1; } + return 0 + fi + + if [[ "$yes" == "1" ]]; then + safe_install_gate_log "project" "$(pwd)" "ACCEPTED_WARN" "verdict=${verdict}" + return 0 + fi + + if [[ ! -t 0 || ! -t 1 ]]; then + safe_install_gate_log "project" "$(pwd)" "REFUSED_WARN" "verdict=${verdict}" + refuse 102 "BLOCKED project audit — verdict ${verdict} needs operator review and this shell is non-interactive; to allow: ask the operator to re-run it interactively, or re-run with --yes after review; details: safe explain" + fi + + if safe_install_project_confirm; then + safe_install_gate_log "project" "$(pwd)" "ACCEPTED_WARN" "verdict=${verdict}" + return 0 + fi + + safe_install_gate_log "project" "$(pwd)" "DECLINED" "verdict=${verdict}" + err "project audit not accepted" + return 1 +} + cmd_install() { local host_mode=0 sandbox_mode=0 yes=0 + local project_mode=0 local trust_host=0 local manager=npm global_mode=0 local -a manager_args=() @@ -1479,6 +1618,10 @@ cmd_install() { yes=1 shift ;; + --project) + project_mode=1 + shift + ;; --trust-host) trust_host=1 host_mode=1 @@ -1505,6 +1648,21 @@ cmd_install() { esac done + # Project mode: no package named, but a manifest here. Bulk-audit what the + # project already depends on instead of dying with usage. --project forces + # it (and says so if there is no manifest). + if [[ "$sandbox_mode" == "0" ]]; then + local -a project_specs=() + mapfile -t project_specs < <(safe_install_audit_specs "$manager" ${manager_args[@]+"${manager_args[@]}"}) + if [[ "$project_mode" == "1" ]] || + { [[ "${#project_specs[@]}" -eq 0 ]] && safe_install_project_manifest_present; }; then + cmd_install_project "$yes" + return $? + fi + elif [[ "$project_mode" == "1" ]]; then + die "--project cannot be used with --sandbox" + fi + if [[ "$sandbox_mode" == "1" && "$host_mode" == "0" ]]; then exec "$SAFE_RUN_PATH" install "${manager_args[@]}" fi diff --git a/bin/safe-audit b/bin/safe-audit index 3eea5ff..a91ce48 100755 --- a/bin/safe-audit +++ b/bin/safe-audit @@ -16,6 +16,10 @@ TOOLS_FILE="$CONFIG_DIR/tools.json" RESULTS_DIR="$DATA_DIR/results" SBOM_DIR="$DATA_DIR/sbom" +SCAN_CACHE_DIR="$DATA_DIR/scan-cache" +# A dependency-evidence scan is deterministic in its inputs, so an unchanged +# lockfile set within the TTL may reuse the previous verdict. +SCAN_CACHE_TTL_SECONDS="${SAFE_AUDIT_SCAN_CACHE_TTL_SECONDS:-86400}" CHECKS_DIR="$DATA_DIR/checks" IOC_DIR="$DATA_DIR/ioc" IOC_CATALOG_DIR="$IOC_DIR/catalogs" @@ -63,6 +67,7 @@ declare -a MACHINE_TARGETS=() declare -A PROJECT_ECOSYSTEMS=() declare -A PROJECT_SCANNERS=() declare -a PROJECT_IGNORES=() +LAST_SCAN_RESULT_PATH="" declare -a CURRENT_LOCKFILES=() declare -a CURRENT_MANIFESTS=() declare -a CURRENT_PROJECT_ROOTS=() @@ -2584,13 +2589,129 @@ build_scan_result_from_artifacts() { jq --argjson delta "$delta_json" '. + {delta: $delta}' "$result_path" > "$tmp" mv "$tmp" "$result_path" + LAST_SCAN_RESULT_PATH="$result_path" render_scan_summary "$result_path" } +# -------------------------------------------------------------------------- +# Scan cache (deps-only) +# +# A --deps-only scan reads exactly one thing: the dependency evidence +# (lockfiles + manifests). When that evidence hashes to something already +# scanned inside the TTL, re-running the scanners cannot produce a different +# verdict, so the recorded result is replayed instead. +# +# Discipline: the cache may only ever SKIP work, never invent a verdict. Every +# failure mode here — missing jq, unreadable entry, corrupt JSON, absent or +# malformed timestamp, empty evidence — falls through to a real scan. Source +# scans are never cached: they stage arbitrary files whose set is not captured +# by the evidence hash. +# -------------------------------------------------------------------------- + +scan_cache_enabled() { + local scan_mode="$1" + [[ "$scan_mode" == "deps" ]] || return 1 + [[ "${SAFE_AUDIT_SCAN_NO_CACHE:-0}" != "1" ]] || return 1 + command -v jq >/dev/null 2>&1 || return 1 + command -v sha256sum >/dev/null 2>&1 || return 1 + return 0 +} + +# sha256 over machine, target, mode, and every evidence file's own sha256. +# Empty evidence is NOT cacheable: there would be nothing to invalidate on. +scan_cache_key() { + local machine="$1" target="$2" scan_mode="$3" + shift 3 + local tmp file digest + tmp=$(mktemp) || return 1 + printf '%s\n%s\n%s\n' "$machine" "$target" "$scan_mode" > "$tmp" + local -a evidence=() + mapfile -t evidence < <(printf '%s\n' "$@" | LC_ALL=C sort -u) + local counted=0 + for file in "${evidence[@]}"; do + [[ -n "$file" ]] || continue + if [[ ! -r "$file" ]]; then + rm -f "$tmp" + return 1 + fi + printf '%s %s\n' "$(sha256sum "$file" | awk '{print $1}')" "$file" >> "$tmp" + counted=$((counted + 1)) + done + if (( counted == 0 )); then + rm -f "$tmp" + return 1 + fi + digest=$(sha256sum "$tmp" | awk '{print $1}') + rm -f "$tmp" + [[ -n "$digest" ]] || return 1 + printf '%s' "$digest" +} + +scan_cache_human_age() { + local seconds="$1" + if (( seconds < 60 )); then + printf '%ds' "$seconds" + elif (( seconds < 3600 )); then + printf '%dm' "$((seconds / 60))" + else + printf '%dh %dm' "$((seconds / 3600))" "$(((seconds % 3600) / 60))" + fi +} + +# Replay a cached result. Returns 1 for ANY doubt so the caller runs a real +# scan; returns 0 only after rendering a structurally valid, fresh entry. +scan_cache_replay() { + local key="$1" + local path="$SCAN_CACHE_DIR/$key.json" + [[ -n "$key" && -r "$path" ]] || return 1 + + local created now age + created="$(jq -r '._cache.created_epoch // empty' "$path" 2>/dev/null || true)" + [[ "$created" =~ ^[0-9]+$ ]] || return 1 + now="$(date +%s)" + age=$((now - created)) + (( age >= 0 )) || return 1 + (( age < SCAN_CACHE_TTL_SECONDS )) || return 1 + jq -e 'has("verdict") and has("summary") and has("cve_scan")' "$path" >/dev/null 2>&1 || return 1 + + printf '[safe audit] scan cache hit (%s) — dependency evidence unchanged; --no-cache forces a fresh scan\n' \ + "$(scan_cache_human_age "$age")" + LAST_SCAN_RESULT_PATH="$path" + render_scan_summary "$path" +} + +# Cache writes never fail a scan: a full result was already produced. +scan_cache_store() { + local key="$1" result="$2" + [[ -n "$key" && -n "$result" && -r "$result" ]] || return 0 + mkdir -p "$SCAN_CACHE_DIR" 2>/dev/null || return 0 + local tmp + tmp=$(mktemp "$SCAN_CACHE_DIR/.tmp.XXXXXX" 2>/dev/null) || return 0 + if jq --arg k "$key" --argjson t "$(date +%s)" \ + '. + {_cache: {key: $k, created_epoch: $t}}' "$result" > "$tmp" 2>/dev/null; then + mv "$tmp" "$SCAN_CACHE_DIR/$key.json" 2>/dev/null || rm -f "$tmp" + else + rm -f "$tmp" + fi + return 0 +} + scan_source_into_result() { local machine="$1" source="$2" original_target="$3" strategy="$4" scan_mode="${5:-source}" allow_source_fallback="${6:-1}" vinfo "$machine: scan source=$(printf '%q' "$source") target=$(printf '%q' "$original_target") strategy=$strategy mode=$scan_mode fallback=$allow_source_fallback" gather_projects_from_source "$source" "$allow_source_fallback" + + local cache_key="" + if scan_cache_enabled "$scan_mode"; then + cache_key="$(scan_cache_key "$machine" "$original_target" "$scan_mode" \ + ${CURRENT_LOCKFILES[@]+"${CURRENT_LOCKFILES[@]}"} \ + ${CURRENT_MANIFESTS[@]+"${CURRENT_MANIFESTS[@]}"} || true)" + vinfo "$machine: scan cache key=${cache_key:-}" + if [[ -n "$cache_key" ]] && scan_cache_replay "$cache_key"; then + return 0 + fi + fi + detect_machine_tools "$machine" local project_tools_rc confirm_scan_with_missing_project_audit_tools "$machine" "$original_target" @@ -2635,7 +2756,12 @@ scan_source_into_result() { jq -s 'add' "${project_audits_files[@]}" > "$audits_tmp" fi - build_scan_result_from_artifacts "$machine" "$original_target" "$strategy" "$sbom_tmp" "$osv_json" "$grype_json" "$audits_tmp" "$scan_mode" "$source_scan_tmp" + local build_rc=0 + build_scan_result_from_artifacts "$machine" "$original_target" "$strategy" "$sbom_tmp" "$osv_json" "$grype_json" "$audits_tmp" "$scan_mode" "$source_scan_tmp" || build_rc=$? + if [[ -n "$cache_key" && "$build_rc" -eq 0 ]]; then + scan_cache_store "$cache_key" "$LAST_SCAN_RESULT_PATH" + fi + return "$build_rc" } scan_source_with_external_sbom_into_result() { @@ -7234,6 +7360,10 @@ scan_command() { scan_mode_set=1 shift ;; + --no-cache) + SAFE_AUDIT_SCAN_NO_CACHE=1 + shift + ;; --project) [[ $# -lt 2 ]] && die "--project requires a path" project_target="$2" @@ -7370,7 +7500,7 @@ safe audit: multi-machine audit tooling USAGE safe audit capabilities [--json] - safe audit scan [--verbose] [--deps-only | --full] [--project ] [--all | --machine ] + safe audit scan [--verbose] [--deps-only | --full] [--no-cache] [--project ] [--all | --machine ] safe audit check @ [--ecosystem ] [--json] safe audit release github --repo OWNER/REPO --version TAG --asset NAME [--json] safe audit vuln github-release --repo OWNER/REPO --version TAG [--json] diff --git a/docs/command-reference.md b/docs/command-reference.md index de6d07c..cfd98e7 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -5,6 +5,7 @@ ```bash safe run safe audit +safe install [--project] [--yes] safe install [-g|--global] [--yes] [...] safe install --manager npm|pnpm|yarn|bun|composer -g [--yes] [--trust-host] [...] safe install --sandbox [--allow-scripts] [...] @@ -167,6 +168,37 @@ safe install --composer -g vendor/pkg:^1 safe install --trust-host -g cowsay@1.6.0 ``` +### Project mode (bulk audit) + +With no package named and a manifest in the current directory — `package.json`, +`requirements.txt`, `pyproject.toml`, `Cargo.toml`, `composer.json`, or +`go.mod` — `safe install` bulk-audits what the project already depends on +instead of printing usage. `--project` forces the same mode. + +```bash +safe install # in a project directory +safe install --project +``` + +It runs `safe audit scan --deps-only --project .` (so it benefits from the scan +cache) and prints a one-screen summary: audited manifests, package count, +finding counts by severity, verdict, and the top critical/high findings with +package and advisory id. + +This mode **audits only** — it never runs a package manager, so there is +nothing to install afterwards; the confirmation records that an operator saw +the findings. + +| Outcome | Interactive | Non-interactive | +| --- | --- | --- | +| Verdict `GO` | prompt to accept (exit 0 / 1 if declined) | exit 0, quietly | +| Verdict `WARN` | prompt to accept, or `--yes` | exit 102 unless `--yes` | +| Critical findings | exit 104 | exit 104 | +| Scan unreadable or failed | exit 100 | exit 100 | + +Critical findings are the project-scale equivalent of a `BLOCK` verdict: +`--yes` accepts WARNs, never those. + `safe install --sandbox ...` preserves the isolated `safe run install` workflow. The PATH-executable gate wrappers cover these command families: diff --git a/docs/safe-audit.md b/docs/safe-audit.md index 204f6ac..c730035 100644 --- a/docs/safe-audit.md +++ b/docs/safe-audit.md @@ -105,6 +105,29 @@ Dependency-only scan: safe audit scan --deps-only --project . ``` +### Scan cache (`--deps-only`) + +A dependency-only scan reads exactly one thing: the dependency evidence +(lockfiles and manifests). When that evidence hashes to a set already scanned +within 24 hours, re-running the scanners cannot produce a different verdict, so +the recorded result is replayed instead: + +```text +[safe audit] scan cache hit (17m) — dependency evidence unchanged; --no-cache forces a fresh scan +``` + +The verdict, counts, and exit code are the recorded ones. Entries live in +`~/.local/share/safe/audit/scan-cache/.json`, keyed on the machine, +target, mode, and each evidence file's own hash — touch a lockfile or a +manifest and the next scan is a real one. `--no-cache` skips the lookup, and +`SAFE_AUDIT_SCAN_CACHE_TTL_SECONDS` overrides the 24h TTL. + +The cache can only ever skip work, never invent a verdict: a missing, expired, +corrupt, or structurally unrecognizable entry falls through to a real scan, and +a scan with no dependency evidence at all is never cached. Source and `--full` +scans are never cached — they stage arbitrary files whose set the evidence hash +does not capture. + Full filesystem scan, including installed dependency trees: ```bash diff --git a/lib/completions/_safe b/lib/completions/_safe index 9eb73e8..53b2253 100644 --- a/lib/completions/_safe +++ b/lib/completions/_safe @@ -39,7 +39,7 @@ _safe_audit() { local subcmd="${words[3]:-}" local nested="${words[4]:-}" audit_cmds=(scan check release vuln verify binary ioc setup diff status capabilities) - scan_opts=(--all --machine --project --verbose --deps-only --full) + scan_opts=(--all --machine --project --verbose --deps-only --full --no-cache) check_opts=(--ecosystem --json) capabilities_opts=(--json) ioc_opts=(--all --machine --list --update --since) diff --git a/lib/gate-lib.sh b/lib/gate-lib.sh index bea54c2..e0629a4 100644 --- a/lib/gate-lib.sh +++ b/lib/gate-lib.sh @@ -452,7 +452,11 @@ safe_gate_scan_project() { return 0 fi - scan_output="$("${SAFE_GATE_AUDIT_BIN}" scan --project . 2>&1)" + # --deps-only: the preflight only cares about the dependency evidence the + # install is about to change, and that mode is the one the scan cache can + # replay — a bare `npm ci` in an unchanged tree costs a cache hit, not a + # full scanner run. + scan_output="$("${SAFE_GATE_AUDIT_BIN}" scan --deps-only --project . 2>&1)" scan_status=$? [[ -n "${scan_output}" ]] && printf '%s\n' "${scan_output}" diff --git a/tests/audit/scan_cache.sh b/tests/audit/scan_cache.sh new file mode 100755 index 0000000..699c060 --- /dev/null +++ b/tests/audit/scan_cache.sh @@ -0,0 +1,339 @@ +#!/usr/bin/env bash +# Scan cache suite: a --deps-only scan whose dependency evidence is unchanged +# replays the recorded result instead of re-running scanners. Hermetic — every +# scanner is a logging stub on PATH, so "did the scanners run again?" is a +# direct assertion on the invocation log, not a timing guess. +# +# The cache may only ever SKIP work. Every doubt (corrupt entry, missing or +# malformed timestamp, expired TTL, unhashable evidence) must fall through to a +# real scan, so those paths are asserted here too. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SAFE_AUDIT="$ROOT/bin/safe-audit" + +PASS_COUNT=0 +FAIL_COUNT=0 + +pass() { printf 'ok - %s\n' "$*"; PASS_COUNT=$((PASS_COUNT + 1)); } +fail() { printf 'not ok - %s\n' "$*" >&2; FAIL_COUNT=$((FAIL_COUNT + 1)); } + +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "$TEST_ROOT"' EXIT + +MOCKBIN="$TEST_ROOT/mockbin" +mkdir -p "$MOCKBIN" + +# Every scanner logs its invocation and emits the minimal valid document the +# scan pipeline expects on stdout. +for tool in osv-scanner grype syft govulncheck cargo-audit pip-audit socket; do + cat > "$MOCKBIN/$tool" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$(basename -- "$0")" >> "${SCANNER_LOG:-/dev/null}" +case "$(basename -- "$0")" in + osv-scanner) printf '{"results":[]}\n' ;; + grype) printf '{"matches":[]}\n' ;; + syft) printf '{"components":[],"metadata":{"tools":[{"name":"syft"}]}}\n' ;; + *) printf '{}\n' ;; +esac +exit 0 +STUB + chmod +x "$MOCKBIN/$tool" +done + +CASE_DIR="" +CASE_HOME="" +CASE_PROJECT="" +SCANNER_LOG="" +OUT_FILE="" + +prepare_case() { + local name="$1" + CASE_DIR="$TEST_ROOT/case-$name" + CASE_HOME="$CASE_DIR/home" + CASE_PROJECT="$CASE_DIR/project" + SCANNER_LOG="$CASE_DIR/scanners.log" + OUT_FILE="$CASE_DIR/scan.out" + mkdir -p "$CASE_HOME" "$CASE_PROJECT" "$CASE_DIR/audit-config" "$CASE_DIR/audit-data" + : > "$SCANNER_LOG" + printf '{"name":"demo","version":"1.0.0"}\n' > "$CASE_PROJECT/package.json" + printf '{"lockfileVersion":3,"packages":{}}\n' > "$CASE_PROJECT/package-lock.json" +} + +# run_scan [scan flags...] +run_scan() { + : > "$SCANNER_LOG" + set +e + ( + cd "$CASE_PROJECT" || exit 99 + env \ + HOME="$CASE_HOME" \ + PATH="$MOCKBIN:/usr/bin:/bin" \ + SCANNER_LOG="$SCANNER_LOG" \ + SAFE_AUDIT_CONFIG_DIR="$CASE_DIR/audit-config" \ + SAFE_AUDIT_DATA_DIR="$CASE_DIR/audit-data" \ + "$SAFE_AUDIT" scan --deps-only --project . "$@" + ) > "$OUT_FILE" 2>&1 + STATUS=$? + set -e +} + +run_scan_source_mode() { + : > "$SCANNER_LOG" + set +e + ( + cd "$CASE_PROJECT" || exit 99 + env \ + HOME="$CASE_HOME" \ + PATH="$MOCKBIN:/usr/bin:/bin" \ + SCANNER_LOG="$SCANNER_LOG" \ + SAFE_AUDIT_CONFIG_DIR="$CASE_DIR/audit-config" \ + SAFE_AUDIT_DATA_DIR="$CASE_DIR/audit-data" \ + "$SAFE_AUDIT" scan --project . + ) > "$OUT_FILE" 2>&1 + STATUS=$? + set -e +} + +scanner_invocations() { + wc -l < "$SCANNER_LOG" | tr -d ' ' +} + +cache_entries() { + find "$CASE_DIR/audit-data/scan-cache" -maxdepth 1 -name '*.json' 2>/dev/null | wc -l | tr -d ' ' +} + +assert_hit() { + local label="$1" + if ! grep -Fq 'scan cache hit' "$OUT_FILE"; then + printf 'expected a cache hit; output:\n%s\n' "$(cat "$OUT_FILE")" >&2 + fail "$label" + return 1 + fi + return 0 +} + +assert_no_hit() { + local label="$1" + if grep -Fq 'scan cache hit' "$OUT_FILE"; then + printf 'unexpected cache hit; output:\n%s\n' "$(cat "$OUT_FILE")" >&2 + fail "$label" + return 1 + fi + return 0 +} + +assert_scanners_ran() { + local label="$1" + if [[ "$(scanner_invocations)" -eq 0 ]]; then + printf 'expected scanner invocations, log is empty\n' >&2 + fail "$label" + return 1 + fi + return 0 +} + +assert_no_scanners() { + local label="$1" + if [[ "$(scanner_invocations)" -ne 0 ]]; then + printf 'expected no scanner invocations, got:\n%s\n' "$(cat "$SCANNER_LOG")" >&2 + fail "$label" + return 1 + fi + return 0 +} + +assert_verdict_rendered() { + local label="$1" + if ! grep -Fq 'VERDICT:' "$OUT_FILE"; then + printf 'cached replay did not render a verdict; output:\n%s\n' "$(cat "$OUT_FILE")" >&2 + fail "$label" + return 1 + fi + return 0 +} + +cache_entry_path() { + find "$CASE_DIR/audit-data/scan-cache" -maxdepth 1 -name '*.json' | head -n 1 +} + +case_cold_scan_runs_scanners_and_caches() { + prepare_case "cold-scan" + run_scan + [[ "$STATUS" -eq 0 ]] || { printf 'scan exited %s\n%s\n' "$STATUS" "$(cat "$OUT_FILE")" >&2; fail "$FUNCNAME"; return; } + assert_no_hit "$FUNCNAME" || return + assert_scanners_ran "$FUNCNAME" || return + [[ "$(cache_entries)" -eq 1 ]] || { printf 'expected 1 cache entry, got %s\n' "$(cache_entries)" >&2; fail "$FUNCNAME"; return; } + pass "$FUNCNAME" +} + +case_unchanged_evidence_hits_cache_without_scanners() { + prepare_case "cache-hit" + run_scan + assert_scanners_ran "$FUNCNAME" || return + local cold_verdict + cold_verdict="$(grep -E '^VERDICT:' "$OUT_FILE" | head -n 1)" + + run_scan + [[ "$STATUS" -eq 0 ]] || { printf 'cached scan exited %s\n' "$STATUS" >&2; fail "$FUNCNAME"; return; } + assert_hit "$FUNCNAME" || return + # The whole point: the scanners are not re-run. + assert_no_scanners "$FUNCNAME" || return + assert_verdict_rendered "$FUNCNAME" || return + # And the replayed verdict is the recorded one, not a fresh guess. + [[ "$(grep -E '^VERDICT:' "$OUT_FILE" | head -n 1)" == "$cold_verdict" ]] || { fail "$FUNCNAME"; return; } + pass "$FUNCNAME" +} + +case_lockfile_change_invalidates() { + prepare_case "invalidate" + run_scan + assert_scanners_ran "$FUNCNAME" || return + run_scan + assert_hit "$FUNCNAME" || return + + printf '{"lockfileVersion":3,"packages":{"node_modules/x":{"version":"1.0.0"}}}\n' > "$CASE_PROJECT/package-lock.json" + run_scan + assert_no_hit "$FUNCNAME" || return + assert_scanners_ran "$FUNCNAME" || return + [[ "$(cache_entries)" -eq 2 ]] || { printf 'expected 2 cache entries, got %s\n' "$(cache_entries)" >&2; fail "$FUNCNAME"; return; } + + # The new evidence caches in its own right. + run_scan + assert_hit "$FUNCNAME" || return + assert_no_scanners "$FUNCNAME" || return + pass "$FUNCNAME" +} + +case_manifest_change_invalidates() { + prepare_case "invalidate-manifest" + run_scan + run_scan + assert_hit "$FUNCNAME" || return + # Manifests are evidence too, not just lockfiles. + printf '{"name":"demo","version":"1.0.1","dependencies":{"left-pad":"^1.3.0"}}\n' > "$CASE_PROJECT/package.json" + run_scan + assert_no_hit "$FUNCNAME" || return + assert_scanners_ran "$FUNCNAME" || return + pass "$FUNCNAME" +} + +case_no_cache_flag_bypasses() { + prepare_case "no-cache-flag" + run_scan + run_scan + assert_hit "$FUNCNAME" || return + + run_scan --no-cache + assert_no_hit "$FUNCNAME" || return + assert_scanners_ran "$FUNCNAME" || return + + # --no-cache bypasses the read; the following plain scan still hits. + run_scan + assert_hit "$FUNCNAME" || return + pass "$FUNCNAME" +} + +case_source_mode_is_never_cached() { + prepare_case "source-mode" + run_scan_source_mode + assert_no_hit "$FUNCNAME" || return + [[ "$(cache_entries)" -eq 0 ]] || { printf 'source scan wrote a cache entry\n' >&2; fail "$FUNCNAME"; return; } + run_scan_source_mode + assert_no_hit "$FUNCNAME" || return + assert_scanners_ran "$FUNCNAME" || return + pass "$FUNCNAME" +} + +case_expired_entry_falls_through() { + prepare_case "expired" + run_scan + run_scan + assert_hit "$FUNCNAME" || return + + # Backdate the entry past the 24h TTL. + local entry tmp + entry="$(cache_entry_path)" + [[ -n "$entry" ]] || { fail "$FUNCNAME"; return; } + tmp="$entry.tmp" + jq --argjson t "$(( $(date +%s) - 90000 ))" '._cache.created_epoch = $t' "$entry" > "$tmp" + mv "$tmp" "$entry" + + run_scan + assert_no_hit "$FUNCNAME" || return + assert_scanners_ran "$FUNCNAME" || return + pass "$FUNCNAME" +} + +case_corrupt_entry_falls_through() { + prepare_case "corrupt" + run_scan + run_scan + assert_hit "$FUNCNAME" || return + + local entry + entry="$(cache_entry_path)" + printf 'not json at all\n' > "$entry" + run_scan + assert_no_hit "$FUNCNAME" || return + assert_scanners_ran "$FUNCNAME" || return + [[ "$STATUS" -eq 0 ]] || { printf 'corrupt cache broke the scan (exit %s)\n' "$STATUS" >&2; fail "$FUNCNAME"; return; } + + # A structurally valid entry that is not a scan result is refused too. + run_scan + entry="$(cache_entry_path)" + printf '{"_cache":{"created_epoch":%s},"unrelated":true}\n' "$(date +%s)" > "$entry" + run_scan + assert_no_hit "$FUNCNAME" || return + assert_scanners_ran "$FUNCNAME" || return + pass "$FUNCNAME" +} + +case_missing_timestamp_falls_through() { + prepare_case "no-timestamp" + run_scan + local entry tmp + entry="$(cache_entry_path)" + tmp="$entry.tmp" + jq 'del(._cache.created_epoch)' "$entry" > "$tmp" + mv "$tmp" "$entry" + run_scan + assert_no_hit "$FUNCNAME" || return + assert_scanners_ran "$FUNCNAME" || return + pass "$FUNCNAME" +} + +case_no_evidence_is_not_cacheable() { + prepare_case "no-evidence" + rm -f "$CASE_PROJECT/package.json" "$CASE_PROJECT/package-lock.json" + printf 'nothing to see\n' > "$CASE_PROJECT/README.md" + run_scan + [[ "$(cache_entries)" -eq 0 ]] || { printf 'cached a scan with no dependency evidence\n' >&2; fail "$FUNCNAME"; return; } + run_scan + assert_no_hit "$FUNCNAME" || return + pass "$FUNCNAME" +} + +main() { + local case + for case in \ + case_cold_scan_runs_scanners_and_caches \ + case_unchanged_evidence_hits_cache_without_scanners \ + case_lockfile_change_invalidates \ + case_manifest_change_invalidates \ + case_no_cache_flag_bypasses \ + case_source_mode_is_never_cached \ + case_expired_entry_falls_through \ + case_corrupt_entry_falls_through \ + case_missing_timestamp_falls_through \ + case_no_evidence_is_not_cacheable + do + "$case" + done + + printf '\n%s passed, %s failed\n' "$PASS_COUNT" "$FAIL_COUNT" + [[ "$FAIL_COUNT" -eq 0 ]] +} + +main "$@" diff --git a/tests/audit/smoke.sh b/tests/audit/smoke.sh index cc7399f..ec66c26 100755 --- a/tests/audit/smoke.sh +++ b/tests/audit/smoke.sh @@ -37,6 +37,7 @@ help_output="$("$SAFE_AUDIT" --help)" grep -q 'safe audit capabilities \[--json\]' <<<"$help_output" || fail "help omits capabilities" grep -q 'safe audit scan \[--verbose\]' <<<"$help_output" || fail "help omits scan --verbose" grep -q '\[--deps-only | --full\]' <<<"$help_output" || fail "help omits scan mode options" +grep -q '\[--no-cache\]' <<<"$help_output" || fail "help omits scan --no-cache" grep -q 'safe audit ioc --update' <<<"$help_output" || fail "help omits ioc --update" grep -q 'safe audit setup --create-bundle' <<<"$help_output" || fail "help omits setup --create-bundle" grep -q 'safe audit verify sigstore-bundle' <<<"$help_output" || fail "help omits verify sigstore-bundle" @@ -45,7 +46,7 @@ pass "help output" grep -q 'capabilities' "$ROOT/lib/completions/_safe" || fail "completion omits capabilities" grep -q 'capabilities_opts=(--json)' "$ROOT/lib/completions/_safe" || fail "completion omits capabilities --json" -grep -q 'scan_opts=(--all --machine --project --verbose --deps-only --full)' "$ROOT/lib/completions/_safe" || fail "completion omits scan mode options" +grep -q 'scan_opts=(--all --machine --project --verbose --deps-only --full --no-cache)' "$ROOT/lib/completions/_safe" || fail "completion omits scan mode options" grep -q 'sigstore-bundle' "$ROOT/lib/completions/_safe" || fail "completion omits sigstore-bundle" grep -q 'tuf-bootstrap' "$ROOT/lib/completions/_safe" || fail "completion omits tuf-bootstrap" pass "completion output" diff --git a/tests/install/run.sh b/tests/install/run.sh index 78598c2..7da3d73 100755 --- a/tests/install/run.sh +++ b/tests/install/run.sh @@ -929,7 +929,7 @@ case_update_family_gates() { touch "${WORK_DIR}/package.json" SAFE_INSTALL_TEST_SCRIPT='npm update lodash' run_zsh assert_status 0 "$FUNCNAME" || return - assert_log_contains $'AUDIT\tscan\t--project\t.' "$FUNCNAME" || return + assert_log_contains $'AUDIT\tscan\t--deps-only\t--project\t.' "$FUNCNAME" || return assert_log_contains $'AUDIT\tcheck\tlodash\t--ecosystem\tnpm\t--gate\tinstall\t--op\tupdate' "$FUNCNAME" || return assert_log_contains $'REAL\tnpm\tupdate\tlodash' "$FUNCNAME" || return SAFE_INSTALL_TEST_SCRIPT='yarn upgrade left-pad' run_zsh @@ -969,7 +969,7 @@ case_local_project_scan() { touch "${WORK_DIR}/package.json" SAFE_INSTALL_TEST_SCRIPT='npm ci' run_zsh assert_status 0 "$FUNCNAME" || return - assert_log_contains $'AUDIT\tscan\t--project\t.' "$FUNCNAME" || return + assert_log_contains $'AUDIT\tscan\t--deps-only\t--project\t.' "$FUNCNAME" || return assert_log_contains $'REAL\tnpm\tci' "$FUNCNAME" || return pass "$FUNCNAME" } @@ -979,7 +979,7 @@ case_add_scans_and_checks() { touch "${WORK_DIR}/package.json" SAFE_INSTALL_TEST_SCRIPT='pnpm add --filter web --workspace-root lodash' run_zsh assert_status 0 "$FUNCNAME" || return - assert_log_contains $'AUDIT\tscan\t--project\t.' "$FUNCNAME" || return + assert_log_contains $'AUDIT\tscan\t--deps-only\t--project\t.' "$FUNCNAME" || return assert_log_contains $'AUDIT\tcheck\tlodash\t--ecosystem\tnpm\t--gate\tinstall\t--op\tinstall' "$FUNCNAME" || return assert_log_not_contains_fragment $'\tweb\t--ecosystem' "$FUNCNAME" || return pass "$FUNCNAME" @@ -1061,7 +1061,7 @@ case_critical_scan_non_tty_aborts() { touch "${WORK_DIR}/package.json" SAFE_AUDIT_SCAN_OUTPUT='critical vulnerability' SAFE_AUDIT_SCAN_STATUS=1 SAFE_INSTALL_TEST_SCRIPT='npm ci' run_zsh assert_status 102 "$FUNCNAME" || return - assert_log_contains $'AUDIT\tscan\t--project\t.' "$FUNCNAME" || return + assert_log_contains $'AUDIT\tscan\t--deps-only\t--project\t.' "$FUNCNAME" || return assert_log_not_contains_fragment $'REAL\tnpm' "$FUNCNAME" || return assert_err_contains_fragment 'safe: BLOCKED install — safe audit scan found critical findings' "$FUNCNAME" || return assert_err_not_contains_fragment 'safe install: safe audit scan reported critical findings' "$FUNCNAME" || return @@ -1118,7 +1118,7 @@ case_requirement_install_scans() { prepare_case "requirement-install-scans" SAFE_INSTALL_TEST_SCRIPT='pip install -r requirements.txt' run_zsh assert_status 0 "$FUNCNAME" || return - assert_log_contains $'AUDIT\tscan\t--project\t.' "$FUNCNAME" || return + assert_log_contains $'AUDIT\tscan\t--deps-only\t--project\t.' "$FUNCNAME" || return assert_log_not_contains_fragment $'AUDIT\tcheck' "$FUNCNAME" || return pass "$FUNCNAME" } diff --git a/tests/integration/dispatcher.sh b/tests/integration/dispatcher.sh index 2b5aa40..2162852 100755 --- a/tests/integration/dispatcher.sh +++ b/tests/integration/dispatcher.sh @@ -49,6 +49,32 @@ case "${1:-}" in --version|-v) echo "safe-audit mock" ;; status) echo "config: ${SAFE_CONFIG_DIR:-$HOME/.config/safe}/audit" ;; check) printf 'safe-audit'; for arg in "$@"; do printf '\t%s' "$arg"; done; printf '\n'; exit "${SAFE_AUDIT_STUB_STATUS:-0}" ;; + scan) + { printf 'safe-audit'; for arg in "$@"; do printf '\t%s' "$arg"; done; printf '\n'; } >> "${SAFE_AUDIT_STUB_SCAN_LOG:-/dev/null}" + { printf 'safe-audit'; for arg in "$@"; do printf '\t%s' "$arg"; done; printf '\n'; } + result="${SAFE_AUDIT_STUB_SCAN_RESULT:-}" + if [[ -n "$result" ]]; then + cat > "$result" < "$PROJECT_SCAN_LOG" + ( + cd "$dir" || exit 99 + PATH="$shim:$PATH" \ + SAFE_CONFIG_DIR="$tmp/project-$label-config" \ + SAFE_DATA_DIR="$tmp/project-$label-data" \ + SAFE_AUDIT_STUB_SCAN_RESULT="$tmp/project-$label-result.json" \ + SAFE_AUDIT_STUB_SCAN_LOG="$PROJECT_SCAN_LOG" \ + "$@" \ + "$shim/safe" install "${PROJECT_ARGS[@]}" + ) >"$PROJECT_OUT" 2>"$PROJECT_ERR" &2 + fail "safe install project $label expected rc=$expected_rc, got rc=$rc" + } +} + +project_dir="$tmp/project-src" +mkdir -p "$project_dir" +printf '{"name":"demo"}\n' > "$project_dir/package.json" + +# Clean project, non-interactive: exit 0 quietly, scan run in deps-only mode. +PROJECT_ARGS=() +project_case clean 0 "$project_dir" env SAFE_AUDIT_STUB_SCAN_VERDICT=GO +grep -Fq $'safe-audit\tscan\t--deps-only\t--project\t.' "$PROJECT_SCAN_LOG" || fail "project mode did not run a deps-only scan" +grep -Fq 'safe install: project audit' "$PROJECT_OUT" || fail "project mode printed no summary" +grep -Fq 'manifests: package.json' "$PROJECT_OUT" || fail "project mode summary omits the audited manifests" +grep -Fq 'verdict: GO' "$PROJECT_OUT" || fail "project mode summary omits the verdict" + +# The explicit flag forces the same mode. +PROJECT_ARGS=(--project) +project_case flag 0 "$project_dir" env SAFE_AUDIT_STUB_SCAN_VERDICT=GO +grep -Fq $'safe-audit\tscan\t--deps-only\t--project\t.' "$PROJECT_SCAN_LOG" || fail "safe install --project did not scan" + +# WARN needs an operator: non-TTY refuses 102, --yes accepts. +PROJECT_ARGS=() +project_case warn-nontty 102 "$project_dir" env SAFE_AUDIT_STUB_SCAN_VERDICT=WARN SAFE_AUDIT_STUB_SCAN_HIGH=2 +grep -Fq 'safe: BLOCKED project audit' "$PROJECT_ERR" || fail "project WARN refusal missing BLOCKED contract" +grep -Fq 'safe explain' "$PROJECT_ERR" || fail "project WARN refusal missing safe explain pointer" +grep -Fq 'top findings:' "$PROJECT_OUT" || fail "project summary omits top findings for high severity" +grep -Fq 'CVE-2026-0001' "$PROJECT_OUT" || fail "project summary omits the advisory id" + +PROJECT_ARGS=(--yes) +project_case warn-yes 0 "$project_dir" env SAFE_AUDIT_STUB_SCAN_VERDICT=WARN SAFE_AUDIT_STUB_SCAN_HIGH=2 + +# Critical findings are the project-scale BLOCK: 104 even under --yes. +PROJECT_ARGS=() +project_case critical 104 "$project_dir" env SAFE_AUDIT_STUB_SCAN_VERDICT=WARN SAFE_AUDIT_STUB_SCAN_CRITICAL=1 +grep -Fq 'critical finding(s)' "$PROJECT_ERR" || fail "project critical refusal does not name the finding count" +grep -Fq 'GHSA-test-crit' "$PROJECT_OUT" || fail "project summary omits the critical advisory id" + +PROJECT_ARGS=(--yes) +project_case critical-yes 104 "$project_dir" env SAFE_AUDIT_STUB_SCAN_VERDICT=WARN SAFE_AUDIT_STUB_SCAN_CRITICAL=3 + +# Fail closed when the scan cannot be trusted: no result document, or a +# scanner that failed outright. +PROJECT_ARGS=() +project_case no-details 100 "$project_dir" env SAFE_AUDIT_STUB_SCAN_NO_DETAILS=1 +grep -Fq 'no readable result' "$PROJECT_ERR" || fail "unreadable scan result did not fail closed legibly" +PROJECT_ARGS=() +project_case scan-failed 100 "$project_dir" env SAFE_AUDIT_STUB_SCAN_EXIT=3 +grep -Fq 'BLOCKED project audit' "$PROJECT_ERR" || fail "failed scan did not fail closed" + +# A directory with no manifest is not a project: the old usage error stands. +empty_dir="$tmp/project-empty" +mkdir -p "$empty_dir" +PROJECT_ARGS=(--project) +project_case no-manifest 1 "$empty_dir" env +grep -Fq 'found no manifest' "$PROJECT_ERR" || fail "--project without a manifest did not explain itself" + +# A named package still takes the spec path, manifest or not. +PROJECT_ARGS=(--yes -g cowsay@1.6.0) +project_case spec-still-audits 0 "$project_dir" env +grep -Fq $'safe-audit\tcheck\tcowsay@1.6.0\t--ecosystem\tnpm' "$PROJECT_OUT" || fail "spec install stopped auditing in a project dir" +grep -Fq $'npm\tinstall\t-g\tcowsay@1.6.0' "$PROJECT_OUT" || fail "spec install stopped delegating in a project dir" +[[ ! -s "$PROJECT_SCAN_LOG" ]] || fail "spec install ran a project scan" +pass "safe install project mode audits, refuses, and fails closed" + explain_output="$("$SAFE" explain)" grep -Fq '100 blocked by policy' <<<"$explain_output" || fail "safe explain missing exit code table" grep -Fq 'never attempt them yourself' <<<"$explain_output" || fail "safe explain missing agent guidance"