Reject native Bash sandboxing on live evidence; keep the evaluation #43
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: verify | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| jobs: | |
| verify: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # The gate needs jq for the JSON checks and the install dry run. plutil does not exist | |
| # on Linux, so the launchd check skips itself and says so. | |
| - name: Install jq | |
| run: sudo apt-get update -qq && sudo apt-get install -y -qq jq | |
| # Check 19 runs `claude plugin validate --strict`; without the CLI it skips, and the | |
| # falsifiability suite skips its case 19 with it. Install it so CI proves the real thing. | |
| - name: Install Claude Code CLI | |
| run: npm install -g @anthropic-ai/claude-code | |
| - name: Run the verification gate | |
| run: ./.claude/verify.sh | |
| # A gate that cannot fail proves nothing. This used to break one skill description and | |
| # call it done, which left the other checks unproven — and two of them were not enforcing | |
| # anything at the time. Every declared check now gets its own mutation, and check 16 of | |
| # the gate fails if a check is added without one. | |
| - name: Prove every check is falsifiable | |
| run: ./tests/gate-falsifiability.sh | |
| - name: Confirm the tree is restored | |
| run: git diff --exit-code | |
| # Everything above runs the gate. This installs for real, on Linux, into a disposable HOME. | |
| # It exists because the worst bug this repo has shipped was Linux-only and invisible on the | |
| # machine it was written on: hooks called /usr/bin/jq by absolute path, which is a macOS | |
| # path, so off macOS the verify gate silently stopped blocking and the session hook injected | |
| # nothing at all. A dry run cannot catch that. Only installing and firing the hooks can. | |
| install-linux: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install jq | |
| run: sudo apt-get update -qq && sudo apt-get install -y -qq jq | |
| - name: install.sh, for real | |
| run: ./install.sh | |
| - name: The installed tree matches the repo | |
| run: ./bin/doctor --drift | |
| # Expected counts are derived from the repo at run time, not hardcoded — a literal here | |
| # went stale the first time a skill was added and failed a green tree. | |
| - name: Everything landed | |
| run: | | |
| set -e | |
| count() { ls -d $1 2>/dev/null | wc -l | tr -d ' '; } | |
| for spec in "$HOME/.claude/skills/*/:claude/skills/*/" \ | |
| "$HOME/.claude/agents/*.md:claude/agents/*.md" \ | |
| "$HOME/.claude/commands/*.md:claude/commands/*.md" \ | |
| "$HOME/.claude/hooks/*.sh:claude/hooks/*.sh"; do | |
| pat="${spec%:*}"; src="${spec##*:}" | |
| got=$(count "$pat"); want=$(count "$src") | |
| [ "$got" = "$want" ] && [ "$want" != 0 ] || { echo "$pat: got $got, repo has $want"; exit 1; } | |
| echo "ok $pat -> $got (repo: $want)" | |
| done | |
| test -x "$HOME/.claude/hooks/verify-gate.sh" | |
| - name: The Stop gate blocks on Linux, with and without jq on PATH | |
| run: | | |
| set -e | |
| mkdir -p /tmp/g/repo/.claude /tmp/g/home/.config/agents /tmp/g/tmp /tmp/g/bin | |
| printf '#!/usr/bin/env bash\necho "seeded failure"\nexit 1\n' > /tmp/g/repo/.claude/verify.sh | |
| chmod +x /tmp/g/repo/.claude/verify.sh | |
| printf '%s %s\n' "$(sha256sum /tmp/g/repo/.claude/verify.sh | cut -d' ' -f1)" \ | |
| /tmp/g/repo/.claude/verify.sh > /tmp/g/home/.config/agents/verify-trust | |
| for t in bash sh cat cut grep sed awk tr rm mkdir env dirname basename sha256sum; do | |
| ln -sf "$(command -v $t)" /tmp/g/bin/$t | |
| done | |
| out=$(printf '{"session_id":"ci"}' | env HOME=/tmp/g/home TMPDIR=/tmp/g/tmp \ | |
| CLAUDE_PROJECT_DIR=/tmp/g/repo bash claude/hooks/verify-gate.sh) | |
| echo "$out" | jq -e '.decision=="block"' >/dev/null || { echo "did not block with jq"; exit 1; } | |
| echo "ok blocks with jq" | |
| sed 's#/usr/bin/jq#/nonexistent/jq#' claude/hooks/verify-gate.sh > /tmp/g/nojq.sh | |
| out=$(printf '{"session_id":"ci2"}' | env PATH=/tmp/g/bin HOME=/tmp/g/home \ | |
| TMPDIR=/tmp/g/tmp CLAUDE_PROJECT_DIR=/tmp/g/repo bash /tmp/g/nojq.sh) | |
| echo "$out" | jq -e '.decision=="block"' >/dev/null || { echo "did not block without jq"; exit 1; } | |
| echo "ok blocks with no jq reachable" | |
| - name: The session hook injects the routing block on Linux | |
| run: | | |
| set -e | |
| n=$(printf '{"hook_event_name":"SessionStart"}' | bash claude/hooks/inject-session-context.sh | wc -c) | |
| [ "$n" -gt 2000 ] || { echo "session hook injected only $n bytes"; exit 1; } | |
| echo "ok session hook injected $n bytes" | |
| - name: uninstall.sh --dry-run | |
| run: ./uninstall.sh --dry-run |