From 9f2a2f16549b0fffff2c8e14e868f4aae2bb7179 Mon Sep 17 00:00:00 2001 From: Kevin Kern Date: Tue, 28 Jul 2026 13:37:51 +0200 Subject: [PATCH 1/5] ci: add risk-based verification fast path --- .github/workflows/ci.yml | 256 +++++-- .github/workflows/linux-receipts.yml | 130 ++++ .github/workflows/security.yml | 99 --- apps/docs/alchemy.run.ts | 8 +- .../docs/operations/docs-deployment.mdx | 14 +- apps/docs/package.json | 6 +- apps/docs/scripts/use-existing-output.mjs | 8 + apps/docs/scripts/verify-live-deployment.mjs | 54 ++ apps/docs/scripts/verify-maintenance.mjs | 18 +- .../docs/scripts/verify-static-deployment.mjs | 19 + docs/RELEASE.md | 38 +- package.json | 9 +- plugins/planr/skills/planr-goal/SKILL.md | 6 +- plugins/planr/skills/planr-loop/SKILL.md | 10 +- plugins/planr/skills/planr-review/SKILL.md | 6 +- .../planr/skills/planr-verify-web/SKILL.md | 4 +- plugins/planr/skills/planr-work/SKILL.md | 13 +- scripts/ci-router.mjs | 138 ++++ scripts/classify-changes.mjs | 71 ++ scripts/deploy-docs.mjs | 66 ++ .../fixtures/verification-policy/cases.json | 16 + scripts/release.sh | 43 +- scripts/test-ci-router.mjs | 71 ++ scripts/test-docs-deployment.mjs | 47 ++ scripts/test-planr-risk-based-guidance.mjs | 33 + scripts/test-release-eval-gate.mjs | 10 +- scripts/test-release-script.mjs | 196 +++++- scripts/test-verification-policy.mjs | 97 +++ scripts/test-verification-runner.mjs | 289 ++++++++ scripts/test-verify-github-actions.mjs | 117 ++- scripts/verification-policy.mjs | 360 ++++++++++ scripts/verification-runner.mjs | 664 ++++++++++++++++++ scripts/verify-github-actions.mjs | 132 ++-- scripts/verify-release-promotion.mjs | 167 +++++ scripts/write-ci-promotion-receipt.mjs | 65 ++ 35 files changed, 2924 insertions(+), 356 deletions(-) create mode 100644 .github/workflows/linux-receipts.yml delete mode 100644 .github/workflows/security.yml create mode 100644 apps/docs/scripts/use-existing-output.mjs create mode 100644 apps/docs/scripts/verify-live-deployment.mjs create mode 100644 scripts/ci-router.mjs create mode 100644 scripts/classify-changes.mjs create mode 100644 scripts/deploy-docs.mjs create mode 100644 scripts/fixtures/verification-policy/cases.json create mode 100644 scripts/test-ci-router.mjs create mode 100644 scripts/test-docs-deployment.mjs create mode 100644 scripts/test-planr-risk-based-guidance.mjs create mode 100644 scripts/test-verification-policy.mjs create mode 100644 scripts/test-verification-runner.mjs create mode 100644 scripts/verification-policy.mjs create mode 100644 scripts/verification-runner.mjs create mode 100644 scripts/verify-release-promotion.mjs create mode 100644 scripts/write-ci-promotion-receipt.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b38fe..ea10a38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: push: branches: [main] pull_request: + workflow_dispatch: permissions: contents: read @@ -13,8 +14,53 @@ concurrency: cancel-in-progress: true jobs: + router: + name: Verification Router + runs-on: ubuntu-latest + outputs: + profile: ${{ steps.route.outputs.profile }} + policy_version: ${{ steps.route.outputs.policy_version }} + policy_digest: ${{ steps.route.outputs.policy_digest }} + changed_files_digest: ${{ steps.route.outputs.changed_files_digest }} + live_browser: ${{ steps.route.outputs.live_browser }} + docs: ${{ steps.route.outputs.docs }} + quality: ${{ steps.route.outputs.quality }} + release: ${{ steps.route.outputs.release }} + linux_portability: ${{ steps.route.outputs.linux_portability }} + steps: + - name: Checkout complete change history + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Route affected verification gates + id: route + env: + PLANR_BASE_REVISION: ${{ github.event.pull_request.base.sha || github.event.before || format('{0}^', github.sha) }} + PLANR_HEAD_REVISION: ${{ github.sha }} + run: | + mkdir -p .planr/ci + node scripts/ci-router.mjs route \ + --base "$PLANR_BASE_REVISION" \ + --head "$PLANR_HEAD_REVISION" \ + --github-output "$GITHUB_OUTPUT" \ + --selection-output .planr/ci/selection.json + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + printf 'docs=false\nquality=false\nrelease=false\nlinux_portability=true\n' >> "$GITHUB_OUTPUT" + fi + + - name: Save explicit routing evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: verification-selection + path: .planr/ci/selection.json + retention-days: 7 + docs: name: Documentation + needs: router + if: needs.router.outputs.docs == 'true' runs-on: ubuntu-latest steps: - name: Checkout @@ -22,6 +68,12 @@ jobs: with: persist-credentials: false + - name: Download exact-source selection + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: verification-selection + path: .planr/ci + - name: Set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: @@ -36,20 +88,12 @@ jobs: - name: Install workspace dependencies run: pnpm install --frozen-lockfile - - name: Validate documentation content - run: pnpm docs:content - - - name: Typecheck documentation - run: pnpm docs:typecheck - - - name: Lint documentation - run: pnpm docs:lint - - - name: Build documentation - run: pnpm docs:build - - - name: Build Cloudflare static deployment artifact - run: pnpm docs:verify-deployment + - name: Run selected documentation gates once + run: | + node scripts/verification-runner.mjs run \ + --input .planr/ci/selection.json \ + --head "$GITHUB_SHA" \ + --receipt .planr/receipts/docs.json - name: Replay documented onboarding against this repository run: pnpm docs:verify-onboarding @@ -72,8 +116,21 @@ jobs: - name: Verify documentation release inventory and links run: pnpm docs:verify-release + - name: Save exact-source documentation output + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: reviewed-docs-${{ github.sha }} + path: | + apps/docs/out + .planr/ci/selection.json + .planr/receipts/docs.json + compression-level: 0 + retention-days: 7 + quality: name: Quality Gates + needs: router + if: needs.router.outputs.quality == 'true' runs-on: ubuntu-latest steps: - name: Checkout @@ -81,11 +138,6 @@ jobs: with: persist-credentials: false - - name: Install system tools - run: | - sudo apt-get update - sudo apt-get install -y shellcheck - - name: Install Rust components run: | rustup component add rustfmt clippy @@ -99,14 +151,33 @@ jobs: - name: Rust tests run: cargo test - - name: Deterministic local release-eval contract - run: npm run verify:release-eval-gate + release-contracts: + name: Release Contracts + needs: router + if: needs.router.outputs.release == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 - - name: Deterministic release orchestration contract - run: npm run verify:release-script + - name: Install system tools + run: | + sudo apt-get update + sudo apt-get install -y shellcheck - - name: Shell lint - run: shellcheck scripts/*.sh + - name: Verify workflow routing and release contracts + run: | + npm run verify:github-actions + npm run verify:release-eval-gate + npm run verify:release-script + shellcheck scripts/*.sh - name: Release dry-run run: | @@ -114,15 +185,10 @@ jobs: npm pack --dry-run node npm/bin/planr.js --version - - name: Cargo audit - run: | - if ! command -v cargo-audit >/dev/null 2>&1; then - cargo install cargo-audit --locked - fi - cargo audit --deny warnings - linux-portability: name: Portable Linux ${{ matrix.target }} + needs: router + if: needs.router.outputs.linux_portability == 'true' strategy: fail-fast: false matrix: @@ -140,46 +206,132 @@ jobs: with: persist-credentials: false - - name: Build portable Linux artifact - env: - PLANR_TARGET: ${{ matrix.target }} - PLANR_CARGO_TARGET: ${{ matrix.rust_target }} - run: scripts/build-linux-release.sh + - name: Download exact-SHA selection + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: verification-selection + path: .planr/ci - - name: Verify static linkage, lifecycle, checksums, and npm bytes - env: - PLANR_TARGET: ${{ matrix.target }} - PLANR_CARGO_TARGET: ${{ matrix.rust_target }} - run: scripts/verify-linux-release-artifact.sh + - name: Build and verify on the compatible native host + run: | + node scripts/verification-runner.mjs run-linux-target \ + --target "${{ matrix.target }}" \ + --receipt ".planr/receipts/${{ matrix.target }}.json" \ + --input .planr/ci/selection.json \ + --head "$GITHUB_SHA" - - name: Save exact Linux release tarball + - name: Save exact Linux archive and target receipt uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: portable-${{ matrix.target }} - path: dist/planr-${{ matrix.target }}.tar.gz + path: | + dist/planr-${{ matrix.target }}.tar.gz + .planr/receipts/${{ matrix.target }}.json compression-level: 0 linux-portability-checksums: name: Portable Linux aggregate checksums - needs: linux-portability + needs: [router, linux-portability] + if: needs.router.outputs.linux_portability == 'true' && needs.linux-portability.result == 'success' runs-on: ubuntu-24.04 steps: + - name: Checkout exact source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download exact-SHA selection + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: verification-selection + path: .planr/ci + - name: Download exact Linux release tarballs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: portable-linux-* - path: assets + path: . merge-multiple: true - - name: Verify complete aggregate checksum set + - name: Verify both receipts and complete aggregate checksum set run: | - test "$(find assets -maxdepth 1 -name 'planr-linux-*.tar.gz' -type f | wc -l)" -eq 2 - cd assets + node scripts/verification-runner.mjs verify-linux-target \ + --receipt .planr/receipts/linux-x86_64.json \ + --input .planr/ci/selection.json \ + --head "$GITHUB_SHA" + node scripts/verification-runner.mjs verify-linux-target \ + --receipt .planr/receipts/linux-arm64.json \ + --input .planr/ci/selection.json \ + --head "$GITHUB_SHA" + test "$(find dist -maxdepth 1 -name 'planr-linux-*.tar.gz' -type f | wc -l)" -eq 2 + cd dist sha256sum planr-linux-arm64.tar.gz planr-linux-x86_64.tar.gz > SHA256SUMS sha256sum -c SHA256SUMS - - name: Save aggregate checksum evidence + - name: Save exact-SHA native Linux evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-linux-receipts-${{ github.sha }} + path: | + dist/planr-linux-x86_64.tar.gz + dist/planr-linux-arm64.tar.gz + dist/SHA256SUMS + .planr/receipts/linux-x86_64.json + .planr/receipts/linux-arm64.json + compression-level: 0 + + summary: + name: CI Summary + if: always() + needs: [router, docs, quality, release-contracts, linux-portability-checksums] + runs-on: ubuntu-latest + steps: + - name: Checkout summary verifier + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Require every selected job and intentional skips + run: | + node scripts/ci-router.mjs summary \ + --router-result "${{ needs.router.result }}" \ + --selected "docs=${{ needs.router.outputs.docs }}" \ + --selected "quality=${{ needs.router.outputs.quality }}" \ + --selected "release=${{ needs.router.outputs.release }}" \ + --selected "linux_portability=${{ needs.router.outputs.linux_portability }}" \ + --result "docs=${{ needs.docs.result }}" \ + --result "quality=${{ needs.quality.result }}" \ + --result "release=${{ needs.release-contracts.result }}" \ + --result "linux_portability=${{ needs.linux-portability-checksums.result }}" + + - name: Record selected policy identity + env: + PLANR_PROFILE: ${{ needs.router.outputs.profile }} + PLANR_POLICY_VERSION: ${{ needs.router.outputs.policy_version }} + PLANR_POLICY_DIGEST: ${{ needs.router.outputs.policy_digest }} + PLANR_CHANGED_FILES_DIGEST: ${{ needs.router.outputs.changed_files_digest }} + PLANR_LIVE_BROWSER: ${{ needs.router.outputs.live_browser }} + run: | + printf 'profile=%s\npolicy_version=%s\npolicy_digest=%s\nchanged_files_digest=%s\nlive_browser=%s\n' \ + "$PLANR_PROFILE" "$PLANR_POLICY_VERSION" "$PLANR_POLICY_DIGEST" "$PLANR_CHANGED_FILES_DIGEST" "$PLANR_LIVE_BROWSER" >> "$GITHUB_STEP_SUMMARY" + + - name: Write exact-SHA promotion receipt + if: github.event_name != 'workflow_dispatch' + env: + PLANR_PROFILE: ${{ needs.router.outputs.profile }} + PLANR_POLICY_VERSION: ${{ needs.router.outputs.policy_version }} + PLANR_POLICY_DIGEST: ${{ needs.router.outputs.policy_digest }} + PLANR_CHANGED_FILES_DIGEST: ${{ needs.router.outputs.changed_files_digest }} + PLANR_DOCS_RESULT: ${{ needs.docs.result }} + PLANR_QUALITY_RESULT: ${{ needs.quality.result }} + PLANR_RELEASE_RESULT: ${{ needs.release-contracts.result }} + PLANR_LINUX_RESULT: ${{ needs.linux-portability-checksums.result }} + run: node scripts/write-ci-promotion-receipt.mjs .planr/ci/promotion-receipt.json + + - name: Save exact-SHA promotion receipt + if: github.event_name != 'workflow_dispatch' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: portable-linux-sha256sums - path: assets/SHA256SUMS + name: release-promotion-${{ github.sha }} + path: .planr/ci/promotion-receipt.json + retention-days: 7 diff --git a/.github/workflows/linux-receipts.yml b/.github/workflows/linux-receipts.yml new file mode 100644 index 0000000..4267878 --- /dev/null +++ b/.github/workflows/linux-receipts.yml @@ -0,0 +1,130 @@ +name: Native Linux receipts + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: native-linux-receipts-${{ github.sha }} + cancel-in-progress: false + +jobs: + selection: + name: Bind exact source selection + runs-on: ubuntu-24.04 + steps: + - name: Checkout exact source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + + - name: Write exact-SHA selection + run: | + mkdir -p .planr/ci + node scripts/ci-router.mjs route \ + --base "${GITHUB_SHA}^" \ + --head "$GITHUB_SHA" \ + --selection-output .planr/ci/selection.json + + - name: Save exact-SHA selection + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-linux-selection-${{ github.sha }} + path: .planr/ci/selection.json + retention-days: 7 + + target: + name: Native ${{ matrix.target }} receipt + needs: selection + strategy: + fail-fast: false + matrix: + include: + - target: linux-x86_64 + runner: ubuntu-24.04 + - target: linux-arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + steps: + - name: Checkout exact source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download exact-SHA selection + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: native-linux-selection-${{ github.sha }} + path: .planr/ci + + - name: Build and verify on the compatible native host + run: | + node scripts/verification-runner.mjs run-linux-target \ + --target "${{ matrix.target }}" \ + --receipt ".planr/receipts/${{ matrix.target }}.json" \ + --input .planr/ci/selection.json \ + --head "$GITHUB_SHA" + + - name: Save exact archive and target receipt + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-${{ matrix.target }}-${{ github.sha }} + path: | + dist/planr-${{ matrix.target }}.tar.gz + .planr/receipts/${{ matrix.target }}.json + compression-level: 0 + retention-days: 7 + + aggregate: + name: Bind native Linux receipt set + needs: [selection, target] + runs-on: ubuntu-24.04 + steps: + - name: Checkout exact source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download exact-SHA selection + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: native-linux-selection-${{ github.sha }} + path: .planr/ci + + - name: Download both native target results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: native-linux-*-${{ github.sha }} + path: . + merge-multiple: true + + - name: Verify both receipts and aggregate exact archives + run: | + node scripts/verification-runner.mjs verify-linux-target \ + --receipt .planr/receipts/linux-x86_64.json \ + --input .planr/ci/selection.json \ + --head "$GITHUB_SHA" + node scripts/verification-runner.mjs verify-linux-target \ + --receipt .planr/receipts/linux-arm64.json \ + --input .planr/ci/selection.json \ + --head "$GITHUB_SHA" + test "$(find dist -maxdepth 1 -name 'planr-linux-*.tar.gz' -type f | wc -l)" -eq 2 + cd dist + sha256sum planr-linux-arm64.tar.gz planr-linux-x86_64.tar.gz > SHA256SUMS + sha256sum -c SHA256SUMS + + - name: Save exact-SHA native Linux evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-linux-receipts-${{ github.sha }} + path: | + dist/planr-linux-x86_64.tar.gz + dist/planr-linux-arm64.tar.gz + dist/SHA256SUMS + .planr/receipts/linux-x86_64.json + .planr/receipts/linux-arm64.json + compression-level: 0 + retention-days: 7 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml deleted file mode 100644 index 598740c..0000000 --- a/.github/workflows/security.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: Security - -on: - push: - branches: [main] - pull_request: - -permissions: - contents: read - -concurrency: - group: security-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - secret-scan: - name: Secret Scan - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - name: Check repository privacy boundaries - run: sh scripts/check-repository-privacy.sh - - - name: Install pinned security scanners - env: - TRUFFLEHOG_SHA256: 7105f1cd6577f058a9e39d0578f1a99c8a1e481e4d3512cd8a09acfe22a0fdc0 - TRIVY_SHA256: 8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9 - run: | - tools_dir="$RUNNER_TEMP/planr-security-tools" - mkdir -p "$tools_dir" - - curl --fail --location --proto '=https' --tlsv1.2 --retry 3 \ - --output "$RUNNER_TEMP/trufflehog.tar.gz" \ - https://github.com/trufflesecurity/trufflehog/releases/download/v3.96.0/trufflehog_3.96.0_linux_amd64.tar.gz - printf '%s %s\n' "$TRUFFLEHOG_SHA256" "$RUNNER_TEMP/trufflehog.tar.gz" | sha256sum --check - - tar -xzf "$RUNNER_TEMP/trufflehog.tar.gz" -C "$tools_dir" trufflehog - - curl --fail --location --proto '=https' --tlsv1.2 --retry 3 \ - --output "$RUNNER_TEMP/trivy.tar.gz" \ - https://github.com/aquasecurity/trivy/releases/download/v0.70.0/trivy_0.70.0_Linux-64bit.tar.gz - printf '%s %s\n' "$TRIVY_SHA256" "$RUNNER_TEMP/trivy.tar.gz" | sha256sum --check - - tar -xzf "$RUNNER_TEMP/trivy.tar.gz" -C "$tools_dir" trivy - - "$tools_dir/trufflehog" --version - "$tools_dir/trivy" --version - echo "$tools_dir" >> "$GITHUB_PATH" - - - name: TruffleHog verified secrets - run: trufflehog git "file://$GITHUB_WORKSPACE" --results=verified --fail --no-update --github-actions - - - name: Trivy secret and misconfig scan - run: | - trivy fs \ - --scanners secret,misconfig \ - --ignorefile .trivyignore.yaml \ - --skip-check-update \ - --skip-dirs target \ - --skip-dirs dist \ - --skip-dirs node_modules \ - --exit-code 1 \ - . - - actions-security: - name: GitHub Actions Security - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install pinned zizmor - env: - ZIZMOR_SHA256: a8000f3c683319a523d3b20df0e75457ba591f049cfcbfa98966631b56733c03 - run: | - tools_dir="$RUNNER_TEMP/planr-security-tools" - mkdir -p "$tools_dir" - curl --fail --location --proto '=https' --tlsv1.2 --retry 3 \ - --output "$RUNNER_TEMP/zizmor.tar.gz" \ - https://github.com/zizmorcore/zizmor/releases/download/v1.24.1/zizmor-x86_64-unknown-linux-gnu.tar.gz - printf '%s %s\n' "$ZIZMOR_SHA256" "$RUNNER_TEMP/zizmor.tar.gz" | sha256sum --check - - tar -xzf "$RUNNER_TEMP/zizmor.tar.gz" -C "$tools_dir" zizmor - "$tools_dir/zizmor" --version - echo "$tools_dir" >> "$GITHUB_PATH" - - - name: Run zizmor - run: | - npm run verify:github-actions - zizmor . \ - --persona regular \ - --min-severity medium \ - --min-confidence medium \ - --no-progress \ - --format plain diff --git a/apps/docs/alchemy.run.ts b/apps/docs/alchemy.run.ts index 09086da..96093f7 100644 --- a/apps/docs/alchemy.run.ts +++ b/apps/docs/alchemy.run.ts @@ -10,6 +10,9 @@ const productionDomain = "planr.so"; const Website = Cloudflare.Website.StaticSite( "Website", Alchemy.Stack.useSync(({ stage }) => { + if (stage === "prod" && process.env.PLANR_DOCS_RECEIPT_VALIDATED !== "1") { + throw new Error("production docs deployment requires a validated exact-revision receipt; use pnpm docs:deploy"); + } const siteUrl = stage === "prod" ? `https://${productionDomain}` @@ -17,7 +20,10 @@ const Website = Cloudflare.Website.StaticSite( return { name: `planr-docs-${stage}`, - command: "pnpm run build", + // Production promotion validates the exact-revision receipt before + // Alchemy starts. This command deliberately consumes the reviewed output + // instead of starting a second Next production build. + command: "node scripts/use-existing-output.mjs", outdir: "out", main: "worker.mjs", domain: stage === "prod" ? productionDomain : undefined, diff --git a/apps/docs/content/docs/operations/docs-deployment.mdx b/apps/docs/content/docs/operations/docs-deployment.mdx index 12f7066..13e9d20 100644 --- a/apps/docs/content/docs/operations/docs-deployment.mdx +++ b/apps/docs/content/docs/operations/docs-deployment.mdx @@ -11,7 +11,9 @@ The documentation site is a Next.js App Router application that reads local MDX - + + +`docs:build` is the only production build in a documentation verification environment. `docs:verify-deployment` consumes the existing `apps/docs/out` tree: it performs the Wrangler dry run and checks HTML, Markdown, search, headers, redirects, and worker routing without invoking Next again. Local deployment credentials belong in the Alchemy profile created by OAuth; do not copy Cloudflare tokens into `.env.local`. CI instead supplies `CLOUDFLARE_ACCOUNT_ID` and a scoped `CLOUDFLARE_API_TOKEN`. The production Alchemy build sets `NEXT_PUBLIC_SITE_URL=https://planr.so` so canonical metadata matches the attached hostname. Archive the commit SHA, stage, Node and pnpm versions, build log, Cloudflare resource identity, and deployed URL. @@ -19,11 +21,11 @@ Local deployment credentials belong in the Alchemy profile created by OAuth; do - + -The command deploys the Alchemy production stack with the complete checked export. HTML, React navigation payloads, the client-side search database, per-page Markdown, LLM indexes, and the recovery 404 are immutable build assets. A small edge worker owns permanent legacy redirects and the response content types for agent-readable Markdown, search, and LLM routes. +The promotion command first validates the green receipt against the exact checked-out source revision, verification policy, changed-file set, commands, and `out` digest. It stops before any external mutation when that binding is stale or altered. It then performs exactly one Alchemy production deploy and runs a bounded five-route HTTP/content oracle. HTML, React navigation payloads, the client-side search database, per-page Markdown, LLM indexes, and the recovery 404 are immutable build assets. A small edge worker owns permanent legacy redirects and the response content types for agent-readable Markdown, search, and LLM routes. -`apps/docs/alchemy.run.ts` is authoritative. `Cloudflare.Website.StaticSite` runs the static export and serves `out` with `404-page` fallback behavior. Its worker-first inventory is limited to exact entries from `redirects.mjs` plus `/docs/*.md`, `/api/search`, `/llms.txt`, and `/llms-full.txt`; all human pages and ordinary static assets bypass the worker. It adopts the named `planr-docs-prod` resource when present and passes `domain: 'planr.so'` only for the `prod` stage. The `planr.so` zone must already be available in the authenticated Cloudflare account; Cloudflare provisions the custom-domain record and certificate. `Cloudflare.state()` keeps stack state remotely available to local and CI deploys. +`apps/docs/alchemy.run.ts` is authoritative. `Cloudflare.Website.StaticSite` hashes and serves the existing `out` with `404-page` fallback behavior; it never starts another Next build during promotion. Its worker-first inventory is limited to exact entries from `redirects.mjs` plus `/docs/*.md`, `/api/search`, `/llms.txt`, and `/llms-full.txt`; all human pages and ordinary static assets bypass the worker. It adopts the named `planr-docs-prod` resource when present and passes `domain: 'planr.so'` only for the `prod` stage. The `planr.so` zone must already be available in the authenticated Cloudflare account; Cloudflare provisions the custom-domain record and certificate. `Cloudflare.state()` keeps stack state remotely available to local and CI deploys. For Cloudflare-local development, use `pnpm docs:alchemy:dev`. Normal content work can continue to use `pnpm docs:dev` without cloud credentials. @@ -31,8 +33,8 @@ For Cloudflare-local development, use `pnpm docs:alchemy:dev`. Normal content wo Require successful HTTP responses and expected content from `/`, `/docs`, a representative deep route such as `/docs/getting-started/installation`, `/docs/getting-started/installation.md`, and `/api/search`. Confirm canonical metadata uses the configured origin, the browser finds Installation through the client-side search index, an unknown route renders the recovery 404, and static assets load without browser errors. -Run `PLANR_DOCS_URL=https://planr.so pnpm docs:verify-release-live` after deployment and record the command output with the release evidence. +The promotion command automatically checks `/`, `/docs`, the Installation HTML and Markdown routes, and `/api/search`, with a ten-second request bound per route. The verification selection records `live_browser=true` only for changed interactive application code. When selected, run one focused browser oracle for the changed interaction and record it separately; text, Markdown, link, and image changes do not select a browser, and promotion never restores a blanket browser suite. ## Promote -Promote only after static, semantic, live-route, and search evidence passes. Record the Alchemy stage, Worker identity, deployment output, commit, and last known-good commit needed by [Rollback](/docs/operations/rollback). Do not run `pnpm docs:destroy` as rollback. +Promote only after the exact-source receipt passes, any selected focused live-browser oracle is recorded, and approval is granted. Record the Alchemy stage, Worker identity, deployment output, commit, and last known-good commit needed by [Rollback](/docs/operations/rollback). Do not run `pnpm docs:destroy` as rollback. diff --git a/apps/docs/package.json b/apps/docs/package.json index 6663db7..b23852f 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -8,7 +8,7 @@ "alchemy:dev": "alchemy dev --stage dev", "build": "next build && node scripts/prepare-static-assets.mjs", "start": "wrangler dev --config wrangler.jsonc --port 3000 --local", - "deploy": "alchemy deploy --stage prod --yes", + "deploy": "node ../../scripts/deploy-docs.mjs", "destroy": "alchemy destroy --stage prod", "content": "fumadocs-mdx && node scripts/verify-grok-inventory.mjs && node scripts/verify-pi-inventory.mjs", "typecheck": "fumadocs-mdx && tsc --noEmit", @@ -22,7 +22,9 @@ "verify:maintenance": "node scripts/test-release-contract.mjs && node scripts/verify-maintenance.mjs", "sync:linux-portability": "node scripts/sync-linux-portability-notices.mjs", "check:linux-portability": "node scripts/sync-linux-portability-notices.mjs --check", - "verify:deployment": "pnpm run build && wrangler deploy --config wrangler.jsonc --dry-run --outdir .wrangler-static && node scripts/verify-static-deployment.mjs", + "verify:artifact": "wrangler deploy --config wrangler.jsonc --dry-run --outdir .wrangler-static && node scripts/verify-static-deployment.mjs", + "verify:deployment": "pnpm run verify:artifact", + "verify:live-deployment": "node scripts/verify-live-deployment.mjs", "verify:release": "node scripts/sync-linux-portability-notices.mjs --check && node scripts/test-linux-portability-contract.mjs && node scripts/verify-release-readiness.mjs", "verify:release-live": "node scripts/verify-release-readiness.mjs --live", "verify:clean-install": "node scripts/verify-clean-install.mjs", diff --git a/apps/docs/scripts/use-existing-output.mjs b/apps/docs/scripts/use-existing-output.mjs new file mode 100644 index 0000000..5d3ce40 --- /dev/null +++ b/apps/docs/scripts/use-existing-output.mjs @@ -0,0 +1,8 @@ +import { access, stat } from 'node:fs/promises'; +import path from 'node:path'; + +const outputRoot = path.resolve(import.meta.dirname, '..', 'out'); +await access(path.join(outputRoot, 'index.html')); +const metadata = await stat(outputRoot); +if (!metadata.isDirectory()) throw new Error('reviewed docs output is not a directory'); +console.log('docs_output=existing'); diff --git a/apps/docs/scripts/verify-live-deployment.mjs b/apps/docs/scripts/verify-live-deployment.mjs new file mode 100644 index 0000000..b7b787a --- /dev/null +++ b/apps/docs/scripts/verify-live-deployment.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +import assert from 'node:assert/strict'; +import { pathToFileURL } from 'node:url'; + +export const LIVE_DOCS_ORACLE = Object.freeze([ + { path: '/', type: 'text/html', markers: ['Planr'] }, + { path: '/docs', type: 'text/html', markers: ['Planr Documentation'] }, + { path: '/docs/getting-started/installation', type: 'text/html', markers: ['Installation'] }, + { path: '/docs/getting-started/installation.md', type: 'text/markdown', markers: ['Installation'] }, + { path: '/api/search', type: 'application/json', markers: ['getting-started/installation'] }, +]); + +export async function verifyLiveDeployment(origin, { + fetchImpl = fetch, + timeoutMs = 10_000, + routes = LIVE_DOCS_ORACLE, +} = {}) { + const base = new URL(origin); + assert.equal(base.protocol, 'https:', 'live docs origin must use HTTPS'); + assert.equal(base.pathname, '/', 'live docs origin must not contain a path'); + const observations = []; + + for (const route of routes) { + const url = new URL(route.path, base); + const response = await fetchImpl(url, { + redirect: 'follow', + signal: AbortSignal.timeout(timeoutMs), + headers: { 'user-agent': 'planr-docs-promotion-oracle/1' }, + }); + assert.equal(response.status, 200, `${route.path} returned ${response.status}`); + const contentType = response.headers.get('content-type') ?? ''; + assert.ok(contentType.toLowerCase().startsWith(route.type), `${route.path} returned ${contentType}, expected ${route.type}`); + const body = await response.text(); + for (const marker of route.markers) assert.ok(body.includes(marker), `${route.path} omits ${marker}`); + observations.push({ path: route.path, status: response.status, contentType, bytes: Buffer.byteLength(body) }); + } + + return observations; +} + +async function main() { + const urlIndex = process.argv.indexOf('--url'); + const origin = urlIndex >= 0 ? process.argv[urlIndex + 1] : process.env.PLANR_DOCS_URL; + if (!origin) throw new Error('usage: verify-live-deployment.mjs --url https://HOST'); + const observations = await verifyLiveDeployment(origin); + console.log(JSON.stringify({ verdict: 'pass', origin, routes: observations }, null, 2)); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/apps/docs/scripts/verify-maintenance.mjs b/apps/docs/scripts/verify-maintenance.mjs index eb8854a..fce68f1 100644 --- a/apps/docs/scripts/verify-maintenance.mjs +++ b/apps/docs/scripts/verify-maintenance.mjs @@ -102,7 +102,7 @@ const deployment = await readPage('operations', 'docs-deployment'); requireMarkers(deployment, 'deployment runbook', [ 'Node.js 22', 'pnpm install --frozen-lockfile', 'NEXT_PUBLIC_SITE_URL', 'Alchemy v2', 'direct Cloudflare assets', 'planr.so', 'pnpm docs:deploy', - '/api/search', 'PLANR_DOCS_URL=https://planr.so pnpm docs:verify-release-live', + '/api/search', '--receipt .planr/receipts/docs.json', 'live_browser=true', ]); const rollback = await readPage('operations', 'rollback'); @@ -155,14 +155,15 @@ for (const [name, version] of Object.entries({ ...packageJson.dependencies, ...p } assert(packageJson.engines.node === '>=22', 'apps/docs must require Node.js 22 or newer'); assert( - packageJson.scripts.deploy === 'alchemy deploy --stage prod --yes', - 'apps/docs deploy must target the Alchemy prod stage', + packageJson.scripts.deploy === 'node ../../scripts/deploy-docs.mjs', + 'apps/docs deploy must validate the reviewed receipt before targeting Alchemy prod', ); assert(packageJson.scripts.destroy === 'alchemy destroy --stage prod', 'apps/docs destroy must target the Alchemy prod stage explicitly'); assert(packageJson.scripts.build === 'next build && node scripts/prepare-static-assets.mjs', 'docs build must prepare the deployable static artifact'); assert(packageJson.scripts.start === 'wrangler dev --config wrangler.jsonc --port 3000 --local', 'docs start must emulate Cloudflare static routing'); -assert(packageJson.scripts['verify:deployment'].includes('wrangler deploy --config wrangler.jsonc --dry-run'), 'deployment verification must run Wrangler without deploying'); -assert(packageJson.scripts['verify:deployment'].includes('verify-static-deployment.mjs'), 'deployment verification must inspect the complete static artifact'); +assert(packageJson.scripts['verify:deployment'] === 'pnpm run verify:artifact', 'deployment verification must consume the existing static artifact'); +assert(packageJson.scripts['verify:artifact'].includes('wrangler deploy --config wrangler.jsonc --dry-run'), 'artifact verification must run Wrangler without deploying'); +assert(packageJson.scripts['verify:artifact'].includes('verify-static-deployment.mjs'), 'artifact verification must inspect the complete static artifact'); assert(packageJson.scripts['verify:shell'] === undefined, 'docs package must not restore the retired browser verifier command'); assert(packageJson.devDependencies['axe-core'] === undefined, 'docs package must not restore the retired browser verifier dependency'); const rootPackageJson = JSON.parse(await read('package.json')); @@ -239,7 +240,8 @@ requireMarkers(alchemyConfig, 'Alchemy deployment wiring', [ 'Alchemy.Stack(', 'Cloudflare.providers()', 'Cloudflare.state()', 'Cloudflare.Website.StaticSite(', 'planr-docs-${stage}', 'stage === "prod"', 'planr.so', 'AdoptPolicy.adopt(true)', - 'command: "pnpm run build"', 'outdir: "out"', 'main: "worker.mjs"', + 'PLANR_DOCS_RECEIPT_VALIDATED', + 'command: "node scripts/use-existing-output.mjs"', 'outdir: "out"', 'main: "worker.mjs"', 'notFoundHandling: "404-page"', 'runWorkerFirst:', 'legacyRedirects.map', 'NEXT_PUBLIC_SITE_URL', ]); requireMarkers(await read('apps/docs/wrangler.jsonc'), 'Wrangler static asset configuration', [ @@ -269,9 +271,9 @@ const sourceChecks = [ ['apps/docs/public/agents/claude.svg', ['` from the successful `CI` run for the exact +main commit. The approval file uses schema `planr.release-approval.v1` and +contains only `approval_id`, `source_sha`, `version`, `decision: "approved"`, +`approved_by`, and `approved_at` in addition to `schema_version`. Publication +queries the recorded GitHub Actions run and rejects a stale SHA, non-main or +non-push run, failed conclusion, repository mismatch, or non-approved decision. + +External evaluation is conditional. When the evaluated workflow subject or its +explicit evaluation policy changed since the previous release tag, also set: + ```bash export PLANR_RELEASE_EVAL_SUITE="$HOME/projects/planr-evals/suites/planr-lean-skills-dogfood.suite.json" export PLANR_RELEASE_EVAL_RECEIPT=/path/to/sanitized-release-eval-receipt.json export PLANR_RELEASE_EVAL_DB=/path/to/planr-evals/results/eval.sqlite -scripts/release.sh 1.2.0 "one-line release summary" +export PLANR_RELEASE_PLANR_BIN=/path/to/reviewed/candidate/planr ``` Maintainer benchmarks, baselines, model/effort runs, and results live outside the public repository in `~/projects/planr-evals`; that workspace and its exact -layout are not a Planr runtime contract. All three paths above are explicit so a +layout are not a Planr runtime contract. All external evaluation paths above are explicit so a release cannot silently use the product repository's ordinary `.planr` database or a stale bundled suite. The receipt is a short-lived local pointer containing only `schema_version`, comparison and candidate-run identities, suite and @@ -76,7 +95,7 @@ The two scripts enforce, in order: 4. the candidate build synchronizes `Cargo.lock`, then regenerates and strictly checks both references without Git mutation; 5. candidate source, changelog, contracts, and generated files are committed and independently reviewed before publication approval; 6. publication requires clean `main`, the exact prepared versions/references, a committed changelog section, and no existing tag; -7. the candidate binary validates the sanitized receipt and recomputed comparison, then deterministic tests, package, and security gates run without changing source; +7. publication validates the exact-SHA CI and approval receipts; when the evaluated subject or policy changed, the reviewed candidate binary also validates the sanitized eval receipt and recomputed comparison; 8. publication creates and pushes only the annotated `vx.y.z` tag for that reviewed commit. Two independent gates back the script: @@ -145,6 +164,11 @@ scripts/ci-local.sh scripts/security-local.sh ``` +`scripts/security-local.sh`, `cargo audit --deny warnings`, and local +`zizmor .` are on-demand maintainer checks. Pull-request and push workflows do +not install or execute BetterLeaks, Trivy, TruffleHog, cargo-audit, zizmor, or +equivalent dependency/security scanners. + The external consumer E2E suite must pass when available on the release machine. Pull-request CI separately builds both Linux architectures through the canonical containerized release script, runs the full portability contract without diff --git a/package.json b/package.json index daef9aa..9bd53a5 100644 --- a/package.json +++ b/package.json @@ -29,12 +29,19 @@ "verify:pnpm-workspace": "node scripts/verify-pnpm-workspace.mjs", "verify:release-eval-gate": "node scripts/test-release-eval-gate.mjs", "verify:release-script": "node scripts/test-release-script.mjs", + "verify:change-classifier": "node scripts/test-verification-policy.mjs", + "verify:runner": "node scripts/test-verification-runner.mjs", + "verify:ci-router": "node scripts/test-ci-router.mjs", + "verify:docs-deployment": "node scripts/test-docs-deployment.mjs", + "classify:changes": "node scripts/classify-changes.mjs", + "verification:run": "node scripts/verification-runner.mjs run", + "verification:verify": "node scripts/verification-runner.mjs verify", "pack:check": "npm pack --dry-run", "docs:dev": "pnpm --filter @planr/docs dev", "docs:alchemy:dev": "pnpm --filter @planr/docs alchemy:dev", "docs:build": "pnpm --filter @planr/docs build", "docs:start": "pnpm --filter @planr/docs start", - "docs:deploy": "pnpm --filter @planr/docs run deploy", + "docs:deploy": "node scripts/deploy-docs.mjs", "docs:destroy": "pnpm --filter @planr/docs run destroy", "docs:content": "pnpm --filter @planr/docs content", "docs:typecheck": "pnpm --filter @planr/docs typecheck", diff --git a/plugins/planr/skills/planr-goal/SKILL.md b/plugins/planr/skills/planr-goal/SKILL.md index 283581d..f0219c3 100644 --- a/plugins/planr/skills/planr-goal/SKILL.md +++ b/plugins/planr/skills/planr-goal/SKILL.md @@ -30,7 +30,9 @@ planr plan check planr map build --from ``` -Fill required plan sections directly. Replace the placeholder task with typically 4-8 independently verifiable `TASK-00n` slices before `planr map build`. Preserve real execution order with `blocks` links. When registry routes use `work_type`, annotate tasks before mapping or retag them afterward; this is prep work, not a user question. +Fill required plan sections directly. Replace the placeholder task with independently verifiable `TASK-00n` slices before `planr map build`. A small coherent change is one implementation item plus one signal-bearing independent review; do not split mechanical stages into separate implementation/review pairs. Larger scopes still use multiple slices where ownership, dependencies, or independently observable outcomes genuinely differ. Preserve real execution order with `blocks` links. When registry routes use `work_type`, annotate tasks before mapping or retag them afterward; this is prep work, not a user question. + +When the repository provides a versioned verification policy and source-bound receipt runner, make that policy the verification owner in the plan. Record the selected profile, exact receipt path/digest, source revision, and the command that validates the receipt. Do not enumerate broad suites independently in every task when the policy already selects them. ## Durable Contract @@ -40,7 +42,7 @@ Store one contract per plan: planr context add "GOAL CONTRACT : DONE when every in-scope item is closed with log evidence, all reviews are complete, approvals are clear, and a live verification log proves . Iteration budget: 10." --tag goal-contract ``` -Never weaken it mid-run. Workers use `planr pick --plan `; termination uses `planr plan audit --json`. Reviews are required only where they add signal; evidence-backed setup work may close directly. +Never weaken it mid-run. Workers use `planr pick --plan `; termination uses `planr plan audit --json`. Reviews are required only where they add signal; evidence-backed setup work may close directly. Where deployment is in scope, the contract must retain human deployment approval and a bounded live oracle against the deployed result. ## Hand Off diff --git a/plugins/planr/skills/planr-loop/SKILL.md b/plugins/planr/skills/planr-loop/SKILL.md index 43a99af..8485dd1 100644 --- a/plugins/planr/skills/planr-loop/SKILL.md +++ b/plugins/planr/skills/planr-loop/SKILL.md @@ -17,12 +17,12 @@ Each iteration follows the Planr stage protocols: 1. `planr plan audit --json`; `holds: true` exits. 2. Use `$planr-plan` or `$planr-task-graph` only when scope or graph structure is missing. -3. Dispatch `$planr-work` for exactly one ready item scoped to ``; makers must use `planr pick --work-type code --plan `, never an unscoped pick, and finish implementation with `planr done ... --review`. -4. Run the target-platform oracle and record `planr log add --item --kind verification --summary ... --cmd ...`. -5. Dispatch `$planr-review`; findings create fix work, while `complete --close-target` settles the target. +3. Dispatch `$planr-work` for exactly one ready item scoped to ``; makers must use `planr pick --work-type code --plan `, never an unscoped pick, select the repository verification policy, and finish implementation with `planr done ... --review`. +4. Run the target-platform oracle when the goal requires one and record `planr log add --item --kind verification --summary ... --cmd ...`. Deployment still requires prior human approval and a bounded live oracle. +5. Dispatch `$planr-review`; the checker independently inspects the diff and validates the exact-source receipt, replaying only cheap, missing, failing, or explicitly high-risk evidence. Findings create fix work, while `complete --close-target` settles the target. 6. Repeat from audit. -One picked item per iteration. Use plain `done` only for low-signal setup/inspection work. Maker and checker stay separate when the host supports another agent; a maker never self-reviews when an independent checker is available, and never manufactures independence by changing worker identity. A worker may use `done --next`, which never returns its own review. +One picked item per iteration. A small coherent change stays one implementation item with one signal-bearing review; do not create a new review boundary for every mechanical stage or for an already-reviewed successful live smoke. Use plain `done` only for low-signal setup/inspection work. Maker and checker stay separate when the host supports another agent; a maker never self-reviews when an independent checker is available, and never manufactures independence by changing worker identity. The reviewer must exercise independent judgment even when it relies on a green receipt rather than replaying an expensive gate. A worker may use `done --next`, which never returns its own review. Pick packets explain null results and include `remaining`; follow their repair command. Destructive or out-of-repository effects require `planr approval request`. Two iterations without map movement must stop. On success or budget exhausted, finish with `$planr-summary`. @@ -41,7 +41,7 @@ For generated Codex roles: The `spawn_agent` tool call itself must include `agen ## Verification And Recovery -“Done” means the feature ran. For web dispatch `$planr-verify-web`; for CLI execute the built binary; for API use real requests; for iOS launch the simulator. Log the replayable command. If the capability is missing, record a blocker context, request approval, and pause—never fake proof. +“Done” means the feature ran. For web dispatch `$planr-verify-web`; for CLI execute the built binary; for API use real requests; for iOS launch the simulator. Log the replayable command. A passing bounded live oracle is evidence for the existing review boundary, not a reason to start another full reviewer replay. If the capability is missing, record a blocker context, request approval, and pause—never fake proof. Recovery starts in a fresh session with audit, map state, the stored contract, and the next scoped pick. Read [recovery and platform details](references/recovery-and-verification.md) only when that branch is active. diff --git a/plugins/planr/skills/planr-review/SKILL.md b/plugins/planr/skills/planr-review/SKILL.md index a971154..2e6a5ee 100644 --- a/plugins/planr/skills/planr-review/SKILL.md +++ b/plugins/planr/skills/planr-review/SKILL.md @@ -18,7 +18,9 @@ planr --json pick --work-type review `--work-type review` leases only review items, so a checker never accidentally takes maker work. Add `--plan ` when your dispatch names a plan so the lease stays inside that scope. The pick packet inlines the target item and its evidence logs under `target` — one command is enough to see what is being audited, its status (`in_review` while waiting on you), files, and verification commands. If you already hold a review id, `planr --json trace item ` returns the same packet. Use `planr log list --item ` or `planr map show --json` only for deeper reads. -Inspect the actual changed files and re-run the logged verification evidence. Then close the review exactly once: +Inspect the actual changed files and acceptance criteria, then independently judge whether the evidence proves them. When the repository owns a versioned verification policy, verify the logged receipt against its exact source revision, policy digest, changed-file digest, selected gates, command results, and artifact digests. Use the repository's receipt validator (for this repository, `npm run verification:verify -- --receipt --base --head `), not a visual read of JSON. + +Replay only evidence that is cheap, missing, failing, or explicitly high-risk. An already-green expensive gate bound to the reviewed source is normally validated from its receipt rather than rerun. Receipt validation does not replace judgment: inspect the diff for security, correctness, scope, and acceptance-criteria gaps, and record a finding when the policy selection or receipt is inadequate. Then close the review exactly once: ```bash planr review close --verdict complete --reviewer --close-target @@ -36,8 +38,10 @@ planr review close --verdict not-complete --reviewer --fin - Findings must be specific and actionable. - Missing tests are findings when acceptance criteria need proof. +- A stale, mismatched, unvalidated, or insufficiently scoped receipt is a finding. - Architecture or ownership drift is a finding when it creates duplicate policy or state owners. - If evidence is insufficient, use `--verdict unclear` rather than complete. +- Deployment remains gated by explicit approval and a bounded live oracle where applicable. A successful live smoke does not by itself require another broad build or a second full review; replay it only under the same cheap/missing/failing/explicitly-high-risk rule. ## Single-Agent Mode diff --git a/plugins/planr/skills/planr-verify-web/SKILL.md b/plugins/planr/skills/planr-verify-web/SKILL.md index d8e45f8..695d7b0 100644 --- a/plugins/planr/skills/planr-verify-web/SKILL.md +++ b/plugins/planr/skills/planr-verify-web/SKILL.md @@ -58,7 +58,9 @@ planr artifact add "verify-web screenshot" --item --path --path --kind video ``` -The replay command is mandatory. The reviewer reruns it instead of trusting this run; a verification that cannot be replayed is not evidence. +The replay command is mandatory. The reviewer validates the evidence and reruns it only when it is cheap, missing, failing, or explicitly high-risk; a verification that cannot be replayed when needed is not evidence. A successful bounded live smoke joins the existing coherent review boundary and does not automatically trigger another full build or reviewer replay. + +For a deployment oracle, require an approved deployment decision before the deploy begins. After deployment, keep the live check bounded to the changed routes, content, or interaction and record the deployed source/receipt identity in the summary. ## When Verification Is Impossible diff --git a/plugins/planr/skills/planr-work/SKILL.md b/plugins/planr/skills/planr-work/SKILL.md index 438266e..4343a7b 100644 --- a/plugins/planr/skills/planr-work/SKILL.md +++ b/plugins/planr/skills/planr-work/SKILL.md @@ -22,6 +22,15 @@ The pick output is one flat work packet — item, links, logs, runtime, recovery planr done --summary "what changed" --files path-a --files path-b --cmd "exact verification command" --tests "exact test command" --review ``` +Before choosing ad hoc checks, inspect the repository's versioned verification policy. When it supplies a runner, use that runner once for the changed-file set and preserve its exact-source receipt. For this repository the canonical flow is: + +```bash +npm run verification:run -- --receipt .planr/receipts/.json --base --head +npm run verification:verify -- --receipt .planr/receipts/.json --base --head +``` + +Record the receipt path, digest, source revision, selected profile/gates, and copy-paste replayable validation command in the completion evidence. Do not manually add broader suites that the selected policy does not require, and do not rerun an expensive green gate merely to hand work to the reviewer. + Put build/serve commands in `--cmd` and test runs in `--tests` — both are recorded as evidence. When the pick packet carries a `routing` block, also report the registry profile you actually ran on: add `--profile ` to `done`/`log add`, or export `PLANR_PROFILE` once per session. It is part of the evidence — a mismatch with the declared route is advisory (never blocks the close) and surfaces in `planr trace item` so silent host overrides get caught. Include the decisive output line in `--summary` (e.g. "12 tests passed", "GET /videos returned 3 entries"): reviewers see your recorded command strings, not your terminal, so the summary must carry what you observed, not just what you ran. Single-quote `--files` values that contain `$` (route files like `watch.$videoId.tsx`), or the shell expands them before planr sees them. `done --review` writes the completion log, requests the review, and moves the item to `in_review` (you keep ownership; it is waiting on the gate, not abandoned) — the response names the target's new status and the plan-scoped reviewer pick command; add `--next` to pick the following item in the same call. Without `--review` it closes the item directly (only for items that need no review gate). Running `done` on a ready item you never picked adopts it: the lease is written retroactively under your worker id so the review always has a maker. The response reports what your settlement `unlocked`, echoes the item's post condition, and hints when downstream work depends on an item closed without command/test evidence. Live verification (browser flow, executed binary, real requests) gets its own log kind so `plan audit` can find it: @@ -30,7 +39,7 @@ Live verification (browser flow, executed binary, real requests) gets its own lo planr log add --item --kind verification --summary "verified : " --cmd "" ``` -The `--cmd` value must be copy-paste replayable: a real shell command (or a small script you committed), never a prose transcript like "start server; curl /; check stats". A reviewer replays your command verbatim — if it cannot run, the verification cannot be independently confirmed. +The `--cmd` value must be copy-paste replayable: a real shell command (or a small script you committed), never a prose transcript like "start server; curl /; check stats". Reviewers validate the exact-source receipt and selectively replay cheap, missing, failing, or explicitly high-risk evidence; the command must still be runnable when that risk decision calls for replay. Log persistent evidence, not transient noise: a failure you immediately fixed belongs in the final log's narrative, not as a standalone failure log. Only record a failure separately when it blocks the item. @@ -59,6 +68,8 @@ planr approval request --reason "release approval" planr approval list --open ``` +Deployment is always such a gate: obtain approval before deployment and record a bounded live oracle afterward. Keep that oracle on the same coherent implementation/review boundary; a successful smoke does not automatically require another full build or review replay. + ## Rules - Do not work on multiple picked items unless the user explicitly asks. diff --git a/scripts/ci-router.mjs b/scripts/ci-router.mjs new file mode 100644 index 0000000..3f184e2 --- /dev/null +++ b/scripts/ci-router.mjs @@ -0,0 +1,138 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { classifyChanges, parseGitNameStatus } from "./verification-policy.mjs"; + +export const CI_JOB_GATES = Object.freeze({ + docs: Object.freeze(["docs-content", "docs-typecheck", "docs-lint", "docs-build", "docs-artifact"]), + quality: Object.freeze(["rust-fmt", "rust-clippy", "rust-test", "generated-reference"]), + release: Object.freeze(["github-actions", "release-contract", "release-evaluation"]), + linux_portability: Object.freeze(["linux-portability"]), +}); + +export function routeSelection(selection) { + if (!selection || !Array.isArray(selection.selectedGates)) throw new Error("verification selection is incomplete"); + const knownGates = new Set(Object.values(CI_JOB_GATES).flat()); + const unknown = selection.selectedGates.filter((gate) => !knownGates.has(gate)); + if (unknown.length > 0) throw new Error(`verification gates have no CI owner: ${unknown.join(", ")}`); + + const outputs = { + profile: requiredOutput(selection.profile, "profile"), + policy_version: requiredOutput(selection.policyVersion, "policy version"), + policy_digest: requiredOutput(selection.policyDigest, "policy digest"), + changed_files_digest: requiredOutput(selection.changedFilesDigest, "changed-files digest"), + live_browser: String(selection.liveVerification?.browser === true), + }; + for (const [job, gates] of Object.entries(CI_JOB_GATES)) { + outputs[job] = String(gates.some((gate) => selection.selectedGates.includes(gate))); + } + return outputs; +} + +export function assertSummary({ selected, results, routerResult = "success" }) { + if (routerResult !== "success") throw new Error(`router did not succeed: ${routerResult || "missing"}`); + for (const job of Object.keys(CI_JOB_GATES)) { + const expected = selected[job]; + const result = results[job] || "missing"; + if (expected === true && result !== "success") throw new Error(`selected CI job ${job} did not succeed: ${result}`); + if (expected === false && result !== "skipped") throw new Error(`unselected CI job ${job} was not intentionally skipped: ${result}`); + if (typeof expected !== "boolean") throw new Error(`CI selection is missing for job ${job}`); + } + return { verdict: "pass", jobs: Object.keys(CI_JOB_GATES).length }; +} + +function requiredOutput(value, label) { + if (typeof value !== "string" || value.length === 0 || /[\r\n]/u.test(value)) throw new Error(`${label} is missing or unsafe`); + return value; +} + +function parsePairs(values, label, transform = (value) => value) { + const parsed = {}; + for (const entry of values) { + const separator = entry.indexOf("="); + if (separator < 1) throw new Error(`invalid ${label}: ${entry}`); + const key = entry.slice(0, separator); + if (!(key in CI_JOB_GATES)) throw new Error(`unknown CI job in ${label}: ${key}`); + parsed[key] = transform(entry.slice(separator + 1)); + } + return parsed; +} + +function takeValues(args, name) { + const values = []; + for (let index = 0; index < args.length;) { + if (args[index] !== name) { + index += 1; + continue; + } + if (index === args.length - 1) throw new Error(`${name} requires a value`); + values.push(args[index + 1]); + args.splice(index, 2); + } + return values; +} + +function takeValue(args, name) { + const values = takeValues(args, name); + if (values.length > 1) throw new Error(`${name} may be specified only once`); + return values[0]; +} + +function selectionFromGit(base, head) { + try { + const output = execFileSync("git", ["diff", "--name-status", "-z", "--find-renames", base, head], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + return classifyChanges(parseGitNameStatus(output), { baseRevision: base, headRevision: head }); + } catch { + return classifyChanges(undefined, { baseRevision: base, headRevision: head }); + } +} + +function writeOutputs(outputPath, outputs) { + const lines = Object.entries(outputs).map(([key, value]) => `${key}=${value}`).join("\n"); + if (outputPath) appendFileSync(outputPath, `${lines}\n`, { encoding: "utf8" }); + process.stdout.write(`${lines}\n`); +} + +function main() { + const args = process.argv.slice(2); + const command = args.shift(); + if (command === "route") { + const base = takeValue(args, "--base"); + const head = takeValue(args, "--head") ?? "HEAD"; + const input = takeValue(args, "--input"); + const output = takeValue(args, "--github-output"); + const selectionPath = takeValue(args, "--selection-output"); + if (args.length > 0 || (!base && !input) || (base && input)) throw new Error("route requires exactly one of --base or --input"); + const selection = input + ? classifyChanges(JSON.parse(readFileSync(input, "utf8")).changes) + : selectionFromGit(base, head); + if (selectionPath) writeFileSync(selectionPath, `${JSON.stringify(selection, null, 2)}\n`, { mode: 0o600 }); + writeOutputs(output, routeSelection(selection)); + return; + } + if (command === "summary") { + const selected = parsePairs(takeValues(args, "--selected"), "selection", (value) => { + if (value !== "true" && value !== "false") throw new Error(`invalid selection boolean: ${value || "missing"}`); + return value === "true"; + }); + const results = parsePairs(takeValues(args, "--result"), "result"); + const routerResult = takeValue(args, "--router-result"); + if (args.length > 0) throw new Error(`unknown arguments: ${args.join(" ")}`); + process.stdout.write(`${JSON.stringify(assertSummary({ selected, results, routerResult }))}\n`); + return; + } + throw new Error("usage: ci-router.mjs ..."); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/classify-changes.mjs b/scripts/classify-changes.mjs new file mode 100644 index 0000000..b7e6d47 --- /dev/null +++ b/scripts/classify-changes.mjs @@ -0,0 +1,71 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { classifyChanges, parseGitNameStatus } from "./verification-policy.mjs"; + +const args = process.argv.slice(2); +const json = takeFlag("--json"); +const inputPath = takeValue("--input"); +const base = takeValue("--base"); +const head = takeValue("--head") ?? "HEAD"; + +let changes; +let selectionContext = {}; +try { + if (inputPath) { + const input = JSON.parse(readFileSync(inputPath, "utf8")); + changes = Array.isArray(input) ? input : input.changes; + if (!Array.isArray(input)) { + selectionContext = { baseRevision: input.baseRevision, headRevision: input.headRevision }; + } + } else if (base) { + const output = execFileSync("git", ["diff", "--name-status", "-z", "--find-renames", base, head], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + changes = parseGitNameStatus(output); + selectionContext = { baseRevision: base, headRevision: head }; + } else { + const input = JSON.parse(readFileSync(0, "utf8")); + changes = Array.isArray(input) ? input : input.changes; + if (!Array.isArray(input)) { + selectionContext = { baseRevision: input.baseRevision, headRevision: input.headRevision }; + } + } +} catch { + changes = undefined; +} + +if (args.length > 0) { + process.stderr.write(`Unknown arguments: ${args.join(" ")}\n`); + process.exit(2); +} + +const selection = classifyChanges(changes, selectionContext); +if (json) { + process.stdout.write(`${JSON.stringify(selection, null, 2)}\n`); +} else { + process.stdout.write(`profile=${selection.profile} policy=${selection.policyVersion} changed=${selection.changes.length}\n`); + for (const reason of selection.escalationReasons) { + process.stdout.write(`escalation=${reason.code}${reason.path ? ` path=${reason.path}` : ""} ${reason.detail}\n`); + } + for (const gate of selection.selectedGates) process.stdout.write(`gate=${gate}\n`); +} + +function takeFlag(name) { + const index = args.indexOf(name); + if (index === -1) return false; + args.splice(index, 1); + return true; +} + +function takeValue(name) { + const index = args.indexOf(name); + if (index === -1) return undefined; + if (index === args.length - 1) { + process.stderr.write(`${name} requires a value\n`); + process.exit(2); + } + const [, value] = args.splice(index, 2); + return value; +} diff --git a/scripts/deploy-docs.mjs b/scripts/deploy-docs.mjs new file mode 100644 index 0000000..b7c3ce5 --- /dev/null +++ b/scripts/deploy-docs.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +export function deploymentCommands({ receipt, input, head = 'HEAD', url = 'https://planr.so' }) { + if (!receipt || !input) throw new Error('docs deployment requires --receipt and --input'); + return [ + { + label: 'reviewed receipt', + executable: process.execPath, + args: ['scripts/verification-runner.mjs', 'verify', '--receipt', receipt, '--input', input, '--head', head], + }, + { + label: 'Alchemy production deployment', + executable: 'pnpm', + args: ['--filter', '@planr/docs', 'exec', 'alchemy', 'deploy', '--stage', 'prod', '--yes'], + env: { PLANR_DOCS_RECEIPT_VALIDATED: '1' }, + }, + { + label: 'bounded live oracle', + executable: process.execPath, + args: ['apps/docs/scripts/verify-live-deployment.mjs', '--url', url], + }, + ]; +} + +export function deployDocs(options, execute = executeCommand) { + const completed = []; + for (const command of deploymentCommands(options)) { + const result = execute(command.executable, command.args, { + cwd: repoRoot, + stdio: 'inherit', + env: { ...process.env, ...command.env }, + }); + if (result.status !== 0) throw new Error(`${command.label} failed with exit code ${result.status ?? 'unknown'}`); + completed.push(command.label); + } + return completed; +} + +function executeCommand(executable, args, options) { + return spawnSync(executable, args, options); +} + +function valueAfter(name) { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : undefined; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const completed = deployDocs({ + receipt: valueAfter('--receipt'), + input: valueAfter('--input'), + head: valueAfter('--head') ?? 'HEAD', + url: valueAfter('--url') ?? 'https://planr.so', + }); + process.stdout.write(`${JSON.stringify({ verdict: 'pass', completed })}\n`); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/fixtures/verification-policy/cases.json b/scripts/fixtures/verification-policy/cases.json new file mode 100644 index 0000000..b0a76e3 --- /dev/null +++ b/scripts/fixtures/verification-policy/cases.json @@ -0,0 +1,16 @@ +[ + {"name":"docs-only","changes":[{"status":"M","path":"apps/docs/content/docs/agents/quickstart.mdx"}],"profile":"focused-docs","classes":["docs-content"],"included":["docs-content","docs-build","docs-artifact"],"excluded":["rust-test"],"browser":false}, + {"name":"interactive-docs","changes":[{"status":"M","path":"apps/docs/components/command-block.tsx"}],"profile":"docs","classes":["docs-interactive"],"included":["docs-build"],"excluded":["rust-test"],"browser":true}, + {"name":"static-docs-image","changes":[{"status":"M","path":"apps/docs/public/agents/pi.svg"}],"profile":"docs","classes":["docs-app"],"included":["docs-artifact"],"excluded":["rust-test"],"browser":false}, + {"name":"rust","changes":[{"status":"M","path":"src/app/application.rs"}],"profile":"core","classes":["rust"],"included":["rust-test","generated-reference"],"excluded":["docs-build"],"browser":false}, + {"name":"mixed","changes":[{"status":"M","path":"README.md"},{"status":"M","path":"src/model.rs"}],"profile":"core","classes":["docs-content","rust"],"included":["docs-content","rust-test"],"browser":false}, + {"name":"release-workflow","changes":[{"status":"M","path":".github/workflows/ci.yml"}],"profile":"release-critical","classes":["release-critical"],"included":["github-actions","release-contract"],"excluded":["docs-build"],"browser":false}, + {"name":"lockfile","changes":[{"status":"M","path":"pnpm-lock.yaml"}],"profile":"full","classes":["lockfile"],"escalation":"lockfile_path","browser":false}, + {"name":"generated-reference","changes":[{"status":"M","path":"apps/docs/content/docs/reference/cli-generated.mdx"}],"profile":"full","classes":["generated-reference"],"escalation":"generated-reference_path","browser":false}, + {"name":"deleted-docs","changes":[{"status":"D","path":"apps/docs/content/docs/retired.mdx"}],"profile":"focused-docs","classes":["docs-content"],"included":["docs-build"],"browser":false}, + {"name":"renamed-docs","changes":[{"status":"R100","oldPath":"apps/docs/content/docs/faq.mdx","newPath":"apps/docs/content/docs/troubleshooting.mdx"}],"profile":"focused-docs","classes":["docs-content"],"browser":false}, + {"name":"renamed-to-interactive","changes":[{"status":"R100","oldPath":"apps/docs/content/docs/widget.mdx","newPath":"apps/docs/components/widget.tsx"}],"profile":"docs","classes":["docs-content","docs-interactive"],"browser":true}, + {"name":"unknown","changes":[{"status":"A","path":"mystery/new-owner.xyz"}],"profile":"full","classes":["unknown"],"escalation":"unknown_path","browser":false}, + {"name":"sensitive-overlap","changes":[{"status":"M","path":"README.md"},{"status":"M","path":"scripts/release.sh"}],"profile":"full","classes":["docs-content","release-critical"],"escalation":"sensitive_overlap","browser":false}, + {"name":"ambiguous","changes":[{"status":"U","path":"src/model.rs"}],"profile":"full","classes":[],"escalation":"ambiguous_status","browser":false} +] diff --git a/scripts/release.sh b/scripts/release.sh index 5e07f9c..feab267 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -53,37 +53,24 @@ for manifest in \ fi done -# Recheck generated, version-derived references and frozen workspace state. -# Every command must leave the reviewed commit byte-clean. -pnpm install --frozen-lockfile -pnpm --filter @planr/docs reference:check -cargo build --quiet -if [ -n "$(git status --porcelain)" ]; then - echo "release verification changed the reviewed candidate source" >&2 - exit 1 -fi - -eval_receipt="${PLANR_RELEASE_EVAL_RECEIPT:-}" -eval_suite="${PLANR_RELEASE_EVAL_SUITE:-}" -eval_db="${PLANR_RELEASE_EVAL_DB:-}" -if [ -z "$eval_receipt" ] || [ -z "$eval_suite" ] || [ -z "$eval_db" ]; then - echo "PLANR_RELEASE_EVAL_RECEIPT, PLANR_RELEASE_EVAL_SUITE, and PLANR_RELEASE_EVAL_DB are required" >&2 +ci_receipt="${PLANR_RELEASE_CI_RECEIPT:-}" +approval="${PLANR_RELEASE_APPROVAL:-}" +if [ -z "$ci_receipt" ] || [ -z "$approval" ]; then + echo "PLANR_RELEASE_CI_RECEIPT and PLANR_RELEASE_APPROVAL are required" >&2 exit 1 fi -node scripts/verify-release-eval-receipt.mjs \ - --receipt "$eval_receipt" \ - --db "$eval_db" \ - --suite "$eval_suite" \ - --planr-bin target/debug/planr -cargo test -npm run verify:release-eval-gate -npm pack --dry-run -scripts/security-local.sh -if [ -n "$(git status --porcelain)" ]; then - echo "release gates changed the reviewed candidate source" >&2 - exit 1 -fi +# Promote independent exact-SHA evidence. Evaluation evidence is required by +# the verifier only when the evaluated subject or its explicit policy changed. +set -- \ + --version "$version" \ + --ci-receipt "$ci_receipt" \ + --approval "$approval" +if [ -n "${PLANR_RELEASE_EVAL_RECEIPT:-}" ]; then set -- "$@" --eval-receipt "$PLANR_RELEASE_EVAL_RECEIPT"; fi +if [ -n "${PLANR_RELEASE_EVAL_SUITE:-}" ]; then set -- "$@" --eval-suite "$PLANR_RELEASE_EVAL_SUITE"; fi +if [ -n "${PLANR_RELEASE_EVAL_DB:-}" ]; then set -- "$@" --eval-db "$PLANR_RELEASE_EVAL_DB"; fi +if [ -n "${PLANR_RELEASE_PLANR_BIN:-}" ]; then set -- "$@" --planr-bin "$PLANR_RELEASE_PLANR_BIN"; fi +node scripts/verify-release-promotion.mjs "$@" git tag -a "v$version" -m "planr v$version: $summary" git push origin HEAD "v$version" diff --git a/scripts/test-ci-router.mjs b/scripts/test-ci-router.mjs new file mode 100644 index 0000000..c827cbe --- /dev/null +++ b/scripts/test-ci-router.mjs @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { assertSummary, routeSelection } from "./ci-router.mjs"; +import { classifyChanges, POLICY_DIGEST, POLICY_VERSION } from "./verification-policy.mjs"; + +const docsSelection = classifyChanges([{ status: "M", path: "apps/docs/content/docs/agents/quickstart.mdx" }]); +assert.deepEqual(routeSelection(docsSelection), { + profile: "focused-docs", + policy_version: POLICY_VERSION, + policy_digest: POLICY_DIGEST, + changed_files_digest: docsSelection.changedFilesDigest, + live_browser: "false", + docs: "true", + quality: "false", + release: "false", + linux_portability: "false", +}); + +const releaseRoute = routeSelection(classifyChanges([{ status: "M", path: ".github/workflows/ci.yml" }])); +assert.equal(releaseRoute.docs, "false"); +assert.equal(releaseRoute.quality, "false"); +assert.equal(releaseRoute.release, "true"); +assert.equal(releaseRoute.linux_portability, "true"); +assert.equal(releaseRoute.live_browser, "false"); +const interactiveRoute = routeSelection(classifyChanges([{ status: "M", path: "apps/docs/components/tabs.tsx" }])); +assert.equal(interactiveRoute.docs, "true"); +assert.equal(interactiveRoute.live_browser, "true"); +const fullRoute = routeSelection(classifyChanges([{ status: "M", path: "scripts/ci-router.mjs" }])); +for (const job of ["docs", "quality", "release", "linux_portability"]) assert.equal(fullRoute[job], "true"); +assert.throws( + () => routeSelection({ ...docsSelection, selectedGates: [...docsSelection.selectedGates, "ownerless-gate"] }), + /no CI owner/u, +); + +const selected = { docs: true, quality: false, release: false, linux_portability: false }; +const passingResults = { docs: "success", quality: "skipped", release: "skipped", linux_portability: "skipped" }; +assert.deepEqual(assertSummary({ selected, results: passingResults }), { verdict: "pass", jobs: 4 }); +for (const result of ["missing", "skipped", "cancelled", "failure"]) { + assert.throws( + () => assertSummary({ selected, results: { ...passingResults, docs: result } }), + new RegExp(`selected CI job docs did not succeed: ${result}`, "u"), + ); +} +assert.throws( + () => assertSummary({ selected, results: { ...passingResults, quality: "success" } }), + /was not intentionally skipped/u, +); +assert.throws(() => assertSummary({ selected, results: passingResults, routerResult: "failure" }), /router did not succeed/u); + +const temp = mkdtempSync(path.join(os.tmpdir(), "planr-ci-router-")); +try { + const fixture = path.join(temp, "changes.json"); + const output = path.join(temp, "github-output.txt"); + const selection = path.join(temp, "selection.json"); + writeFileSync(fixture, JSON.stringify({ changes: [{ status: "M", path: "README.md" }] })); + const result = spawnSync(process.execPath, [ + new URL("./ci-router.mjs", import.meta.url).pathname, + "route", "--input", fixture, "--github-output", output, "--selection-output", selection, + ], { encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr); + assert.match(readFileSync(output, "utf8"), /^docs=true$/mu); + assert.match(readFileSync(output, "utf8"), /^quality=false$/mu); + assert.equal(JSON.parse(readFileSync(selection, "utf8")).profile, "focused-docs"); +} finally { + rmSync(temp, { recursive: true, force: true }); +} + +console.log("ci_router=passed docs_only_skips=quality,release,linux-portability summary_fail_closed=missing,skipped,cancelled,failure"); diff --git a/scripts/test-docs-deployment.mjs b/scripts/test-docs-deployment.mjs new file mode 100644 index 0000000..3e60e90 --- /dev/null +++ b/scripts/test-docs-deployment.mjs @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import { deployDocs, deploymentCommands } from './deploy-docs.mjs'; +import { LIVE_DOCS_ORACLE, verifyLiveDeployment } from '../apps/docs/scripts/verify-live-deployment.mjs'; + +const options = { + receipt: '.planr/receipts/docs.json', + input: '.planr/ci/selection.json', + head: '0123456789abcdef0123456789abcdef01234567', + url: 'https://planr.so', +}; +const commands = deploymentCommands(options); +assert.equal(commands.filter(({ args }) => args.includes('deploy')).length, 1, 'promotion performs exactly one deployment'); +assert.equal(commands.filter(({ args }) => args.includes('build')).length, 0, 'promotion never starts another build'); +assert.equal(commands.find(({ args }) => args.includes('deploy')).env.PLANR_DOCS_RECEIPT_VALIDATED, '1'); +assert.deepEqual(commands.map(({ label }) => label), ['reviewed receipt', 'Alchemy production deployment', 'bounded live oracle']); + +const calls = []; +assert.deepEqual(deployDocs(options, (executable, args) => { + calls.push([executable, ...args]); + return { status: 0 }; +}), commands.map(({ label }) => label)); +assert.equal(calls.length, 3); +assert.throws( + () => deployDocs(options, (_executable, args) => ({ status: args.includes('verify') ? 1 : 0 })), + /reviewed receipt failed/, + 'a stale or invalid receipt stops promotion before deploy', +); + +const requested = []; +const observations = await verifyLiveDeployment('https://planr.so', { + fetchImpl: async (url) => { + requested.push(url.pathname); + const route = LIVE_DOCS_ORACLE.find(({ path }) => path === url.pathname); + return new Response(route.markers.join('\n'), { status: 200, headers: { 'content-type': `${route.type}; charset=utf-8` } }); + }, +}); +assert.deepEqual(requested, LIVE_DOCS_ORACLE.map(({ path }) => path)); +assert.equal(observations.length, LIVE_DOCS_ORACLE.length); +assert.ok(LIVE_DOCS_ORACLE.length <= 5, 'live promotion oracle stays intentionally bounded'); + +console.log(JSON.stringify({ + verdict: 'pass', + production_builds: 0, + alchemy_deploys: 1, + live_routes: observations.length, + receipt_failures_stop_before_deploy: true, +}, null, 2)); diff --git a/scripts/test-planr-risk-based-guidance.mjs b/scripts/test-planr-risk-based-guidance.mjs new file mode 100644 index 0000000..46b8224 --- /dev/null +++ b/scripts/test-planr-risk-based-guidance.mjs @@ -0,0 +1,33 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const read = (name) => readFileSync(path.join(root, "plugins/planr/skills", name, "SKILL.md"), "utf8"); + +const goal = read("planr-goal"); +const loop = read("planr-loop"); +const work = read("planr-work"); +const review = read("planr-review"); +const web = read("planr-verify-web"); + +assert.match(goal, /small coherent change is one implementation item plus one signal-bearing independent review/u); +assert.match(goal, /versioned verification policy and source-bound receipt runner/u); +assert.match(loop, /cheap, missing, failing, or explicitly high-risk evidence/u); +assert.match(loop, /maker never self-reviews when an independent checker is available/u); +assert.match(loop, /Keep one active write item/u); +assert.match(work, /npm run verification:run -- --receipt/u); +assert.match(work, /receipt path, digest, source revision, selected profile\/gates/u); +assert.match(review, /npm run verification:verify -- --receipt/u); +assert.match(review, /Receipt validation does not replace judgment/u); +assert.match(review, /Never export a second identity/u); +assert.match(web, /approved deployment decision before the deploy begins/u); +assert.match(web, /does not automatically trigger another full build or reviewer replay/u); + +for (const [name, contents] of [["loop", loop], ["review", review], ["web", web]]) { + assert.doesNotMatch(contents, /reviewer reruns (?:it|the logged verification evidence)/iu, `${name} must not require unconditional replay`); +} + +process.stdout.write("planr risk-based guidance contract: ok (5 skills)\n"); diff --git a/scripts/test-release-eval-gate.mjs b/scripts/test-release-eval-gate.mjs index e2ba081..1b3eacd 100644 --- a/scripts/test-release-eval-gate.mjs +++ b/scripts/test-release-eval-gate.mjs @@ -144,8 +144,8 @@ const staleCandidateSource = spawnSync(process.execPath, [ assert.notEqual(staleCandidateSource.status, 0, "receipt must fail after any release source file changes"); const release = fs.readFileSync(path.join(repo, "scripts/release.sh"), "utf8"); -const gateIndex = release.indexOf("node scripts/verify-release-eval-receipt.mjs"); -assert.ok(gateIndex > release.indexOf("cargo build --quiet"), "eval gate must run after candidate build"); +const gateIndex = release.indexOf("node scripts/verify-release-promotion.mjs"); +assert.ok(gateIndex >= 0, "publication must verify exact-SHA promotion evidence"); for (const mutation of ["git tag ", "git push "]) { assert.ok(gateIndex < release.indexOf(mutation), `eval gate must precede ${mutation.trim()}`); } @@ -155,9 +155,9 @@ for (const forbiddenMutation of ["git add ", "git commit "]) { for (const forbidden of ["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "NPM_TOKEN", "raw_prompt", "raw_completion"]) { assert.ok(!release.includes(forbidden), `release path must not request ${forbidden}`); } -assert.ok(!release.includes("PLANR_RELEASE_PLANR_BIN"), "release gate must execute the freshly built candidate binary"); -assert.ok(release.includes("PLANR_RELEASE_EVAL_SUITE"), "release path must require an explicit external suite"); -assert.ok(release.includes("PLANR_RELEASE_EVAL_DB"), "release path must require an explicit external eval database"); +assert.ok(release.includes("PLANR_RELEASE_PLANR_BIN"), "conditional eval verification must use an explicit reviewed candidate binary"); +assert.ok(release.includes("PLANR_RELEASE_EVAL_SUITE"), "conditional eval path must accept an explicit external suite"); +assert.ok(release.includes("PLANR_RELEASE_EVAL_DB"), "conditional eval path must accept an explicit external eval database"); const privateSuitePath = ["examples", "eval", "lean-skills"].join("/"); for (const source of [release, fs.readFileSync(verifier, "utf8"), fs.readFileSync(fileURLToPath(import.meta.url), "utf8")]) { assert.ok(!source.includes(privateSuitePath), "public gate code must not depend on the private lean-skills path"); diff --git a/scripts/test-release-script.mjs b/scripts/test-release-script.mjs index 29f8a0f..6967b74 100644 --- a/scripts/test-release-script.mjs +++ b/scripts/test-release-script.mjs @@ -7,10 +7,15 @@ import path from "node:path"; import process from "node:process"; import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; +import { routeSelection } from "./ci-router.mjs"; +import { classifyChanges } from "./verification-policy.mjs"; const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const prepareSource = fs.readFileSync(path.join(repo, "scripts/prepare-release-candidate.sh"), "utf8"); const releaseSource = fs.readFileSync(path.join(repo, "scripts/release.sh"), "utf8"); +const promotionSource = fs.readFileSync(path.join(repo, "scripts/verify-release-promotion.mjs"), "utf8"); +const ciRouterSource = fs.readFileSync(path.join(repo, "scripts/ci-router.mjs"), "utf8"); +const verificationPolicySource = fs.readFileSync(path.join(repo, "scripts/verification-policy.mjs"), "utf8"); const changelogLinksSource = fs.readFileSync(path.join(repo, "scripts/verify-changelog-release-links.sh"), "utf8"); const repositoryVersion = JSON.parse(fs.readFileSync(path.join(repo, "package.json"), "utf8")).version; const version = "9.9.9"; @@ -128,6 +133,9 @@ function run(name, script, prepared, env = {}) { PLANR_RELEASE_EVAL_RECEIPT: path.join(test.root, "receipt.json"), PLANR_RELEASE_EVAL_SUITE: path.join(test.root, "suite.json"), PLANR_RELEASE_EVAL_DB: path.join(test.root, "eval.sqlite"), + PLANR_RELEASE_PLANR_BIN: path.join(test.root, "planr"), + PLANR_RELEASE_CI_RECEIPT: path.join(test.root, "ci-receipt.json"), + PLANR_RELEASE_APPROVAL: path.join(test.root, "approval.json"), }, }); const calls = fs.readFileSync(test.log, "utf8").trim().split("\n").filter(Boolean); @@ -225,14 +233,7 @@ assert.ok(!unprepared.calls.some((call) => call.startsWith("git tag") || call.st const released = run("release-pass", "release.sh", true); assert.equal(released.result.status, 0, released.result.stderr); const expected = [ - "pnpm install --frozen-lockfile", - "pnpm --filter @planr/docs reference:check", - "cargo build --quiet", - "node scripts/verify-release-eval-receipt.mjs", - "cargo test", - "npm run verify:release-eval-gate", - "npm pack --dry-run", - "security-local", + "node scripts/verify-release-promotion.mjs", `git tag -a v${version}`, `git push origin HEAD v${version}`, ]; @@ -242,6 +243,181 @@ for (const prefix of expected) { assert.ok(index > cursor, `missing or reordered publication command: ${prefix}`); cursor = index; } +for (const repeatedGate of ["pnpm install", "cargo build", "cargo test", "npm pack", "security-local"]) { + assert.ok(!released.calls.some((call) => call.startsWith(repeatedGate)), `publication must not replay ${repeatedGate}`); +} + +const promotionRoot = path.join(tmp, "promotion-verifier"); +const promotionBin = path.join(promotionRoot, "bin"); +const sourceSha = "a".repeat(40); +const baseSha = "d".repeat(40); +write(path.join(promotionRoot, "scripts/verify-release-promotion.mjs"), promotionSource); +write(path.join(promotionRoot, "scripts/ci-router.mjs"), ciRouterSource); +write(path.join(promotionRoot, "scripts/verification-policy.mjs"), verificationPolicySource); +write(path.join(promotionBin, "git"), `#!/bin/sh +set -eu +case "$1 \${2:-}" in + "rev-parse --verify") + case "\${3:-}" in HEAD*) printf '%s\\n' "$SOURCE_SHA" ;; *) printf '%s\\n' "$BASE_SHA" ;; esac + ;; + "merge-base --is-ancestor") ;; + "describe --tags") printf 'v9.9.8\\n' ;; + "diff --name-only") printf '%s\\n' "\${DIFF_PATHS:-README.md}" ;; + "diff --name-status") printf 'M\\0%s\\0' "\${DIFF_PATH:-README.md}" ;; + *) echo "unexpected git command: $*" >&2; exit 2 ;; +esac +`, 0o755); +write(path.join(promotionBin, "gh"), `#!/bin/sh +set -eu +if [ "$1" = "api" ]; then + printf '{"id":123,"run_attempt":1,"name":"CI","event":"push","head_branch":"main","head_sha":"%s","conclusion":"%s","repository":{"full_name":"instructa/planr"}}\\n' "$SOURCE_SHA" "\${GH_CONCLUSION:-success}" + exit 0 +fi +if [ "$1 $2" = "run download" ]; then + destination="" + while [ "$#" -gt 0 ]; do + if [ "$1" = "--dir" ]; then destination="$2"; shift 2; continue; fi + shift + done + cp "$AUTHENTIC_RECEIPT" "$destination/promotion-receipt.json" + exit 0 +fi +echo "unexpected gh command: $*" >&2 +exit 2 +`, 0o755); +const fixtureSelection = classifyChanges([{ status: "M", path: "README.md" }], { + baseRevision: baseSha, + headRevision: sourceSha, +}); +const fixtureRouting = routeSelection(fixtureSelection); +const ciReceipt = { + schema_version: "planr.ci-promotion-receipt.v1", + repository: "instructa/planr", + workflow: "CI", + run_id: "123", + run_attempt: "1", + event: "push", + source_ref: "refs/heads/main", + source_base_sha: baseSha, + source_sha: sourceSha, + conclusion: "success", + policy: { + profile: fixtureSelection.profile, + version: fixtureSelection.policyVersion, + digest: fixtureSelection.policyDigest, + changed_files_digest: fixtureSelection.changedFilesDigest, + }, + jobs: Object.fromEntries(Object.entries(fixtureRouting) + .filter(([key]) => ["docs", "quality", "release", "linux_portability"].includes(key)) + .map(([key, selected]) => [key, selected === "true" ? "success" : "skipped"])), +}; +const approval = { + schema_version: "planr.release-approval.v1", + approval_id: "approval-123", + source_sha: sourceSha, + version, + decision: "approved", + approved_by: "maintainer@example.invalid", + approved_at: new Date(Date.now() - 60_000).toISOString(), +}; +write(path.join(promotionRoot, "ci.json"), `${JSON.stringify(ciReceipt)}\n`); +write(path.join(promotionRoot, "approval.json"), `${JSON.stringify(approval)}\n`); +function verifyPromotion({ receipt = "ci.json", authenticReceipt = "ci.json", env = {} } = {}) { + return spawnSync(process.execPath, [ + "scripts/verify-release-promotion.mjs", + "--version", version, + "--ci-receipt", receipt, + "--approval", "approval.json", + ], { + cwd: promotionRoot, + encoding: "utf8", + env: { + ...process.env, + ...env, + PATH: `${promotionBin}:${process.env.PATH}`, + SOURCE_SHA: sourceSha, + BASE_SHA: baseSha, + AUTHENTIC_RECEIPT: path.join(promotionRoot, authenticReceipt), + }, + }); +} +const promoted = verifyPromotion(); +assert.equal(promoted.status, 0, promoted.stderr); +assert.equal(JSON.parse(promoted.stdout).evaluation, "not_required"); +const failedCi = verifyPromotion({ env: { GH_CONCLUSION: "failure" } }); +assert.notEqual(failedCi.status, 0, "authoritative failed CI must block promotion"); +const evalRequired = verifyPromotion({ env: { DIFF_PATHS: "plugins/planr/skills/planr-loop/SKILL.md" } }); +assert.notEqual(evalRequired.status, 0, "evaluated-subject changes must require external eval evidence"); + +function rejectBoundMutation(name, mutate, errorPattern, message) { + const candidate = mutate(structuredClone(ciReceipt)); + const file = `${name}.json`; + write(path.join(promotionRoot, file), `${JSON.stringify(candidate)}\n`); + const result = verifyPromotion({ receipt: file, authenticReceipt: file }); + assert.notEqual(result.status, 0, message); + assert.match(result.stderr, errorPattern, `${name} must fail for its binding check`); +} +rejectBoundMutation("forged-policy", (receipt) => { + receipt.policy.digest = `sha256:${"b".repeat(64)}`; + return receipt; +}, /policy digest is stale/u, "forged current-run policy digest must fail closed"); +rejectBoundMutation("forged-policy-version", (receipt) => { + receipt.policy.version = "0.0.0"; + return receipt; +}, /policy version is stale/u, "forged current-run policy version must fail closed"); +rejectBoundMutation("forged-changes", (receipt) => { + receipt.policy.changed_files_digest = `sha256:${"c".repeat(64)}`; + return receipt; +}, /changed-files digest mismatch/u, "forged current-run changed-files digest must fail closed"); +rejectBoundMutation("forged-profile", (receipt) => { + receipt.policy.profile = "core"; + return receipt; +}, /profile mismatch/u, "forged current-run profile must fail closed"); +rejectBoundMutation("forged-jobs", (receipt) => { + receipt.jobs.docs = "skipped"; + return receipt; +}, /does not match the current policy selection/u, "selected job recorded as skipped must fail closed"); +rejectBoundMutation("copied-receipt", (receipt) => { + receipt.source_sha = "f".repeat(40); + return receipt; +}, /does not bind the current candidate SHA/u, "receipt copied from another source SHA must fail closed"); + +const unauthenticatedReceipt = structuredClone(ciReceipt); +unauthenticatedReceipt.policy.digest = `sha256:${"e".repeat(64)}`; +write(path.join(promotionRoot, "unauthenticated-local.json"), `${JSON.stringify(unauthenticatedReceipt)}\n`); +const unauthenticated = verifyPromotion({ receipt: "unauthenticated-local.json", authenticReceipt: "ci.json" }); +assert.notEqual(unauthenticated.status, 0, "locally forged receipt must not replace the run artifact"); +assert.match(unauthenticated.stderr, /does not match the authenticated run artifact/u); + +const writtenReceipt = path.join(tmp, "written-promotion-receipt.json"); +const eventPath = path.join(tmp, "push-event.json"); +write(eventPath, `${JSON.stringify({ before: baseSha })}\n`); +const writeReceiptResult = spawnSync(process.execPath, ["scripts/write-ci-promotion-receipt.mjs", writtenReceipt], { + cwd: repo, + encoding: "utf8", + env: { + ...process.env, + GITHUB_REPOSITORY: "instructa/planr", + GITHUB_WORKFLOW: "CI", + GITHUB_RUN_ID: "123", + GITHUB_RUN_ATTEMPT: "1", + GITHUB_EVENT_NAME: "push", + GITHUB_REF: "refs/heads/main", + GITHUB_SHA: sourceSha, + GITHUB_EVENT_PATH: eventPath, + PLANR_PROFILE: fixtureSelection.profile, + PLANR_POLICY_VERSION: fixtureSelection.policyVersion, + PLANR_POLICY_DIGEST: fixtureSelection.policyDigest, + PLANR_CHANGED_FILES_DIGEST: fixtureSelection.changedFilesDigest, + PLANR_DOCS_RESULT: ciReceipt.jobs.docs, + PLANR_QUALITY_RESULT: ciReceipt.jobs.quality, + PLANR_RELEASE_RESULT: ciReceipt.jobs.release, + PLANR_LINUX_RESULT: ciReceipt.jobs.linux_portability, + }, +}); +assert.equal(writeReceiptResult.status, 0, writeReceiptResult.stderr); +assert.equal(JSON.parse(fs.readFileSync(writtenReceipt, "utf8")).source_sha, sourceSha); +assert.equal(JSON.parse(fs.readFileSync(writtenReceipt, "utf8")).source_base_sha, baseSha); fs.rmSync(tmp, { recursive: true, force: true }); console.log(JSON.stringify({ @@ -251,4 +427,8 @@ console.log(JSON.stringify({ reviewed_source_mutation_during_publication: false, candidate_git_mutation: false, fail_closed_cases: ["lockfile drift", "reference drift", "stale changelog comparison", "missing changelog comparison", "unprepared version"], + exact_sha_promotion: true, + repeated_publication_gates: 0, + conditional_external_evaluation: true, + rejected_receipt_binding_regressions: 7, }, null, 2)); diff --git a/scripts/test-verification-policy.mjs b/scripts/test-verification-policy.mjs new file mode 100644 index 0000000..d4223e9 --- /dev/null +++ b/scripts/test-verification-policy.mjs @@ -0,0 +1,97 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { + classifyChanges, + GATES, + parseGitNameStatus, + POLICY_DIGEST, + POLICY_RULES, + policyDigestForRules, +} from "./verification-policy.mjs"; + +const fixtureUrl = new URL("./fixtures/verification-policy/cases.json", import.meta.url); +const fixtures = JSON.parse(readFileSync(fixtureUrl, "utf8")); + +for (const fixture of fixtures) { + const selection = classifyChanges(fixture.changes); + assert.equal(selection.profile, fixture.profile, `${fixture.name}: profile`); + assert.deepEqual(selection.matchedPathClasses, fixture.classes, `${fixture.name}: path classes`); + for (const gate of fixture.included ?? []) { + assert.ok(selection.selectedGates.includes(gate), `${fixture.name}: expected gate ${gate}`); + } + for (const gate of fixture.excluded ?? []) { + assert.ok(!selection.selectedGates.includes(gate), `${fixture.name}: excluded gate ${gate}`); + } + if (fixture.escalation) { + assert.ok(selection.escalationReasons.some(({ code }) => code === fixture.escalation), `${fixture.name}: escalation reason`); + } + assert.equal(selection.reasons.length, selection.selectedGates.length, `${fixture.name}: every gate has a reason`); + assert.ok(selection.reasons.every(({ gate, detail }) => gate && detail), `${fixture.name}: reasons are explanatory`); + assert.equal(selection.liveVerification.browser, fixture.browser, `${fixture.name}: browser selection`); + if (fixture.browser) { + assert.ok(selection.liveVerification.paths.length > 0, `${fixture.name}: browser paths are explicit`); + } +} + +assert.equal("security" in GATES, false, "automatic security scanning is not a verification gate"); + +const deterministicA = classifyChanges([ + { status: "M", path: "README.md" }, + { status: "M", path: "src/model.rs" }, +]); +const deterministicB = classifyChanges([ + { status: "modified", path: "src/model.rs" }, + { status: "modified", path: "README.md" }, +]); +assert.equal(deterministicA.policyDigest, POLICY_DIGEST); +assert.equal(deterministicA.changedFilesDigest, deterministicB.changedFilesDigest, "change digest is order-independent"); +assert.deepEqual(deterministicA.selectedGates, deterministicB.selectedGates, "selection is order-independent"); + +const remappedRules = structuredClone(POLICY_RULES); +const docsContentRule = remappedRules.find(({ id }) => id === "docs-content"); +docsContentRule.matchers[0].source = "^only-this-path-would-be-docs-content$"; +assert.notEqual( + policyDigestForRules(remappedRules), + POLICY_DIGEST, + "changing a path-to-profile matcher changes policy identity", +); + +const revisionBound = classifyChanges([{ status: "M", path: "README.md" }], { + baseRevision: "main", + headRevision: "HEAD", +}); +assert.equal(revisionBound.baseRevision, "main"); +assert.equal(revisionBound.headRevision, "HEAD"); + +for (const runnerPath of ["scripts/verification-runner.mjs", "scripts/test-verification-runner.mjs"]) { + const runnerSelection = classifyChanges([{ status: "M", path: runnerPath }]); + assert.equal(runnerSelection.profile, "full", `${runnerPath}: runner changes require full verification`); + assert.deepEqual(runnerSelection.matchedPathClasses, ["policy"], `${runnerPath}: runner is owned by policy infrastructure`); +} + +assert.deepEqual(parseGitNameStatus("M\0README.md\0R100\0src/old.rs\0src/new.rs\0"), [ + { status: "M", path: "README.md" }, + { status: "R", oldPath: "src/old.rs", newPath: "src/new.rs" }, +]); + +for (const invalid of [ + undefined, + [], + [{ status: "M", path: "../escape" }], + [{ status: "R", oldPath: "README.md" }], + [{ status: "AA", path: "src/model.rs" }], + [{ status: "Modified-ish", path: "src/model.rs" }], +]) { + const selection = classifyChanges(invalid); + assert.equal(selection.profile, "full", "invalid input fails closed"); + assert.equal(selection.escalatedToFull, true, "invalid input records escalation"); +} + +// This suite imports the pure classifier directly. Gate identifiers are data; +// no command runner or child process is reachable from classifier tests. +console.log(JSON.stringify({ + verdict: "pass", + fixtures: fixtures.length, + expensive_gate_commands_run: 0, + policy_digest: POLICY_DIGEST, +}, null, 2)); diff --git a/scripts/test-verification-runner.mjs b/scripts/test-verification-runner.mjs new file mode 100644 index 0000000..7616b3a --- /dev/null +++ b/scripts/test-verification-runner.mjs @@ -0,0 +1,289 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { classifyChanges, POLICY_DIGEST } from "./verification-policy.mjs"; +import { + commandPlanFor, + linuxTargetCommandPlan, + runLinuxTargetVerification, + runVerification, + verifyLinuxTargetReceipt, + verifyReceipt, +} from "./verification-runner.mjs"; + +const root = mkdtempSync(path.join(tmpdir(), "planr-verification-runner-")); +process.on("exit", () => rmSync(root, { recursive: true, force: true })); +mkdirSync(path.join(root, "apps/docs/components"), { recursive: true }); +mkdirSync(path.join(root, "apps/docs/out"), { recursive: true }); +writeFileSync(path.join(root, "apps/docs/components/card.tsx"), "export const Card = () => null;\n"); +writeFileSync(path.join(root, "apps/docs/out/index.html"), "

Planr

\n"); +writeFileSync(path.join(root, "README.md"), "# Fixture\n"); +git("init", "-q"); +git("config", "user.name", "Planr Test"); +git("config", "user.email", "planr-test@example.invalid"); +git("add", "."); +git("commit", "-qm", "fixture"); + +const docsSelection = classifyChanges([{ status: "M", path: "apps/docs/components/card.tsx" }], { + baseRevision: "HEAD^", + headRevision: "HEAD", +}); +const calls = []; +const receipt = runVerification({ + selection: docsSelection, + repoRoot: root, + execute(executable, args) { + calls.push([executable, ...args]); + return { status: 0 }; + }, +}); + +assert.equal(receipt.verdict, "pass"); +assert.equal(receipt.policy.digest, POLICY_DIGEST); +assert.equal(calls.filter((call) => call.join(" ").includes("@planr/docs build")).length, 1, "docs build runs exactly once"); +assert.equal(calls.filter((call) => call.join(" ").includes("cargo install")).length, 0, "docs work never runs release-profile cargo install"); +assert.equal(calls.filter((call) => call.join(" ").includes("security:check")).length, 0, "docs work never runs local security tooling"); +assert.equal(receipt.selection.liveVerification.browser, true, "interactive docs receipts retain the live-browser decision"); +assert.equal(new Set(calls.map(JSON.stringify)).size, calls.length, "runner de-duplicates exact commands"); +assert.deepEqual(verifyReceipt(receipt, { selection: docsSelection, repoRoot: root }).verdict, "pass"); +const receiptPath = ".planr/receipts/verification-receipt.json"; +mkdirSync(path.dirname(path.join(root, receiptPath)), { recursive: true }); +writeFileSync(path.join(root, receiptPath), `${JSON.stringify(receipt)}\n`); +assert.deepEqual( + verifyReceipt(receipt, { selection: docsSelection, repoRoot: root, receiptPath }).verdict, + "pass", + "the explicitly supplied receipt output is not treated as a source input", +); +rmSync(path.join(root, receiptPath)); + +writeFileSync(path.join(root, "README.md"), `${JSON.stringify(receipt)}\n`); +assert.throws( + () => verifyReceipt(receipt, { selection: docsSelection, repoRoot: root, receiptPath: "README.md" }), + /receipt path must be a JSON file directly under \.planr\/receipts/, + "a tracked source path cannot be disguised as the receipt output exclusion", +); +writeFileSync(path.join(root, "README.md"), "# Fixture\n"); + +const receiptAliasPath = ".planr/receipts/alias.json"; +symlinkSync(path.join(root, "README.md"), path.join(root, receiptAliasPath)); +assert.throws( + () => verifyReceipt(receipt, { selection: docsSelection, repoRoot: root, receiptPath: receiptAliasPath }), + /receipt path must not contain symbolic-link aliases/, + "a symbolic-link alias cannot disguise a source path as receipt output", +); +rmSync(path.join(root, receiptAliasPath)); + +writeFileSync(path.join(root, receiptPath), "{}\n"); +assert.throws( + () => verifyReceipt(receipt, { selection: docsSelection, repoRoot: root, receiptPath }), + /receipt path content does not match/, + "the excluded receipt output must contain the exact receipt being verified", +); +rmSync(path.join(root, receiptPath)); + +writeFileSync(path.join(root, "README.md"), "# Dirty fixture\n"); +assert.throws( + () => runVerification({ selection: docsSelection, repoRoot: root, execute: () => ({ status: 0 }) }), + /source worktree must be clean/, + "an unselected tracked modification invalidates source binding", +); +writeFileSync(path.join(root, "README.md"), "# Fixture\n"); +writeFileSync(path.join(root, "unselected-input.txt"), "dirty\n"); +assert.throws( + () => runVerification({ selection: docsSelection, repoRoot: root, execute: () => ({ status: 0 }) }), + /source worktree must be clean/, + "an unselected untracked input invalidates source binding", +); +rmSync(path.join(root, "unselected-input.txt")); + +assert.throws( + () => runVerification({ + selection: docsSelection, + repoRoot: root, + execute() { + writeFileSync(path.join(root, "created-during-gates.txt"), "dirty\n"); + return { status: 0 }; + }, + }), + /source worktree must be clean/, + "a gate cannot create an unbound source input and still emit a green receipt", +); +rmSync(path.join(root, "created-during-gates.txt")); + +const stalePolicy = structuredClone(receipt); +stalePolicy.policy.digest = `sha256:${"0".repeat(64)}`; +assert.throws(() => verifyReceipt(stalePolicy, { selection: docsSelection, repoRoot: root }), /policy digest is stale/); +const missingArtifact = structuredClone(receipt); +missingArtifact.artifacts = []; +assert.throws(() => verifyReceipt(missingArtifact, { selection: docsSelection, repoRoot: root }), /artifact set mismatch/); + +writeFileSync(path.join(root, "apps/docs/components/card.tsx"), "export const Card = () => 'altered';\n"); +assert.throws(() => verifyReceipt(receipt, { selection: docsSelection, repoRoot: root }), /source worktree must be clean/); +writeFileSync(path.join(root, "apps/docs/components/card.tsx"), "export const Card = () => null;\n"); +writeFileSync(path.join(root, "apps/docs/out/index.html"), "

Altered

\n"); +assert.throws(() => verifyReceipt(receipt, { selection: docsSelection, repoRoot: root }), /artifact changed/); +writeFileSync(path.join(root, "apps/docs/out/index.html"), "

Planr

\n"); + +for (const mutation of [ + (value) => ({ ...value, environment: { PATH: process.env.PATH } }), + (value) => ({ ...value, raw_prompt: "private prompt" }), + (value) => ({ ...value, raw_completion: "private completion" }), + (value) => ({ ...value, credential: "xai-example-secret-value" }), + (value) => ({ ...value, diagnostics: { path: "examples/eval/private-suite.json" } }), +]) { + assert.throws(() => verifyReceipt(mutation(structuredClone(receipt)), { selection: docsSelection, repoRoot: root }), /allowlisted|forbidden|private path/); +} + +const fullSelection = classifyChanges([{ status: "M", path: "scripts/verification-policy.mjs" }]); +const plan = commandPlanFor(fullSelection); +assert.equal(plan.filter(({ executable, args }) => [executable, ...args].join(" ").includes("@planr/docs build")).length, 1); +assert.equal(plan.filter(({ executable, args }) => [executable, ...args].join(" ").includes("cargo install")).length, 0); +assert.equal(new Set(plan.map((entry) => JSON.stringify([entry.executable, entry.args]))).size, plan.length); + +const releaseSelection = classifyChanges([{ status: "M", path: "scripts/release.sh" }]); +const releasePlan = commandPlanFor(releaseSelection); +for (const candidatePlan of [releasePlan, plan]) { + const linuxCommands = candidatePlan.filter(({ gate }) => gate === "linux-portability"); + assert.equal(linuxCommands.length, 1, "the candidate host only executes the aggregate Linux checksum command"); + assert.equal( + linuxCommands.filter(({ executable, args }) => executable === "sh" && args.join(" ").includes("sha256sum -c SHA256SUMS")).length, + 1, + "Linux portability records exactly one aggregate checksum command", + ); +} +assert.equal(releaseSelection.profile, "release-critical"); +assert.equal(releasePlan.some(({ args }) => args.includes("security:check")), false, "release CI never runs local security tooling"); +assert.equal(new Set(releasePlan.map((entry) => JSON.stringify([entry.executable, entry.args]))).size, releasePlan.length); + +const targetDefinitions = [ + ["linux-x86_64", "x86_64-unknown-linux-musl", "x64"], + ["linux-arm64", "aarch64-unknown-linux-musl", "arm64"], +]; +for (const [target, cargoTarget] of targetDefinitions) { + const targetPlan = linuxTargetCommandPlan(target); + assert.equal(targetPlan.length, 2, `${target} has one build and one verifier`); + assert.ok(targetPlan.every(({ executable, args }) => + executable === "env" + && args.includes(`PLANR_TARGET=${target}`) + && args.includes(`PLANR_CARGO_TARGET=${cargoTarget}`)), `${target} commands carry explicit target bindings`); + assert.deepEqual( + targetPlan.map(({ args }) => args.at(-1)), + ["scripts/build-linux-release.sh", "scripts/verify-linux-release-artifact.sh"], + `${target} build and verification order is deterministic`, + ); + assert.equal(new Set(targetPlan.map(({ executable, args }) => JSON.stringify([executable, args]))).size, 2); +} + +let incompatibleHostCalls = 0; +assert.throws( + () => runLinuxTargetVerification({ + selection: releaseSelection, + target: "linux-arm64", + repoRoot: root, + host: { platform: "linux", architecture: "x64" }, + execute: () => { incompatibleHostCalls += 1; return { status: 0 }; }, + }), + /requires native linux\/arm64/, + "a target pair cannot start on an incompatible native host", +); +assert.equal(incompatibleHostCalls, 0); +assert.throws( + () => runVerification({ selection: releaseSelection, repoRoot: root, execute: () => ({ status: 0 }) }), + /both independent Linux target receipts are required/, + "zero target receipts and no dist artifacts cannot produce a green release receipt", +); + +mkdirSync(path.join(root, "dist"), { recursive: true }); +const archiveContents = new Map([ + ["linux-x86_64", "independent-x86_64-archive\n"], + ["linux-arm64", "independent-arm64-archive\n"], +]); +for (const [target, contents] of archiveContents) writeFileSync(path.join(root, `dist/planr-${target}.tar.gz`), contents); +const checksumContents = "arm64-digest planr-linux-arm64.tar.gz\nx86_64-digest planr-linux-x86_64.tar.gz\n"; +writeFileSync(path.join(root, "dist/SHA256SUMS"), checksumContents); + +const linuxTargetReceipts = targetDefinitions.map(([target, , architecture]) => runLinuxTargetVerification({ + selection: releaseSelection, + target, + repoRoot: root, + host: { platform: "linux", architecture }, + execute: () => ({ status: 0 }), +})); +for (const targetReceipt of linuxTargetReceipts) { + assert.equal(verifyLinuxTargetReceipt(targetReceipt, { selection: releaseSelection, repoRoot: root }).verdict, "pass"); +} +const releaseReceipt = runVerification({ + selection: releaseSelection, + repoRoot: root, + linuxTargetReceipts, + execute: () => ({ status: 0 }), +}); +assert.equal(releaseReceipt.linuxTargets.length, 2, "both independent target receipts join one candidate receipt"); +assert.deepEqual(releaseReceipt.linuxTargets.map(({ target }) => target), ["linux-x86_64", "linux-arm64"]); +assert.deepEqual( + releaseReceipt.artifacts.map(({ path: artifactPath }) => artifactPath), + ["dist/planr-linux-x86_64.tar.gz", "dist/planr-linux-arm64.tar.gz", "dist/SHA256SUMS"], + "promotion evidence binds both archives and the aggregate checksum file", +); +assert.equal(verifyReceipt(releaseReceipt, { selection: releaseSelection, repoRoot: root }).verdict, "pass"); + +const swappedArtifacts = structuredClone(linuxTargetReceipts); +swappedArtifacts[0].artifact = structuredClone(swappedArtifacts[1].artifact); +assert.throws( + () => runVerification({ selection: releaseSelection, repoRoot: root, linuxTargetReceipts: swappedArtifacts, execute: () => ({ status: 0 }) }), + /linux-x86_64 artifact path mismatch/, + "target receipts cannot swap archive identities", +); + +rmSync(path.join(root, "dist/planr-linux-x86_64.tar.gz")); +assert.throws(() => verifyReceipt(releaseReceipt, { selection: releaseSelection, repoRoot: root }), /artifact is missing/); +writeFileSync(path.join(root, "dist/planr-linux-x86_64.tar.gz"), archiveContents.get("linux-x86_64")); +writeFileSync(path.join(root, "dist/planr-linux-arm64.tar.gz"), "tampered-arm64\n"); +assert.throws(() => verifyReceipt(releaseReceipt, { selection: releaseSelection, repoRoot: root }), /linux-arm64 artifact changed/); +writeFileSync(path.join(root, "dist/planr-linux-arm64.tar.gz"), archiveContents.get("linux-arm64")); +rmSync(path.join(root, "dist/SHA256SUMS")); +assert.throws(() => verifyReceipt(releaseReceipt, { selection: releaseSelection, repoRoot: root }), /artifact is missing: dist\/SHA256SUMS/); +writeFileSync(path.join(root, "dist/SHA256SUMS"), "tampered-checksums\n"); +assert.throws(() => verifyReceipt(releaseReceipt, { selection: releaseSelection, repoRoot: root }), /artifact changed: dist\/SHA256SUMS/); +writeFileSync(path.join(root, "dist/SHA256SUMS"), checksumContents); +rmSync(path.join(root, "dist"), { recursive: true, force: true }); + +const historicalRevision = git("rev-parse", "HEAD").trim(); +writeFileSync(path.join(root, "README.md"), "# New revision\n"); +git("add", "README.md"); +git("commit", "-qm", "advance fixture head"); +const historicalSelection = classifyChanges([{ status: "M", path: "apps/docs/components/card.tsx" }], { + baseRevision: `${historicalRevision}^`, + headRevision: historicalRevision, +}); +assert.throws( + () => runVerification({ selection: historicalSelection, repoRoot: root, execute: () => ({ status: 0 }) }), + /selected source revision .* is not checked out at HEAD/, + "a historical --head cannot claim gates executed from the current checkout", +); + +console.log(JSON.stringify({ + verdict: "pass", + docs_commands: calls.length, + docs_build_commands: 1, + docs_cargo_install_commands: 0, + full_commands: plan.length, + release_commands: releasePlan.length, + linux_commands_per_candidate_profile: 1, + independent_linux_target_commands: 4, + release_receipt_commands: releaseReceipt.commands.length, + bound_linux_artifacts: releaseReceipt.artifacts.length, + rejected_incompatible_linux_hosts: 1, + rejected_missing_linux_compositions: 1, + rejected_linux_evidence_mutations: 5, + rejected_sensitive_receipts: 5, + rejected_dirty_worktrees: 3, + rejected_unbound_receipt_paths: 3, + rejected_historical_heads: 1, +}, null, 2)); + +function git(...args) { + return execFileSync("git", args, { cwd: root, encoding: "utf8" }); +} diff --git a/scripts/test-verify-github-actions.mjs b/scripts/test-verify-github-actions.mjs index eb2d570..9d3bf86 100644 --- a/scripts/test-verify-github-actions.mjs +++ b/scripts/test-verify-github-actions.mjs @@ -12,7 +12,9 @@ const fixtureWorkflows = path.join(fixtureRoot, ".github", "workflows"); const verifier = path.join(fixtureScripts, "verify-github-actions.mjs"); const releaseWorkflow = path.join(fixtureWorkflows, "release.yml"); const ciWorkflow = path.join(fixtureWorkflows, "ci.yml"); +const linuxReceiptsWorkflow = path.join(fixtureWorkflows, "linux-receipts.yml"); const securityWorkflow = path.join(fixtureWorkflows, "security.yml"); +const fixturePackageJson = path.join(fixtureRoot, "package.json"); const linuxBuildScript = path.join(fixtureScripts, "build-linux-release.sh"); const linuxBuilderDockerfile = path.join(fixtureScripts, "linux-release-builder.Dockerfile"); const linuxVerifyScript = path.join(fixtureScripts, "verify-linux-release-artifact.sh"); @@ -20,7 +22,6 @@ const publicLifecycleScript = path.join(fixtureScripts, "verify-public-lifecycle const buildReleaseScript = path.join(fixtureScripts, "build-release.sh"); const prepareReleaseScript = path.join(fixtureScripts, "prepare-release-candidate.sh"); const releaseScript = path.join(fixtureScripts, "release.sh"); -const trivyIgnoreFile = path.join(fixtureRoot, ".trivyignore.yaml"); const localSecurityScript = path.join(fixtureScripts, "security-local.sh"); function runVerifier() { @@ -41,13 +42,13 @@ try { await cp(path.join(repoRoot, "scripts", "prepare-release-candidate.sh"), prepareReleaseScript); await cp(path.join(repoRoot, "scripts", "release.sh"), releaseScript); await cp(path.join(repoRoot, "scripts", "security-local.sh"), localSecurityScript); - await cp(path.join(repoRoot, ".trivyignore.yaml"), trivyIgnoreFile); + await cp(path.join(repoRoot, "package.json"), fixturePackageJson); await cp(path.join(repoRoot, ".github", "workflows"), fixtureWorkflows, { recursive: true }); const baseline = runVerifier(); assert.equal(baseline.status, 0, `baseline workflow fixture must pass:\n${baseline.stderr}`); - const fixtureFiles = [releaseWorkflow, ciWorkflow, securityWorkflow, linuxBuildScript, linuxBuilderDockerfile, linuxVerifyScript, publicLifecycleScript, buildReleaseScript, prepareReleaseScript, releaseScript, localSecurityScript, trivyIgnoreFile]; + const fixtureFiles = [releaseWorkflow, ciWorkflow, linuxReceiptsWorkflow, linuxBuildScript, linuxBuilderDockerfile, linuxVerifyScript, publicLifecycleScript, buildReleaseScript, prepareReleaseScript, releaseScript, localSecurityScript, fixturePackageJson]; const baselineSources = new Map( await Promise.all(fixtureFiles.map(async (file) => [file, await readFile(file, "utf8")])), ); @@ -205,91 +206,53 @@ try { "incomplete aggregate checksums", ); await expectRejected( - securityWorkflow, - (value) => value.replace("7105f1cd6577f058a9e39d0578f1a99c8a1e481e4d3512cd8a09acfe22a0fdc0", "0".repeat(64)), - /must pin the reviewed TruffleHog release digest/u, - "mutated TruffleHog binary digest", + linuxReceiptsWorkflow, + (value) => value.replace("runner: ubuntu-24.04-arm", "runner: ubuntu-24.04"), + /must bind linux-arm64 to ubuntu-24\.04-arm/u, + "incompatible native receipt host", ); await expectRejected( - securityWorkflow, - (value) => value.replace("8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9", "0".repeat(64)), - /must pin the reviewed Trivy release digest/u, - "mutated Trivy binary digest", + linuxReceiptsWorkflow, + (value) => value.replace(" .planr/receipts/${{ matrix.target }}.json\n", ""), + /each native target upload must retain its runner receipt/u, + "missing native target receipt upload", ); await expectRejected( - securityWorkflow, - (value) => value.replace(" --results=verified --fail --no-update --github-actions", " --results=verified --no-update --github-actions"), - /must fail closed while scanning verified secrets/u, - "non-blocking TruffleHog scan", - ); - await expectRejected( - securityWorkflow, - (value) => value.replace(" --results=verified --fail", " --fail"), - /must fail closed while scanning verified secrets/u, - "unverified-only TruffleHog contract removed", - ); - await expectRejected( - securityWorkflow, - (value) => value.replace("--scanners secret,misconfig", "--scanners secret"), - /must scan secrets and misconfigurations/u, - "Trivy misconfiguration scan removed", - ); - await expectRejected( - securityWorkflow, - (value) => value.replace(" - name: Install pinned security scanners\n", " - uses: trufflesecurity/trufflehog@6f3c981e7b77f235fd2702dd74af25fc4b72bf11 # v3.96.0\n\n - name: Install pinned security scanners\n"), - /uses unreviewed action trufflesecurity\/trufflehog|GitHub-owned-only repository action policy/u, - "disallowed third-party Security action", - ); - await expectRejected( - trivyIgnoreFile, - (value) => value.replaceAll("scripts/linux-release-builder.Dockerfile", "**"), - /exceptions must remain limited to the two expiring build-only Dockerfile findings/u, - "globally broadened Trivy exceptions", - ); - await expectRejected( - trivyIgnoreFile, - (value) => `${value} - id: AVD-DS-0001\n`, - /exceptions must remain limited to the two expiring build-only Dockerfile findings/u, - "additional Trivy exception", - ); - await expectRejected( - trivyIgnoreFile, - (value) => value.replaceAll(" expired_at: 2027-07-26\n", ""), - /exceptions must remain limited to the two expiring build-only Dockerfile findings/u, - "missing Trivy exception expiry", - ); - await expectRejected( - securityWorkflow, - (value) => value.replace("a8000f3c683319a523d3b20df0e75457ba591f049cfcbfa98966631b56733c03", "0".repeat(64)), - /must pin the reviewed zizmor release digest/u, - "mutated zizmor binary digest", - ); - await expectRejected( - securityWorkflow, - (value) => value.replaceAll("v1.24.1", "v1.24.2"), - /must pin the reviewed zizmor release URL/u, - "mutated zizmor version", + linuxReceiptsWorkflow, + (value) => value.replace( + / node scripts\/verification-runner\.mjs verify-linux-target \\\n --receipt \.planr\/receipts\/linux-arm64\.json \\\n --input \.planr\/ci\/selection\.json \\\n --head "\$GITHUB_SHA"\n/u, + "", + ), + /must replay exactly two target receipts/u, + "missing aggregate target receipt replay", ); await expectRejected( - securityWorkflow, - (value) => value.replace(" npm run verify:github-actions\n", " npm run verify:github-actions\n python3 -m pip install --user uv\n uvx zizmor==1.24.1 .\n"), - /must not install mutable uv or zizmor inputs/u, - "mutable uv and zizmor installation", + ciWorkflow, + (value) => `${value}\n# cargo audit --deny warnings\n`, + /must not run automatic security, secret, or dependency scanners/u, + "automatic cargo audit", ); await expectRejected( - securityWorkflow, - (value) => value.replace(" --skip-check-update \\\n", ""), - /must use the checks bundled with the reviewed binary/u, - "unpinned Trivy checks update", + ciWorkflow, + (value) => `${value}\n# trivy fs --scanners vuln,secret .\n`, + /must not run automatic security, secret, or dependency scanners/u, + "automatic Trivy scan", ); await expectRejected( - localSecurityScript, - (value) => value.replace(" --skip-check-update \\\n", ""), - /Local Trivy must use the checks bundled with the reviewed binary/u, - "unpinned local Trivy checks update", - ); + ciWorkflow, + (value) => `${value}\n# pnpm docs:verify-shell\n`, + /retired blanket browser suite/u, + "automatic blanket browser suite", + ); + await resetFixtures(); + await writeFile(securityWorkflow, "name: Security\non:\n pull_request:\njobs:\n scan:\n runs-on: ubuntu-latest\n steps:\n - run: trivy fs .\n"); + const addedSecurityWorkflow = runVerifier(); + assert.notEqual(addedSecurityWorkflow.status, 0, "a new automatic security workflow must fail verification"); + assert.match(`${addedSecurityWorkflow.stdout}\n${addedSecurityWorkflow.stderr}`, /must not run automatic|must remain absent/u); + await rm(securityWorkflow); + adversarialCases += 1; - console.log(`github_actions_regression=passed adversarial_cases=${adversarialCases} same_runner_smoke_insufficient=true musl_native_pins_lifecycle_checksums_npm_fail_closed=true security_jobs_fail_closed=true immutable_security_toolchain=true`); + console.log(`github_actions_regression=passed adversarial_cases=${adversarialCases} same_runner_smoke_insufficient=true musl_native_pins_lifecycle_checksums_npm_fail_closed=true automatic_scanners_absent=true docs_build_once=true`); } finally { await rm(fixtureRoot, { recursive: true, force: true }); } diff --git a/scripts/verification-policy.mjs b/scripts/verification-policy.mjs new file mode 100644 index 0000000..a4264b4 --- /dev/null +++ b/scripts/verification-policy.mjs @@ -0,0 +1,360 @@ +import { createHash } from "node:crypto"; + +export const POLICY_VERSION = "1.1.0"; + +export const GATES = Object.freeze({ + "docs-content": "Validate generated docs content and links", + "docs-typecheck": "Type-check the documentation application", + "docs-lint": "Lint the documentation application", + "docs-build": "Build the documentation application once", + "docs-artifact": "Verify the existing documentation artifact", + "rust-fmt": "Check Rust formatting", + "rust-clippy": "Run strict Rust lints", + "rust-test": "Run the Rust test suite", + "generated-reference": "Check generated CLI and MCP reference pages", + "github-actions": "Verify GitHub Actions contracts", + "release-contract": "Verify release and packaging contracts", + "linux-portability": "Verify portable Linux release artifacts", + "release-evaluation": "Verify the synthetic release evaluation contract", +}); + +const PROFILES = deepFreeze({ + "focused-docs": ["docs-content", "docs-typecheck", "docs-lint", "docs-build", "docs-artifact"], + docs: ["docs-content", "docs-typecheck", "docs-lint", "docs-build", "docs-artifact"], + core: [ + "rust-fmt", "rust-clippy", "rust-test", "generated-reference", + ], + "release-critical": [ + "github-actions", "release-contract", "linux-portability", + "release-evaluation", + ], +}); + +const FULL_GATES = Object.freeze([...new Set(Object.values(PROFILES).flat())]); + +export const POLICY_RULES = deepFreeze([ + { + id: "policy", + description: "Classifier, package graph, and policy fixtures", + profile: "full", + sensitive: true, + detail: "The verification policy or command graph changed.", + matchers: [ + { source: "^(scripts\\/(?:verification-(?:policy|runner)|classify-changes|test-verification-(?:policy|runner)|ci-router|test-ci-router)\\.mjs|scripts\\/fixtures\\/verification-policy\\/|package\\.json$|pnpm-workspace\\.yaml$)", flags: "u" }, + ], + }, + { + id: "lockfile", + description: "Dependency lockfiles", + profile: "full", + sensitive: true, + detail: "A dependency lockfile changed.", + matchers: [{ source: "^(?:Cargo\\.lock|pnpm-lock\\.yaml)$", flags: "u" }], + }, + { + id: "generated-reference", + description: "Generated reference output", + profile: "full", + sensitive: true, + detail: "Generated reference output changed and must be checked against its owning source.", + matchers: [{ source: "^apps\\/docs\\/content\\/docs\\/reference\\/(?:cli-generated|mcp-schemas-generated)\\.mdx$", flags: "u" }], + }, + { + id: "release-critical", + description: "Workflows, release, packaging, and local security tooling", + profile: "release-critical", + sensitive: true, + detail: "Release, workflow, packaging, contract, or security infrastructure changed.", + matchers: [ + { source: "^(?:\\.github\\/|npm\\/|docs\\/contracts\\/|docs\\/RELEASE\\.md$|CHANGELOG\\.md$|scripts\\/(?:release|build-release|build-linux-release|prepare-release-candidate|verify-linux-release-artifact|verify-release|test-release|verify-github-actions|test-verify-github-actions|security-local|check-repository-privacy|install|generate-formula|verify-changelog-release-links))", flags: "u" }, + ], + }, + { + id: "rust", + description: "Rust sources, manifests, and tests", + profile: "core", + sensitive: false, + detail: "Rust product code, tests, or its manifest changed.", + matchers: [{ source: "^(?:src\\/.*\\.rs|tests\\/.*\\.rs|Cargo\\.toml)$", flags: "u" }], + }, + { + id: "docs-interactive", + description: "Interactive documentation application code", + profile: "docs", + sensitive: false, + liveBrowser: true, + detail: "Interactive documentation behavior changed; request one focused live browser oracle outside automatic CI.", + matchers: [ + { source: "^apps/docs/(?:app|components)/.*[.](?:js|jsx|mjs|ts|tsx)$", flags: "u" }, + ], + }, + { + id: "docs-app", + description: "Documentation application and verification code", + profile: "docs", + sensitive: false, + detail: "Documentation application or verification code changed.", + matchers: [ + { source: "^apps\\/docs\\/(?:app|components|lib|scripts|public)\\/", flags: "u" }, + { source: "^apps\\/docs\\/(?:package\\.json|next\\.config\\.mjs|eslint\\.config\\.mjs|tsconfig\\.json|source\\.config\\.ts|postcss\\.config\\.mjs|wrangler\\.jsonc|alchemy\\.run\\.ts)$", flags: "u" }, + ], + }, + { + id: "docs-content", + description: "Documentation prose and static content", + profile: "focused-docs", + sensitive: false, + detail: "Documentation content changed.", + matchers: [{ source: "^(?:apps\\/docs\\/content\\/|docs\\/(?!contracts\\/)|README\\.md$|LICENSE\\.md$)", flags: "u" }], + }, + { + id: "unknown", + description: "Paths without a policy owner", + profile: "full", + sensitive: true, + detail: "No verification policy rule owns this path.", + matchers: [], + }, +]); + +const COMPILED_RULES = POLICY_RULES.map((rule) => ({ + rule, + matchers: rule.matchers.map(({ source, flags }) => new RegExp(source, flags)), +})); + +const POLICY_DOCUMENT = deepFreeze({ + schemaVersion: 1, + policyVersion: POLICY_VERSION, + profiles: PROFILES, + rules: POLICY_RULES, +}); + +export const POLICY_DIGEST = digest(POLICY_DOCUMENT); + +export function policyDigestForRules(rules) { + return digest({ + schemaVersion: POLICY_DOCUMENT.schemaVersion, + policyVersion: POLICY_VERSION, + profiles: PROFILES, + rules, + }); +} + +const STATUS_ALIASES = Object.freeze({ + A: "added", + added: "added", + M: "modified", + modified: "modified", + D: "deleted", + deleted: "deleted", + T: "type_changed", + type_changed: "type_changed", + R: "renamed", + renamed: "renamed", + C: "copied", + copied: "copied", +}); + +export function classifyChanges(input, { baseRevision = null, headRevision = null } = {}) { + const normalized = normalizeChanges(input); + const pathMatches = []; + const escalationReasons = [...normalized.errors]; + + for (const change of normalized.changes) { + for (const path of change.paths) { + const match = classifyPath(path); + pathMatches.push({ path, status: change.status, ...match }); + if (match.profile === "full") { + escalationReasons.push({ code: `${match.pathClass}_path`, path, detail: match.detail }); + } + } + } + + const matchedClasses = [...new Set(pathMatches.map((match) => match.pathClass))].sort(); + const sensitive = pathMatches.filter((match) => match.sensitive); + const nonSensitive = pathMatches.filter((match) => !match.sensitive); + if (sensitive.length > 0 && nonSensitive.length > 0) { + escalationReasons.push({ + code: "sensitive_overlap", + paths: [...new Set(pathMatches.map((match) => match.path))].sort(), + detail: "Sensitive and non-sensitive path classes overlap; full verification is required.", + }); + } + + const escalatedToFull = escalationReasons.length > 0; + const profile = escalatedToFull ? "full" : selectProfile(pathMatches); + const selectedGates = profile === "full" ? [...FULL_GATES] : gatesForMatches(pathMatches); + const liveBrowserPaths = pathMatches.filter((match) => match.liveBrowser).map((match) => match.path); + const reasons = selectedGates.map((gate) => { + const owners = pathMatches.filter((match) => profile === "full" || PROFILES[match.profile]?.includes(gate)); + return { + gate, + code: profile === "full" ? "full_profile" : "path_class_match", + detail: profile === "full" + ? "Selected because fail-closed full verification is required." + : GATES[gate], + paths: [...new Set(owners.map((owner) => owner.path))].sort(), + }; + }); + + return { + schemaVersion: 1, + policyVersion: POLICY_VERSION, + policyDigest: POLICY_DIGEST, + baseRevision: safeRevision(baseRevision), + headRevision: safeRevision(headRevision), + changedFilesDigest: digest(normalized.changes), + profile, + escalatedToFull, + escalationReasons: deduplicateReasons(escalationReasons), + matchedPathClasses: matchedClasses, + selectedGates, + liveVerification: { + browser: liveBrowserPaths.length > 0, + paths: [...new Set(liveBrowserPaths)].sort(), + detail: liveBrowserPaths.length > 0 + ? "Run one focused live browser oracle for the changed interaction; automatic CI remains browser-free." + : "No browser oracle selected for text, Markdown, link, image, or non-interactive changes.", + }, + reasons, + changes: normalized.changes, + pathMatches, + }; +} + +function safeRevision(value) { + return typeof value === "string" && /^[A-Za-z0-9._/@{}^~:+-]{1,200}$/u.test(value) ? value : null; +} + +export function parseGitNameStatus(output) { + if (typeof output !== "string") return [{ status: "ambiguous", path: "" }]; + const fields = output.split("\0"); + if (fields.at(-1) === "") fields.pop(); + const changes = []; + for (let index = 0; index < fields.length;) { + const rawStatus = fields[index++]; + const statusCode = rawStatus?.[0]; + if (statusCode === "R" || statusCode === "C") { + changes.push({ status: statusCode, oldPath: fields[index++], newPath: fields[index++] }); + } else { + changes.push({ status: rawStatus, path: fields[index++] }); + } + } + return changes; +} + +function normalizeChanges(input) { + if (!Array.isArray(input) || input.length === 0) { + return { + changes: [], + errors: [{ code: "ambiguous_input", detail: "The changed-file set is missing or empty." }], + }; + } + + const changes = []; + const errors = []; + for (const [index, value] of input.entries()) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + errors.push({ code: "ambiguous_change", index, detail: "A change entry is not an object." }); + continue; + } + const rawStatus = typeof value.status === "string" ? value.status : ""; + const status = normalizeStatus(rawStatus); + if (!status) { + errors.push({ code: "ambiguous_status", index, detail: `Unsupported change status: ${rawStatus || ""}.` }); + continue; + } + + const rawPaths = status === "renamed" || status === "copied" + ? [value.oldPath, value.newPath] + : [value.path]; + const paths = rawPaths.map(normalizePath); + if (paths.some((path) => path === null)) { + errors.push({ code: "ambiguous_path", index, detail: "A changed path is missing, absolute, or unsafe." }); + continue; + } + changes.push({ status, paths }); + } + + changes.sort((left, right) => `${left.paths.join("\0")}\0${left.status}`.localeCompare(`${right.paths.join("\0")}\0${right.status}`)); + if (changes.length === 0 && errors.length === 0) { + errors.push({ code: "ambiguous_input", detail: "No classifiable changes were supplied." }); + } + return { changes, errors }; +} + +function normalizeStatus(rawStatus) { + if (STATUS_ALIASES[rawStatus]) return STATUS_ALIASES[rawStatus]; + if (/^R\d{1,3}$/u.test(rawStatus)) return "renamed"; + if (/^C\d{1,3}$/u.test(rawStatus)) return "copied"; + return null; +} + +function normalizePath(value) { + if (typeof value !== "string" || value.length === 0 || value.includes("\\") || value.startsWith("/")) return null; + const parts = value.split("/"); + if (parts.some((part) => part === "" || part === "." || part === ".." || /[\0\r\n]/u.test(part))) return null; + return parts.join("/"); +} + +function classifyPath(path) { + for (const { rule, matchers } of COMPILED_RULES) { + if (matchers.length === 0 || matchers.some((matcher) => matcher.test(path))) { + return pathResult(rule); + } + } + throw new Error("Verification policy must end with a fallback rule."); +} + +function pathResult(rule) { + return { + pathClass: rule.id, + profile: rule.profile, + sensitive: rule.sensitive, + liveBrowser: rule.liveBrowser === true, + detail: rule.detail, + }; +} + +function selectProfile(matches) { + if (matches.some((match) => match.profile === "release-critical")) return "release-critical"; + if (matches.some((match) => match.profile === "core")) return "core"; + if (matches.some((match) => match.profile === "docs")) return "docs"; + return "focused-docs"; +} + +function gatesForMatches(matches) { + const selected = new Set(); + for (const match of matches) { + for (const gate of PROFILES[match.profile] ?? []) selected.add(gate); + } + return FULL_GATES.filter((gate) => selected.has(gate)); +} + +function deduplicateReasons(reasons) { + const seen = new Set(); + return reasons.filter((reason) => { + const key = canonicalJson(reason); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function digest(value) { + return `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`; +} + +function canonicalJson(value) { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function deepFreeze(value) { + if (!value || typeof value !== "object" || Object.isFrozen(value)) return value; + for (const child of Object.values(value)) deepFreeze(child); + return Object.freeze(value); +} diff --git a/scripts/verification-runner.mjs b/scripts/verification-runner.mjs new file mode 100644 index 0000000..1c3cc1b --- /dev/null +++ b/scripts/verification-runner.mjs @@ -0,0 +1,664 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + existsSync, + lstatSync, + mkdirSync, + readFileSync, + readlinkSync, + readdirSync, + realpathSync, + writeFileSync, +} from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { classifyChanges, parseGitNameStatus, POLICY_DIGEST, POLICY_VERSION } from "./verification-policy.mjs"; + +export const RECEIPT_SCHEMA = "planr.verification-receipt.v3"; +export const LINUX_TARGET_RECEIPT_SCHEMA = "planr.linux-target-receipt.v1"; +export const RUNNER_VERSION = "1.2.0"; + +const LINUX_TARGETS = deepFreeze({ + "linux-x86_64": { cargoTarget: "x86_64-unknown-linux-musl", hostArchitecture: "x64" }, + "linux-arm64": { cargoTarget: "aarch64-unknown-linux-musl", hostArchitecture: "arm64" }, +}); + +const GATE_COMMANDS = deepFreeze({ + "docs-content": [["pnpm", "--filter", "@planr/docs", "content"]], + "docs-typecheck": [["pnpm", "--filter", "@planr/docs", "typecheck"]], + "docs-lint": [["pnpm", "--filter", "@planr/docs", "lint"]], + "docs-build": [["pnpm", "--filter", "@planr/docs", "build"]], + "docs-artifact": [["pnpm", "--filter", "@planr/docs", "verify:artifact"]], + "rust-fmt": [["cargo", "fmt", "--all", "--", "--check"]], + "rust-clippy": [["cargo", "clippy", "--all-targets", "--all-features", "--", "-D", "warnings"]], + "rust-test": [["cargo", "test", "--all-features"]], + "generated-reference": [["pnpm", "--filter", "@planr/docs", "reference:check"]], + "github-actions": [["npm", "run", "verify:github-actions"]], + "release-contract": [ + ["npm", "run", "verify:release-script"], + ["npm", "run", "pack:check"], + ], + "linux-portability": [[ + "sh", + "-c", + "test \"$(find dist -maxdepth 1 -name 'planr-linux-*.tar.gz' -type f | wc -l)\" -eq 2 && cd dist && sha256sum planr-linux-arm64.tar.gz planr-linux-x86_64.tar.gz > SHA256SUMS && sha256sum -c SHA256SUMS", + ]], + "release-evaluation": [["npm", "run", "verify:release-eval-gate"]], +}); + +const LINUX_ARTIFACTS = Object.freeze([ + "dist/planr-linux-x86_64.tar.gz", + "dist/planr-linux-arm64.tar.gz", + "dist/SHA256SUMS", +]); +const DEFAULT_ARTIFACTS = deepFreeze({ + "docs-artifact": ["apps/docs/out"], + "linux-portability": LINUX_ARTIFACTS, +}); +const RECEIPT_OUTPUT_DIRECTORY = ".planr/receipts"; +const TOP_LEVEL_KEYS = [ + "artifacts", "changedFiles", "commands", "durationMs", "finishedAt", "linuxTargets", "policy", "runner", + "schemaVersion", "selection", "source", "startedAt", "verdict", +]; +const FORBIDDEN_KEY = /(^|_)(?:authorization|completion|credential|environment|env|password|prompt|secret|session|token)(_|$)/iu; +const SECRET_VALUE = /(?:-----BEGIN [A-Z ]*PRIVATE KEY-----|(?:api|access|auth|secret)[_-]?key\s*[=:]|(?:gh[oprsu]|sk|xai)-[A-Za-z0-9_-]{16,})/iu; +const PRIVATE_PATH = /(?:^|\/)(?:\.codex|\.claude|\.cursor|\.planr\/(?:eval|private)|examples\/eval)(?:\/|$)/u; + +export function commandPlanFor(selection) { + assertSelection(selection); + const plan = []; + const seen = new Set(); + for (const gate of selection.selectedGates) { + const commands = GATE_COMMANDS[gate]; + if (!commands) throw new Error(`unknown verification gate: ${gate}`); + for (const [executable, ...args] of commands) { + const identity = JSON.stringify([executable, args]); + if (seen.has(identity)) continue; + seen.add(identity); + plan.push({ gate, executable, args }); + } + } + return plan; +} + +export function linuxTargetCommandPlan(target) { + const config = LINUX_TARGETS[target]; + if (!config) throw new Error(`unknown Linux release target: ${target}`); + const bindings = [`PLANR_TARGET=${target}`, `PLANR_CARGO_TARGET=${config.cargoTarget}`]; + return [ + { gate: "linux-portability", executable: "env", args: [...bindings, "sh", "scripts/build-linux-release.sh"] }, + { gate: "linux-portability", executable: "env", args: [...bindings, "sh", "scripts/verify-linux-release-artifact.sh"] }, + ]; +} + +export function runLinuxTargetVerification({ + selection, + target, + repoRoot = process.cwd(), + execute = executeCommand, + host = { platform: process.platform, architecture: process.arch }, + now = () => new Date(), + monotonicNow = () => performance.now(), +}) { + assertSelection(selection); + const config = assertCompatibleLinuxHost(target, host); + const root = canonicalRoot(repoRoot); + const source = currentSourceIdentity(root, selection, { allowedDirtyPaths: LINUX_ARTIFACTS }); + const startedAt = now().toISOString(); + const startedTick = monotonicNow(); + const commands = executePlan(linuxTargetCommandPlan(target), execute, root, monotonicNow); + const artifact = artifactRecord(root, "linux-portability", `dist/planr-${target}.tar.gz`); + const sourceAfter = currentSourceIdentity(root, selection, { allowedDirtyPaths: LINUX_ARTIFACTS }); + assertEqual(sourceAfter.revision, source.revision, "source revision changed during Linux target verification"); + assertEqual(sourceAfter.stateDigest, source.stateDigest, "source inputs changed during Linux target verification"); + const receipt = { + schemaVersion: LINUX_TARGET_RECEIPT_SCHEMA, + runner: { version: RUNNER_VERSION, digest: runnerDigest() }, + source, + target, + cargoTarget: config.cargoTarget, + host, + commands, + artifact, + startedAt, + finishedAt: now().toISOString(), + durationMs: elapsedMilliseconds(startedTick, monotonicNow()), + verdict: commands.every(({ status }) => status === "passed") && artifact.present ? "pass" : "fail", + }; + validateLinuxTargetReceipt(receipt, { root, selection }); + return receipt; +} + +export function verifyLinuxTargetReceipt(receipt, { selection, repoRoot = process.cwd() } = {}) { + assertSelection(selection); + const root = canonicalRoot(repoRoot); + validateLinuxTargetReceipt(receipt, { root, selection }); + return { verdict: "pass", target: receipt.target, sourceRevision: receipt.source.revision, artifactDigest: receipt.artifact.digest }; +} + +export function receiptDigest(receipt) { + validateReceiptShape(receipt); + return digest(receipt); +} + +export function runVerification({ + selection, + repoRoot = process.cwd(), + receiptPath, + artifactPaths = DEFAULT_ARTIFACTS, + linuxTargetReceipts = [], + execute = executeCommand, + now = () => new Date(), + monotonicNow = () => performance.now(), +}) { + assertSelection(selection); + const root = canonicalRoot(repoRoot); + const startedAt = now().toISOString(); + const startedTick = monotonicNow(); + const outputPaths = selection.selectedGates.flatMap((gate) => artifactPaths[gate] ?? []); + const source = currentSourceIdentity(root, selection, { allowedDirtyPaths: outputPaths }); + const commandResults = []; + let priorFailure = false; + + const linuxTargets = selection.selectedGates.includes("linux-portability") + ? validateLinuxTargetReceipts(linuxTargetReceipts, { root, selection, source }) + : []; + commandResults.push(...executePlan(commandPlanFor(selection), execute, root, monotonicNow)); + priorFailure ||= commandResults.some(({ status }) => status !== "passed"); + + const artifacts = []; + for (const gate of selection.selectedGates) { + for (const artifactPath of artifactPaths[gate] ?? []) { + const record = artifactRecord(root, gate, artifactPath); + artifacts.push(record); + if (!record.present) priorFailure = true; + } + } + + const sourceAfterExecution = currentSourceIdentity(root, selection, { allowedDirtyPaths: outputPaths }); + assertEqual(sourceAfterExecution.revision, source.revision, "source revision changed during verification"); + assertEqual(sourceAfterExecution.stateDigest, source.stateDigest, "source inputs changed during verification"); + + const finishedAt = now().toISOString(); + const durationMs = elapsedMilliseconds(startedTick, monotonicNow()); + const receipt = { + schemaVersion: RECEIPT_SCHEMA, + runner: { version: RUNNER_VERSION, digest: runnerDigest() }, + policy: { version: selection.policyVersion, digest: selection.policyDigest }, + source, + changedFiles: { digest: selection.changedFilesDigest, changes: selection.changes }, + selection: { + profile: selection.profile, + escalatedToFull: selection.escalatedToFull, + matchedPathClasses: selection.matchedPathClasses, + selectedGates: selection.selectedGates, + liveVerification: selection.liveVerification, + }, + commands: commandResults, + artifacts, + linuxTargets, + startedAt, + finishedAt, + durationMs, + verdict: priorFailure ? "fail" : "pass", + }; + validateReceiptShape(receipt); + if (receiptPath) writeReceipt(root, receiptPath, receipt); + return receipt; +} + +export function verifyReceipt(receipt, { selection, repoRoot = process.cwd(), artifactPaths = DEFAULT_ARTIFACTS, receiptPath } = {}) { + validateReceiptShape(receipt); + assertSelection(selection); + const root = canonicalRoot(repoRoot); + if (receiptPath) assertReceiptPathBinding(root, receiptPath, receipt); + const allowedDirtyPaths = [ + ...selection.selectedGates.flatMap((gate) => artifactPaths[gate] ?? []), + ...(receiptPath ? [receiptPath] : []), + ]; + const expectedSource = currentSourceIdentity(root, selection, { allowedDirtyPaths }); + assertEqual(receipt.policy.version, POLICY_VERSION, "receipt policy version is stale"); + assertEqual(receipt.policy.digest, POLICY_DIGEST, "receipt policy digest is stale"); + assertEqual(receipt.policy.version, selection.policyVersion, "selection policy version mismatch"); + assertEqual(receipt.policy.digest, selection.policyDigest, "selection policy digest mismatch"); + assertEqual(receipt.source.revision, expectedSource.revision, "receipt source revision is stale"); + assertEqual(receipt.source.stateDigest, expectedSource.stateDigest, "receipt source inputs were altered"); + assertEqual(receipt.changedFiles.digest, selection.changedFilesDigest, "changed-file set digest mismatch"); + assertEqual(canonicalJson(receipt.changedFiles.changes), canonicalJson(selection.changes), "changed-file set mismatch"); + assertEqual(canonicalJson(receipt.selection.selectedGates), canonicalJson(selection.selectedGates), "required gate set mismatch"); + assertEqual(receipt.selection.profile, selection.profile, "verification profile mismatch"); + assertEqual(receipt.selection.escalatedToFull, selection.escalatedToFull, "selection escalation mismatch"); + assertEqual(canonicalJson(receipt.selection.matchedPathClasses), canonicalJson(selection.matchedPathClasses), "matched path classes mismatch"); + assertEqual(canonicalJson(receipt.selection.liveVerification), canonicalJson(selection.liveVerification), "live verification selection mismatch"); + assertEqual(receipt.runner.version, RUNNER_VERSION, "runner version mismatch"); + assertEqual(receipt.runner.digest, runnerDigest(), "runner implementation changed"); + assertEqual(receipt.verdict, "pass", "verification receipt is not green"); + const expectedLinuxTargets = selection.selectedGates.includes("linux-portability") + ? validateLinuxTargetReceipts(receipt.linuxTargets, { root, selection, source: receipt.source }) + : []; + assertEqual(canonicalJson(receipt.linuxTargets), canonicalJson(expectedLinuxTargets), "Linux target receipt set mismatch"); + + const expectedCommands = commandPlanFor(selection); + assertEqual(receipt.commands.length, expectedCommands.length, "receipt command count mismatch"); + for (const [index, expected] of expectedCommands.entries()) { + const actual = receipt.commands[index]; + assertEqual(canonicalJson(pick(actual, ["gate", "executable", "args"])), canonicalJson(expected), `command ${index} mismatch`); + assertEqual(actual.status, "passed", `command ${index} did not pass`); + assertEqual(actual.exitCode, 0, `command ${index} exit code is not zero`); + } + const expectedArtifacts = selection.selectedGates.flatMap((gate) => (artifactPaths[gate] ?? []).map((artifactPath) => ({ gate, path: artifactPath }))); + assertEqual( + canonicalJson(receipt.artifacts.map(({ gate, path: artifactPath }) => ({ gate, path: artifactPath }))), + canonicalJson(expectedArtifacts), + "receipt artifact set mismatch", + ); + for (const artifact of receipt.artifacts) { + const current = artifactRecord(root, artifact.gate, artifact.path); + assertEqual(current.present, true, `artifact is missing: ${artifact.path}`); + assertEqual(current.digest, artifact.digest, `artifact changed: ${artifact.path}`); + assertEqual(current.files, artifact.files, `artifact file count changed: ${artifact.path}`); + assertEqual(current.bytes, artifact.bytes, `artifact byte count changed: ${artifact.path}`); + } + return { verdict: "pass", sourceRevision: receipt.source.revision, gates: receipt.selection.selectedGates.length }; +} + +function executePlan(plan, execute, root, monotonicNow) { + const results = []; + let priorFailure = false; + for (const command of plan) { + if (priorFailure) { + results.push({ ...command, durationMs: null, exitCode: null, status: "not_run_after_failure" }); + continue; + } + const commandStarted = monotonicNow(); + const result = execute(command.executable, command.args, { cwd: root }); + const durationMs = elapsedMilliseconds(commandStarted, monotonicNow()); + const exitCode = Number.isInteger(result?.status) ? result.status : 1; + const status = exitCode === 0 ? "passed" : "failed"; + results.push({ ...command, durationMs, exitCode, status }); + priorFailure ||= exitCode !== 0; + } + return results; +} + +function assertCompatibleLinuxHost(target, host) { + const config = LINUX_TARGETS[target]; + if (!config) throw new Error(`unknown Linux release target: ${target}`); + if (host?.platform !== "linux" || host?.architecture !== config.hostArchitecture) { + throw new Error(`${target} requires native linux/${config.hostArchitecture}, received ${host?.platform ?? "unknown"}/${host?.architecture ?? "unknown"}`); + } + return config; +} + +function validateLinuxTargetReceipts(receipts, { root, selection, source }) { + if (!Array.isArray(receipts) || receipts.length !== Object.keys(LINUX_TARGETS).length) { + throw new Error("both independent Linux target receipts are required"); + } + const byTarget = new Map(receipts.map((receipt) => [receipt?.target, receipt])); + if (byTarget.size !== receipts.length) throw new Error("Linux target receipts must be unique"); + return Object.keys(LINUX_TARGETS).map((target) => { + const receipt = byTarget.get(target); + if (!receipt) throw new Error(`missing Linux target receipt: ${target}`); + validateLinuxTargetReceipt(receipt, { root, selection }); + assertEqual(receipt.source.revision, source.revision, `${target} source revision mismatch`); + assertEqual(receipt.source.stateDigest, source.stateDigest, `${target} source inputs mismatch`); + return receipt; + }); +} + +function validateLinuxTargetReceipt(receipt, { root, selection }) { + assertKeys(receipt, [ + "artifact", "cargoTarget", "commands", "durationMs", "finishedAt", "host", "runner", "schemaVersion", + "source", "startedAt", "target", "verdict", + ], "Linux target receipt"); + if (receipt.schemaVersion !== LINUX_TARGET_RECEIPT_SCHEMA) throw new Error("unsupported Linux target receipt schema"); + const config = assertCompatibleLinuxHost(receipt.target, receipt.host); + assertEqual(receipt.cargoTarget, config.cargoTarget, `${receipt.target} cargo target mismatch`); + assertKeys(receipt.host, ["architecture", "platform"], "Linux target host"); + assertKeys(receipt.runner, ["digest", "version"], "Linux target runner"); + assertKeys(receipt.source, ["revision", "stateDigest"], "Linux target source"); + assertKeys(receipt.artifact, ["bytes", "digest", "files", "gate", "path", "present"], "Linux target artifact"); + assertEqual(receipt.runner.version, RUNNER_VERSION, "Linux target runner version mismatch"); + assertEqual(receipt.runner.digest, runnerDigest(), "Linux target runner implementation changed"); + assertEqual(receipt.verdict, "pass", `${receipt.target} receipt is not green`); + const expectedSource = currentSourceIdentity(root, selection, { allowedDirtyPaths: LINUX_ARTIFACTS }); + assertEqual(receipt.source.revision, expectedSource.revision, `${receipt.target} source revision is stale`); + assertEqual(receipt.source.stateDigest, expectedSource.stateDigest, `${receipt.target} source inputs were altered`); + const expectedCommands = linuxTargetCommandPlan(receipt.target); + assertEqual(receipt.commands.length, expectedCommands.length, `${receipt.target} command count mismatch`); + for (const [index, expected] of expectedCommands.entries()) { + const actual = receipt.commands[index]; + assertKeys(actual, ["args", "durationMs", "executable", "exitCode", "gate", "status"], `Linux target commands[${index}]`); + assertEqual(canonicalJson(pick(actual, ["gate", "executable", "args"])), canonicalJson(expected), `${receipt.target} command ${index} mismatch`); + assertEqual(actual.status, "passed", `${receipt.target} command ${index} did not pass`); + assertEqual(actual.exitCode, 0, `${receipt.target} command ${index} exit code is not zero`); + } + const expectedPath = `dist/planr-${receipt.target}.tar.gz`; + assertEqual(receipt.artifact.path, expectedPath, `${receipt.target} artifact path mismatch`); + const currentArtifact = artifactRecord(root, "linux-portability", expectedPath); + assertEqual(currentArtifact.present, true, `artifact is missing: ${expectedPath}`); + assertEqual(canonicalJson(receipt.artifact), canonicalJson(currentArtifact), `${receipt.target} artifact changed`); + scanUnsafe(receipt, ["linuxTarget"]); +} + +function currentSourceIdentity(root, selection, { allowedDirtyPaths = [] } = {}) { + const requestedRevision = git(root, ["rev-parse", "--verify", `${selection.headRevision ?? "HEAD"}^{commit}`]).trim(); + const revision = git(root, ["rev-parse", "--verify", "HEAD^{commit}"]).trim(); + if (!/^[0-9a-f]{40,64}$/u.test(revision)) throw new Error("current source revision is not a commit SHA"); + if (requestedRevision !== revision) { + throw new Error(`selected source revision ${requestedRevision} is not checked out at HEAD ${revision}`); + } + const exclusions = allowedDirtyPaths.map((relativePath) => { + safeRepositoryPath(root, relativePath); + return `:(literal,exclude)${relativePath}`; + }); + const worktreeState = git(root, ["status", "--porcelain=v1", "-z", "--untracked-files=all", "--", ".", ...exclusions]); + if (worktreeState.length > 0) { + throw new Error("source worktree must be clean, including untracked files"); + } + const tree = git(root, ["rev-parse", "--verify", "HEAD^{tree}"]).trim(); + const inputs = []; + for (const change of selection.changes) { + for (const relativePath of change.paths) { + inputs.push(sourcePathRecord(root, relativePath)); + } + } + return { revision, stateDigest: digest({ tree, inputs }) }; +} + +function sourcePathRecord(root, relativePath) { + const resolved = safeRepositoryPath(root, relativePath); + if (!existsSync(resolved)) return { path: relativePath, kind: "missing", digest: null }; + const stat = lstatSync(resolved); + if (stat.isSymbolicLink()) return { path: relativePath, kind: "symlink", digest: digest(readlinkSync(resolved)) }; + if (!stat.isFile()) throw new Error(`changed input is not a regular file: ${relativePath}`); + return { path: relativePath, kind: "file", digest: digest(readFileSync(resolved)) }; +} + +function artifactRecord(root, gate, relativePath) { + const resolved = safeRepositoryPath(root, relativePath); + if (!existsSync(resolved)) return { gate, path: relativePath, present: false, digest: null, files: 0, bytes: 0 }; + const records = []; + walkArtifact(resolved, relativePath, records); + if (records.length === 0) return { gate, path: relativePath, present: false, digest: null, files: 0, bytes: 0 }; + return { + gate, + path: relativePath, + present: true, + digest: digest(records), + files: records.length, + bytes: records.reduce((sum, record) => sum + record.bytes, 0), + }; +} + +function walkArtifact(absolutePath, relativePath, records) { + const stat = lstatSync(absolutePath); + if (stat.isSymbolicLink()) throw new Error(`artifact contains a symlink: ${relativePath}`); + if (stat.isFile()) { + records.push({ path: relativePath, bytes: stat.size, digest: digest(readFileSync(absolutePath)) }); + return; + } + if (!stat.isDirectory()) throw new Error(`artifact contains an unsupported entry: ${relativePath}`); + for (const name of readdirSync(absolutePath).sort()) { + walkArtifact(path.join(absolutePath, name), `${relativePath}/${name}`, records); + } +} + +function validateReceiptShape(receipt) { + if (!receipt || typeof receipt !== "object" || Array.isArray(receipt)) throw new Error("receipt must be an object"); + assertKeys(receipt, TOP_LEVEL_KEYS, "receipt"); + assertKeys(receipt.runner, ["digest", "version"], "runner"); + assertKeys(receipt.policy, ["digest", "version"], "policy"); + assertKeys(receipt.source, ["revision", "stateDigest"], "source"); + assertKeys(receipt.changedFiles, ["changes", "digest"], "changedFiles"); + assertKeys(receipt.selection, ["escalatedToFull", "liveVerification", "matchedPathClasses", "profile", "selectedGates"], "selection"); + requiredArray(receipt.linuxTargets, "linuxTargets"); + for (const [index, command] of requiredArray(receipt.commands, "commands").entries()) { + assertKeys(command, ["args", "durationMs", "executable", "exitCode", "gate", "status"], `commands[${index}]`); + } + for (const [index, artifact] of requiredArray(receipt.artifacts, "artifacts").entries()) { + assertKeys(artifact, ["bytes", "digest", "files", "gate", "path", "present"], `artifacts[${index}]`); + } + if (receipt.schemaVersion !== RECEIPT_SCHEMA) throw new Error("unsupported receipt schema"); + scanUnsafe(receipt, []); +} + +function scanUnsafe(value, pathParts) { + if (Array.isArray(value)) { + value.forEach((child, index) => scanUnsafe(child, [...pathParts, String(index)])); + return; + } + if (value && typeof value === "object") { + for (const [key, child] of Object.entries(value)) { + if (FORBIDDEN_KEY.test(key)) throw new Error(`forbidden receipt field: ${[...pathParts, key].join(".")}`); + scanUnsafe(child, [...pathParts, key]); + } + return; + } + if (typeof value !== "string") return; + if (value.includes("\0") || /[\r\n]/u.test(value)) throw new Error(`unsafe receipt string: ${pathParts.join(".")}`); + if (SECRET_VALUE.test(value)) throw new Error(`secret-like receipt value: ${pathParts.join(".")}`); + if (path.isAbsolute(value) || PRIVATE_PATH.test(value)) throw new Error(`private path in receipt: ${pathParts.join(".")}`); +} + +function assertSelection(selection) { + if (!selection || typeof selection !== "object") throw new Error("verification selection is required"); + if (selection.policyVersion !== POLICY_VERSION || selection.policyDigest !== POLICY_DIGEST) throw new Error("selection policy is stale"); + if ( + !Array.isArray(selection.changes) + || !Array.isArray(selection.selectedGates) + || typeof selection.liveVerification?.browser !== "boolean" + || !Array.isArray(selection.liveVerification?.paths) + ) throw new Error("selection is incomplete"); + for (const gate of selection.selectedGates) if (!GATE_COMMANDS[gate]) throw new Error(`unknown verification gate: ${gate}`); +} + +function safeRepositoryPath(root, relativePath) { + if (typeof relativePath !== "string" || relativePath.length === 0 || path.isAbsolute(relativePath) || relativePath.includes("\\")) { + throw new Error("receipt paths must be repository-relative"); + } + const resolved = path.resolve(root, relativePath); + if (resolved !== root && !resolved.startsWith(`${root}${path.sep}`)) throw new Error("receipt path escapes repository"); + return resolved; +} + +function safeReceiptPath(root, relativePath, { mustExist = false } = {}) { + const resolved = safeRepositoryPath(root, relativePath); + const segments = relativePath.split("/"); + if ( + segments.length !== 3 + || segments[0] !== ".planr" + || segments[1] !== "receipts" + || !/^[A-Za-z0-9][A-Za-z0-9._-]*\.json$/u.test(segments[2]) + ) { + throw new Error(`receipt path must be a JSON file directly under ${RECEIPT_OUTPUT_DIRECTORY}`); + } + for (const candidate of [path.join(root, ".planr"), path.join(root, RECEIPT_OUTPUT_DIRECTORY), resolved]) { + if (existsSync(candidate) && lstatSync(candidate).isSymbolicLink()) { + throw new Error("receipt path must not contain symbolic-link aliases"); + } + } + const tracked = spawnSync("git", ["ls-files", "--error-unmatch", "--", relativePath], { + cwd: root, + encoding: "utf8", + }); + if (tracked.status === 0) throw new Error("receipt path must not be a tracked source path"); + if (mustExist && (!existsSync(resolved) || !lstatSync(resolved).isFile())) { + throw new Error("receipt path must identify an existing regular file"); + } + return resolved; +} + +function assertReceiptPathBinding(root, relativePath, receipt) { + const resolved = safeReceiptPath(root, relativePath, { mustExist: true }); + let storedReceipt; + try { + storedReceipt = JSON.parse(readFileSync(resolved, "utf8")); + } catch { + throw new Error("receipt path must contain valid receipt JSON"); + } + assertEqual(canonicalJson(storedReceipt), canonicalJson(receipt), "receipt path content does not match the verified receipt"); +} + +function canonicalRoot(repoRoot) { + const root = realpathSync(repoRoot); + git(root, ["rev-parse", "--is-inside-work-tree"]); + return root; +} + +function executeCommand(executable, args, options) { + return spawnSync(executable, args, { ...options, stdio: "inherit", env: process.env }); +} + +function writeReceipt(root, relativePath, receipt) { + const output = safeReceiptPath(root, relativePath); + mkdirSync(path.dirname(output), { recursive: true }); + writeFileSync(output, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); +} + +function runnerDigest() { + return digest(readFileSync(fileURLToPath(import.meta.url))); +} + +function git(root, args) { + const result = spawnSync("git", args, { cwd: root, encoding: "utf8" }); + if (result.status !== 0) throw new Error(`git ${args[0]} failed`); + return result.stdout; +} + +function digest(value) { + const input = Buffer.isBuffer(value) ? value : Buffer.from(canonicalJson(value)); + return `sha256:${createHash("sha256").update(input).digest("hex")}`; +} + +function canonicalJson(value) { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function assertKeys(value, keys, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`); + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (canonicalJson(actual) !== canonicalJson(expected)) throw new Error(`${label} fields are not allowlisted`); +} + +function requiredArray(value, label) { + if (!Array.isArray(value)) throw new Error(`${label} must be an array`); + return value; +} + +function assertEqual(actual, expected, message) { + if (actual !== expected) throw new Error(message); +} + +function elapsedMilliseconds(start, end) { + const elapsed = Math.max(0, end - start); + return Math.round(elapsed * 1000) / 1000; +} + +function pick(value, keys) { + return Object.fromEntries(keys.map((key) => [key, value[key]])); +} + +function deepFreeze(value) { + if (!value || typeof value !== "object" || Object.isFrozen(value)) return value; + for (const child of Object.values(value)) deepFreeze(child); + return Object.freeze(value); +} + +function parseArgs(argv) { + const args = [...argv]; + const command = args.shift(); + const values = new Map(); + while (args.length > 0) { + const name = args.shift(); + if (!name?.startsWith("--") || args.length === 0) throw new Error(`invalid argument: ${name ?? ""}`); + values.set(name, args.shift()); + } + return { command, values }; +} + +function selectionFromCli(values, root) { + const inputPath = values.get("--input"); + let base = values.get("--base"); + const head = values.get("--head") ?? "HEAD"; + const explicitProfile = values.get("--profile"); + let changes; + if (inputPath) { + const input = JSON.parse(readFileSync(safeRepositoryPath(root, inputPath), "utf8")); + changes = Array.isArray(input) ? input : input.changes; + } else if (base || explicitProfile) { + base ??= `${head}^`; + changes = parseGitNameStatus(git(root, ["diff", "--name-status", "-z", "--find-renames", base, head])); + } else { + throw new Error("--input, --base, or --profile is required"); + } + const selection = classifyChanges(changes, { baseRevision: base ?? null, headRevision: head }); + if (explicitProfile && explicitProfile !== selection.profile) { + throw new Error(`explicit profile ${explicitProfile} does not match classified profile ${selection.profile}`); + } + return selection; +} + +function main() { + const { command, values } = parseArgs(process.argv.slice(2)); + const root = canonicalRoot(process.cwd()); + if (command === "run-linux-target") { + const receiptPath = values.get("--receipt"); + const target = values.get("--target"); + if (!receiptPath || !target) throw new Error("run-linux-target requires --target and --receipt"); + const selection = selectionFromCli(values, root); + if (!selection.selectedGates.includes("linux-portability")) throw new Error("selection does not require Linux portability"); + const receipt = runLinuxTargetVerification({ selection, target, repoRoot: root }); + writeReceipt(root, receiptPath, receipt); + process.stdout.write(`${JSON.stringify({ verdict: receipt.verdict, target, receipt: receiptPath, receiptDigest: digest(receipt), sourceRevision: receipt.source.revision })}\n`); + return; + } + if (command === "verify-linux-target") { + const receiptPath = values.get("--receipt"); + if (!receiptPath) throw new Error("verify-linux-target requires --receipt"); + const selection = selectionFromCli(values, root); + const receipt = JSON.parse(readFileSync(safeReceiptPath(root, receiptPath, { mustExist: true }), "utf8")); + process.stdout.write(`${JSON.stringify(verifyLinuxTargetReceipt(receipt, { selection, repoRoot: root }))}\n`); + return; + } + if (command === "run") { + const receiptPath = values.get("--receipt"); + if (!receiptPath) throw new Error("run requires --receipt"); + const selection = selectionFromCli(values, root); + const linuxTargetReceipts = selection.selectedGates.includes("linux-portability") + ? Object.keys(LINUX_TARGETS).map((target) => { + const targetReceiptPath = values.get(`--${target}-receipt`); + if (!targetReceiptPath) throw new Error(`run requires --${target}-receipt for Linux portability`); + return JSON.parse(readFileSync(safeReceiptPath(root, targetReceiptPath, { mustExist: true }), "utf8")); + }) + : []; + const receipt = runVerification({ selection, repoRoot: root, receiptPath, linuxTargetReceipts }); + process.stdout.write(`${JSON.stringify({ verdict: receipt.verdict, receipt: receiptPath, receiptDigest: receiptDigest(receipt), sourceRevision: receipt.source.revision })}\n`); + if (receipt.verdict !== "pass") process.exitCode = 1; + return; + } + if (command === "verify") { + const receiptPath = values.get("--receipt"); + if (!receiptPath) throw new Error("verify requires --receipt"); + const selection = selectionFromCli(values, root); + const receipt = JSON.parse(readFileSync(safeReceiptPath(root, receiptPath, { mustExist: true }), "utf8")); + process.stdout.write(`${JSON.stringify({ ...verifyReceipt(receipt, { selection, repoRoot: root, receiptPath }), receiptDigest: receiptDigest(receipt) })}\n`); + return; + } + throw new Error(`usage: verification-runner.mjs --receipt ${RECEIPT_OUTPUT_DIRECTORY}/NAME.json (--input PATH | --base REV | --profile PROFILE) [--head REV]`); +} + +if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/verify-github-actions.mjs b/scripts/verify-github-actions.mjs index 4f1837e..7543223 100644 --- a/scripts/verify-github-actions.mjs +++ b/scripts/verify-github-actions.mjs @@ -5,26 +5,12 @@ import { fileURLToPath } from "node:url"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const workflowsRoot = path.join(repoRoot, ".github", "workflows"); -const trivyIgnoreFile = await readFile(path.join(repoRoot, ".trivyignore.yaml"), "utf8"); const localSecurityScript = await readFile(path.join(repoRoot, "scripts", "security-local.sh"), "utf8"); - -assert.equal( - trivyIgnoreFile, - `misconfigurations: - - id: AVD-DS-0002 - paths: - - scripts/linux-release-builder.Dockerfile - statement: This image is an ephemeral build environment that requires root to install verified local APKs and is never shipped or run as a service. - expired_at: 2027-07-26 - - id: AVD-DS-0026 - paths: - - scripts/linux-release-builder.Dockerfile - statement: This image performs one finite build command and exits, so it has no long-running service to health-check. - expired_at: 2027-07-26 -`, - "Trivy exceptions must remain limited to the two expiring build-only Dockerfile findings", -); -assert.match(localSecurityScript, /--skip-check-update/u, "Local Trivy must use the checks bundled with the reviewed binary"); +const packageJson = JSON.parse(await readFile(path.join(repoRoot, "package.json"), "utf8")); +assert.equal(packageJson.scripts["security:check"], "sh scripts/security-local.sh", "local security command must remain available"); +assert.equal(packageJson.scripts["security:privacy"], "sh scripts/check-repository-privacy.sh", "local privacy command must remain available"); +assert.match(localSecurityScript, /betterleaks/u, "BetterLeaks must remain available for deliberate local use"); +assert.match(localSecurityScript, /trivy fs/u, "Trivy must remain available for deliberate local use"); const expectedActions = new Map([ ["actions/checkout", { sha: "3d3c42e5aac5ba805825da76410c181273ba90b1", version: "v7.0.1", runtime: "node24" }], @@ -90,41 +76,15 @@ for (const [action, expected] of expectedActions) { } } -const securityWorkflow = await readFile(path.join(workflowsRoot, "security.yml"), "utf8"); -assert.doesNotMatch( - securityWorkflow, - /^\s*(?:-\s*)?uses:\s*(?:aquasecurity|trufflesecurity)\//mu, - "Security workflow must respect the GitHub-owned-only repository action policy", -); -for (const [expected, label] of [ - ["https://github.com/trufflesecurity/trufflehog/releases/download/v3.96.0/trufflehog_3.96.0_linux_amd64.tar.gz", "TruffleHog release URL"], - ["7105f1cd6577f058a9e39d0578f1a99c8a1e481e4d3512cd8a09acfe22a0fdc0", "TruffleHog release digest"], - ["https://github.com/aquasecurity/trivy/releases/download/v0.70.0/trivy_0.70.0_Linux-64bit.tar.gz", "Trivy release URL"], - ["8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9", "Trivy release digest"], - ["https://github.com/zizmorcore/zizmor/releases/download/v1.24.1/zizmor-x86_64-unknown-linux-gnu.tar.gz", "zizmor release URL"], - ["a8000f3c683319a523d3b20df0e75457ba591f049cfcbfa98966631b56733c03", "zizmor release digest"], -]) { - assert.ok(securityWorkflow.includes(expected), `Security workflow must pin the reviewed ${label}`); +const automaticScannerPattern = /(?:cargo\s+(?:install\s+cargo-audit|audit\b)|cargo-audit|betterleaks|trivy|trufflehog|zizmor|dependency-review-action|osv-scanner|snyk)/iu; +for (const [file, source] of workflowSources) { + assert.doesNotMatch(source, automaticScannerPattern, `${file} must not run automatic security, secret, or dependency scanners`); } -assert.match(securityWorkflow, /sha256sum --check -/u, "Security scanner downloads must be checksum verified"); -assert.match( - securityWorkflow, - /trufflehog git "file:\/\/\$GITHUB_WORKSPACE" --results=verified --fail --no-update --github-actions/u, - "TruffleHog must fail closed while scanning verified secrets across Git history", -); -assert.match(securityWorkflow, /--scanners secret,misconfig/u, "Trivy must scan secrets and misconfigurations"); -assert.match(securityWorkflow, /--ignorefile \.trivyignore\.yaml/u, "Trivy CI must use the reviewed narrow ignore file"); -assert.match(securityWorkflow, /--skip-check-update/u, "Trivy must use the checks bundled with the reviewed binary"); -assert.match(securityWorkflow, /--exit-code 1/u, "Trivy findings must fail the Security job"); -assert.doesNotMatch( - securityWorkflow, - /(?:python3\s+-m\s+pip\s+install[^\n]*\buv\b|\buvx\b)/u, - "Security workflow must not install mutable uv or zizmor inputs", -); -assert.match(securityWorkflow, /\n\s*zizmor \. \\\n/u, "GitHub Actions Security must run the checksum-verified zizmor binary"); +assert.ok(!workflowFiles.some((file) => /security|secret|dependenc/iu.test(file)), "automatic scanner workflows must remain absent"); const releaseWorkflow = await readFile(path.join(workflowsRoot, "release.yml"), "utf8"); const ciWorkflow = await readFile(path.join(workflowsRoot, "ci.yml"), "utf8"); +const linuxReceiptsWorkflow = await readFile(path.join(workflowsRoot, "linux-receipts.yml"), "utf8"); const linuxBuildScript = await readFile(path.join(repoRoot, "scripts", "build-linux-release.sh"), "utf8"); const linuxBuilderDockerfile = await readFile(path.join(repoRoot, "scripts", "linux-release-builder.Dockerfile"), "utf8"); const linuxVerifyScript = await readFile(path.join(repoRoot, "scripts", "verify-linux-release-artifact.sh"), "utf8"); @@ -184,13 +144,81 @@ for (const command of ["project init", "plan new", "plan split", "map build", "p assert.match(ciWorkflow, /^ linux-portability:\n/mu, "PR CI must contain a Linux portability matrix job"); assert.match(ciWorkflow, /^ linux-portability-checksums:\n/mu, "PR CI must aggregate both Linux tarball checksums"); +assert.match(ciWorkflow, /^ router:\n/mu, "PR CI must contain an always-running verification router"); +assert.match(ciWorkflow, /^ workflow_dispatch:$/mu, "CI must support an explicit native-Linux-only dispatch on a reviewed branch SHA"); +const routerStart = ciWorkflow.indexOf("\n router:\n"); +const routerEnd = ciWorkflow.indexOf("\n docs:\n", routerStart); +const routerJob = ciWorkflow.slice(routerStart, routerEnd); +assert.doesNotMatch(routerJob, /^\s+if:/mu, "verification router must always run"); +for (const output of ["profile", "policy_version", "policy_digest", "changed_files_digest", "live_browser", "docs", "quality", "release", "linux_portability"]) { + assert.match(routerJob, new RegExp(`^ ${output}: \\$\\{\\{ steps\\.route\\.outputs\\.${output} \\}\\}$`, "mu"), `verification router must export ${output}`); +} +assert.match(routerJob, /node scripts\/ci-router\.mjs route/u, "verification router must use the repository-owned deterministic helper"); +assert.match(routerJob, /name: verification-selection/u, "verification routing evidence must cross jobs only as an explicit artifact"); +assert.match(routerJob, /docs=false\\nquality=false\\nrelease=false\\nlinux_portability=true/u, "manual CI dispatch must select only native Linux evidence"); +for (const [jobHeader, condition] of [ + [" docs:\n name: Documentation\n", "if: needs.router.outputs.docs == 'true'"], + [" quality:\n name: Quality Gates\n", "if: needs.router.outputs.quality == 'true'"], + [" release-contracts:\n name: Release Contracts\n", "if: needs.router.outputs.release == 'true'"], + [" linux-portability:\n name: Portable Linux ${{ matrix.target }}\n", "if: needs.router.outputs.linux_portability == 'true'"], +]) { + const start = ciWorkflow.indexOf(jobHeader); + assert.notEqual(start, -1, `PR CI must contain ${jobHeader.trim()}`); + assert.ok(ciWorkflow.slice(start, start + 240).includes(condition), `${jobHeader.trim()} must use its router output`); +} +assert.doesNotMatch(ciWorkflow, /(?:secrets\.|actions\/cache@|\bcache:)\b/u, "PR CI dependency acceleration must not consume secrets or masquerade as evidence"); +assert.doesNotMatch(ciWorkflow, /docs:verify-shell|Verify browser interactions|google-chrome/u, "automatic CI must remain free of the retired blanket browser suite"); +const docsStart = ciWorkflow.indexOf("\n docs:\n"); +const docsEnd = ciWorkflow.indexOf("\n quality:\n", docsStart); +const docsJob = ciWorkflow.slice(docsStart, docsEnd); +assert.equal((docsJob.match(/verification-runner\.mjs run/g) ?? []).length, 1, "docs CI must invoke the exact-source runner once"); +assert.equal((docsJob.match(/docs:build|next build/g) ?? []).length, 0, "docs CI must not add a second production build outside the runner"); +assert.match(docsJob, /name: reviewed-docs-\$\{\{ github\.sha \}\}/u, "docs CI must name its artifact by exact source SHA"); +for (const artifactPath of ["apps/docs/out", ".planr/ci/selection.json", ".planr/receipts/docs.json"]) { + assert.ok(docsJob.includes(artifactPath), `docs CI artifact must include ${artifactPath}`); +} +const summaryStart = ciWorkflow.indexOf("\n summary:\n"); +assert.notEqual(summaryStart, -1, "PR CI must contain one stable summary job"); +const summaryJob = ciWorkflow.slice(summaryStart); +assert.match(summaryJob, /name: CI Summary/u, "PR CI summary check name must remain stable"); +assert.match(summaryJob, /if: always\(\)/u, "PR CI summary must run after failures and skips"); +assert.match(summaryJob, /node scripts\/ci-router\.mjs summary/u, "PR CI summary must use the fail-closed result verifier"); +for (const result of ["needs.docs.result", "needs.quality.result", "needs.release-contracts.result", "needs.linux-portability-checksums.result"]) { + assert.ok(summaryJob.includes(result), `PR CI summary must inspect ${result}`); +} const portabilityStart = ciWorkflow.indexOf("\n linux-portability:\n"); const portabilityEnd = ciWorkflow.indexOf("\n linux-portability-checksums:\n", portabilityStart); const portabilityJob = ciWorkflow.slice(portabilityStart, portabilityEnd); assert.doesNotMatch(portabilityJob, /secrets\./u, "PR Linux portability CI must not consume secrets"); -assert.match(portabilityJob, /scripts\/build-linux-release\.sh/u, "PR Linux portability CI must use the canonical pinned build"); -assert.match(portabilityJob, /scripts\/verify-linux-release-artifact\.sh/u, "PR Linux portability CI must use the canonical compatibility verifier"); +assert.equal((portabilityJob.match(/run-linux-target/g) ?? []).length, 1, "the native matrix must invoke each target runner exactly once"); +assert.match(portabilityJob, /^ \.planr\/receipts\/\$\{\{ matrix\.target \}\}\.json$/mu, "PR Linux portability CI must upload each native target receipt"); +const portabilityAggregate = ciWorkflow.slice(portabilityEnd, summaryStart); +assert.equal((portabilityAggregate.match(/verify-linux-target/g) ?? []).length, 2, "PR CI aggregate must replay exactly two native target receipts"); +assert.match(portabilityAggregate, /name: native-linux-receipts-\$\{\{ github\.sha \}\}/u, "PR CI must retain exact-SHA native receipt evidence"); assert.match(ciWorkflow, /sha256sum planr-linux-arm64\.tar\.gz planr-linux-x86_64\.tar\.gz > SHA256SUMS/u, "PR CI must aggregate the exact two Linux tarballs"); +assert.equal((summaryJob.match(/if: github\.event_name != 'workflow_dispatch'/g) ?? []).length, 2, "manual native-only runs must not emit a promotion receipt"); + +assert.match(linuxReceiptsWorkflow, /^name: Native Linux receipts$/mu, "native Linux evidence must have one stable workflow identity"); +assert.match(linuxReceiptsWorkflow, /^ workflow_dispatch:$/mu, "native Linux evidence must be explicitly dispatched for a reviewed SHA"); +assert.doesNotMatch(linuxReceiptsWorkflow, /(?:secrets\.|pull_request_target|push:)/u, "native Linux evidence must not consume secrets or run implicitly"); +const nativeTargetStart = linuxReceiptsWorkflow.indexOf("\n target:\n"); +const nativeAggregateStart = linuxReceiptsWorkflow.indexOf("\n aggregate:\n", nativeTargetStart); +assert.ok(nativeTargetStart >= 0 && nativeAggregateStart > nativeTargetStart, "native Linux evidence must separate target and aggregate jobs"); +const nativeTargetJob = linuxReceiptsWorkflow.slice(nativeTargetStart, nativeAggregateStart); +assert.match(nativeTargetJob, /^ \.planr\/receipts\/\$\{\{ matrix\.target \}\}\.json$/mu, "each native target upload must retain its runner receipt"); +for (const [target, runner] of [ + ["linux-x86_64", "ubuntu-24.04"], + ["linux-arm64", "ubuntu-24.04-arm"], +]) { + const matrixEntry = `- target: ${target}\n runner: ${runner}`; + assert.ok(linuxReceiptsWorkflow.includes(matrixEntry), `native receipt workflow must bind ${target} to ${runner}`); + assert.ok(linuxReceiptsWorkflow.includes(`.planr/receipts/${target}.json`), `native aggregate must consume the ${target} receipt`); + assert.ok(linuxReceiptsWorkflow.includes(`dist/planr-${target}.tar.gz`), `native receipt workflow must retain the ${target} archive`); +} +assert.equal((linuxReceiptsWorkflow.match(/run-linux-target/g) ?? []).length, 1, "the matrix must invoke each native target exactly once"); +assert.equal((linuxReceiptsWorkflow.match(/verify-linux-target/g) ?? []).length, 2, "the aggregate must replay exactly two target receipts"); +assert.match(linuxReceiptsWorkflow, /name: native-linux-receipts-\$\{\{ github\.sha \}\}/u, "aggregate native evidence must be named by exact SHA"); +assert.match(linuxReceiptsWorkflow, /sha256sum planr-linux-arm64\.tar\.gz planr-linux-x86_64\.tar\.gz > SHA256SUMS/u, "native evidence must aggregate the exact two archives"); const smokeStepMarker = " - name: Smoke-test binary\n"; const smokeStepStart = releaseWorkflow.indexOf(smokeStepMarker); diff --git a/scripts/verify-release-promotion.mjs b/scripts/verify-release-promotion.mjs new file mode 100644 index 0000000..02472e6 --- /dev/null +++ b/scripts/verify-release-promotion.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { routeSelection } from "./ci-router.mjs"; +import { classifyChanges, parseGitNameStatus, POLICY_DIGEST, POLICY_VERSION } from "./verification-policy.mjs"; + +const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const EVALUATED_SUBJECT_PATHS = [ + "plugins/planr/skills/planr-goal/SKILL.md", + "plugins/planr/skills/planr-loop/SKILL.md", + "plugins/planr/skills/planr-loop/references/host-dispatch.md", + "plugins/planr/skills/planr-loop/references/recovery-and-verification.md", + "plugins/planr/skills/planr-task-graph/SKILL.md", +]; +const EVALUATION_POLICY_PATHS = [ + "docs/contracts/EVAL_CONTRACT_V1.md", + "scripts/test-release-eval-gate.mjs", + "scripts/verify-release-eval-receipt.mjs", + "scripts/verify-release-promotion.mjs", +]; + +function option(name, { required = false } = {}) { + const index = process.argv.indexOf(name); + const value = index >= 0 ? process.argv[index + 1] : undefined; + if (required) assert.ok(value, `missing ${name}`); + return value; +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { cwd: repo, encoding: "utf8", ...options }); + assert.equal(result.status, 0, `${command} ${args.join(" ")} failed: ${result.stderr.trim()}`); + return result.stdout.trim(); +} + +function jsonFile(input, label) { + try { + return JSON.parse(fs.readFileSync(path.resolve(repo, input), "utf8")); + } catch { + throw new Error(`${label} is unreadable or invalid JSON`); + } +} + +function exactKeys(value, keys, label) { + assert.ok(value && typeof value === "object" && !Array.isArray(value), `${label} must be an object`); + assert.deepEqual(Object.keys(value).sort(), [...keys].sort(), `${label} fields are not allowlisted`); +} + +const version = option("--version", { required: true }); +const ciReceipt = jsonFile(option("--ci-receipt", { required: true }), "CI promotion receipt"); +const approval = jsonFile(option("--approval", { required: true }), "release approval"); +const head = run("git", ["rev-parse", "--verify", "HEAD^{commit}"]); + +exactKeys(ciReceipt, [ + "schema_version", "repository", "workflow", "run_id", "run_attempt", "event", "source_ref", + "source_base_sha", "source_sha", "conclusion", "policy", "jobs", +], "CI promotion receipt"); +exactKeys(ciReceipt.policy, ["profile", "version", "digest", "changed_files_digest"], "CI receipt policy"); +exactKeys(ciReceipt.jobs, ["docs", "quality", "release", "linux_portability"], "CI receipt jobs"); +assert.equal(ciReceipt.schema_version, "planr.ci-promotion-receipt.v1", "unsupported CI promotion receipt schema"); +assert.match(ciReceipt.repository, /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u, "invalid CI repository identity"); +assert.equal(ciReceipt.workflow, "CI", "promotion requires the canonical CI workflow"); +assert.match(ciReceipt.run_id, /^[1-9][0-9]*$/u, "invalid CI run identity"); +assert.match(ciReceipt.run_attempt, /^[1-9][0-9]*$/u, "invalid CI run attempt"); +assert.equal(ciReceipt.event, "push", "promotion requires CI from a push event"); +assert.equal(ciReceipt.source_ref, "refs/heads/main", "promotion requires CI from main"); +assert.match(ciReceipt.source_base_sha, /^[0-9a-f]{40}$/u, "invalid CI base SHA"); +assert.equal(ciReceipt.source_sha, head, "CI receipt does not bind the current candidate SHA"); +assert.equal(ciReceipt.conclusion, "success", "CI receipt is not green"); +assert.match(ciReceipt.policy.digest, /^sha256:[0-9a-f]{64}$/u, "invalid CI policy digest"); +assert.match(ciReceipt.policy.changed_files_digest, /^sha256:[0-9a-f]{64}$/u, "invalid CI changed-files digest"); +for (const [job, result] of Object.entries(ciReceipt.jobs)) { + assert.match(result, /^(?:success|skipped)$/u, `CI job ${job} is not settled`); +} + +const runRecord = JSON.parse(run("gh", ["api", `repos/${ciReceipt.repository}/actions/runs/${ciReceipt.run_id}`])); +assert.equal(String(runRecord.id), ciReceipt.run_id, "GitHub CI run identity mismatch"); +assert.equal(String(runRecord.run_attempt), ciReceipt.run_attempt, "GitHub CI run attempt mismatch"); +assert.equal(runRecord.name, ciReceipt.workflow, "GitHub workflow identity mismatch"); +assert.equal(runRecord.event, ciReceipt.event, "GitHub CI event mismatch"); +assert.equal(runRecord.head_branch, "main", "GitHub CI run is not for main"); +assert.equal(runRecord.head_sha, head, "GitHub CI run does not bind the current candidate SHA"); +assert.equal(runRecord.conclusion, "success", "GitHub CI run is not green"); +assert.equal(runRecord.repository?.full_name, ciReceipt.repository, "GitHub CI repository identity mismatch"); + +const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), "planr-release-promotion-")); +try { + run("gh", [ + "run", "download", ciReceipt.run_id, + "--repo", ciReceipt.repository, + "--name", `release-promotion-${head}`, + "--dir", artifactRoot, + ]); + const authenticReceipt = jsonFile(path.join(artifactRoot, "promotion-receipt.json"), "authenticated CI promotion artifact"); + assert.deepEqual(ciReceipt, authenticReceipt, "supplied CI receipt does not match the authenticated run artifact"); +} finally { + fs.rmSync(artifactRoot, { recursive: true, force: true }); +} + +run("git", ["rev-parse", "--verify", `${ciReceipt.source_base_sha}^{commit}`]); +run("git", ["merge-base", "--is-ancestor", ciReceipt.source_base_sha, head]); +const nameStatus = run("git", [ + "diff", "--name-status", "-z", "--find-renames", ciReceipt.source_base_sha, head, +]); +const selection = classifyChanges(parseGitNameStatus(nameStatus.endsWith("\0") ? nameStatus : `${nameStatus}\0`), { + baseRevision: ciReceipt.source_base_sha, + headRevision: head, +}); +const routing = routeSelection(selection); +assert.equal(ciReceipt.policy.version, POLICY_VERSION, "CI receipt policy version is stale"); +assert.equal(ciReceipt.policy.digest, POLICY_DIGEST, "CI receipt policy digest is stale"); +assert.equal(ciReceipt.policy.version, selection.policyVersion, "CI receipt policy version mismatch"); +assert.equal(ciReceipt.policy.digest, selection.policyDigest, "CI receipt policy digest mismatch"); +assert.equal(ciReceipt.policy.changed_files_digest, selection.changedFilesDigest, "CI receipt changed-files digest mismatch"); +assert.equal(ciReceipt.policy.profile, selection.profile, "CI receipt profile mismatch"); +for (const [job, result] of Object.entries(ciReceipt.jobs)) { + const expected = routing[job] === "true" ? "success" : "skipped"; + assert.equal(result, expected, `CI job ${job} does not match the current policy selection`); +} + +exactKeys(approval, ["schema_version", "approval_id", "source_sha", "version", "decision", "approved_by", "approved_at"], "release approval"); +assert.equal(approval.schema_version, "planr.release-approval.v1", "unsupported release approval schema"); +assert.match(approval.approval_id, /^[A-Za-z0-9][A-Za-z0-9._-]{2,127}$/u, "invalid approval identity"); +assert.equal(approval.source_sha, head, "approval does not bind the current candidate SHA"); +assert.equal(approval.version, version, "approval does not bind the requested version"); +assert.equal(approval.decision, "approved", "release is not approved"); +assert.match(approval.approved_by, /^[A-Za-z0-9][A-Za-z0-9._@-]{1,127}$/u, "invalid approver identity"); +const approvedAt = Date.parse(approval.approved_at); +assert.ok(Number.isFinite(approvedAt) && approvedAt <= Date.now(), "approval timestamp is invalid or in the future"); + +let baseTag = null; +const base = spawnSync("git", ["describe", "--tags", "--abbrev=0", "HEAD^"], { cwd: repo, encoding: "utf8" }); +if (base.status === 0) baseTag = base.stdout.trim(); +const changed = baseTag + ? run("git", ["diff", "--name-only", `${baseTag}..HEAD`]).split("\n").filter(Boolean) + : [...EVALUATED_SUBJECT_PATHS, ...EVALUATION_POLICY_PATHS]; +const evalTriggerPaths = changed.filter((candidate) => + EVALUATED_SUBJECT_PATHS.includes(candidate) || EVALUATION_POLICY_PATHS.includes(candidate)); +const evaluationRequired = evalTriggerPaths.length > 0; + +if (evaluationRequired) { + const evalReceipt = option("--eval-receipt", { required: true }); + const evalDb = option("--eval-db", { required: true }); + const evalSuite = option("--eval-suite", { required: true }); + const planrBin = option("--planr-bin", { required: true }); + run(process.execPath, [ + "scripts/verify-release-eval-receipt.mjs", + "--receipt", evalReceipt, + "--db", evalDb, + "--suite", evalSuite, + "--planr-bin", planrBin, + ], { stdio: ["ignore", "pipe", "pipe"] }); +} + +console.log(JSON.stringify({ + verdict: "pass", + source_sha: head, + ci_run_id: ciReceipt.run_id, + approval_id: approval.approval_id, + evaluation: evaluationRequired ? "verified" : "not_required", + evaluation_trigger_paths: evalTriggerPaths, +})); diff --git a/scripts/write-ci-promotion-receipt.mjs b/scripts/write-ci-promotion-receipt.mjs new file mode 100644 index 0000000..8e0b78e --- /dev/null +++ b/scripts/write-ci-promotion-receipt.mjs @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; + +const output = process.argv[2]; +if (!output) throw new Error("usage: write-ci-promotion-receipt.mjs "); + +let event; +try { + event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); +} catch { + throw new Error("GITHUB_EVENT_PATH is missing or invalid"); +} + +const required = (name, pattern = /^\S+$/u) => { + const value = process.env[name]; + if (!value || !pattern.test(value)) throw new Error(`${name} is missing or invalid`); + return value; +}; + +const result = (name) => { + const value = required(name, /^(?:success|skipped)$/u); + return value; +}; + +const receipt = { + schema_version: "planr.ci-promotion-receipt.v1", + repository: required("GITHUB_REPOSITORY", /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u), + workflow: required("GITHUB_WORKFLOW"), + run_id: required("GITHUB_RUN_ID", /^[1-9][0-9]*$/u), + run_attempt: required("GITHUB_RUN_ATTEMPT", /^[1-9][0-9]*$/u), + event: required("GITHUB_EVENT_NAME"), + source_ref: required("GITHUB_REF"), + source_base_sha: requiredBaseSha(event), + source_sha: required("GITHUB_SHA", /^[0-9a-f]{40}$/u), + conclusion: "success", + policy: { + profile: required("PLANR_PROFILE"), + version: required("PLANR_POLICY_VERSION"), + digest: required("PLANR_POLICY_DIGEST", /^sha256:[0-9a-f]{64}$/u), + changed_files_digest: required("PLANR_CHANGED_FILES_DIGEST", /^sha256:[0-9a-f]{64}$/u), + }, + jobs: { + docs: result("PLANR_DOCS_RESULT"), + quality: result("PLANR_QUALITY_RESULT"), + release: result("PLANR_RELEASE_RESULT"), + linux_portability: result("PLANR_LINUX_RESULT"), + }, +}; + +function requiredBaseSha(payload) { + const value = process.env.GITHUB_EVENT_NAME === "pull_request" + ? payload?.pull_request?.base?.sha + : payload?.before; + if (typeof value !== "string" || !/^[0-9a-f]{40}$/u.test(value)) { + throw new Error("GitHub event is missing a valid CI base SHA"); + } + return value; +} + +fs.mkdirSync(path.dirname(path.resolve(output)), { recursive: true }); +fs.writeFileSync(output, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); +console.log(JSON.stringify({ verdict: "pass", source_sha: receipt.source_sha, run_id: receipt.run_id })); From e6f2392dd6ea1feb4077e6c438d85bd6bf38028e Mon Sep 17 00:00:00 2001 From: Kevin Kern Date: Tue, 28 Jul 2026 13:41:16 +0200 Subject: [PATCH 2/5] ci: scope verification receipts per job --- .github/workflows/ci.yml | 1 + scripts/deploy-docs.mjs | 8 +++++++- scripts/test-docs-deployment.mjs | 5 +++++ scripts/test-verification-runner.mjs | 18 ++++++++++++++++++ scripts/verification-runner.mjs | 26 ++++++++++++++++++++++++-- 5 files changed, 55 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea10a38..0e07503 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,7 @@ jobs: node scripts/verification-runner.mjs run \ --input .planr/ci/selection.json \ --head "$GITHUB_SHA" \ + --gates "docs-content,docs-typecheck,docs-lint,docs-build,docs-artifact" \ --receipt .planr/receipts/docs.json - name: Replay documented onboarding against this repository diff --git a/scripts/deploy-docs.mjs b/scripts/deploy-docs.mjs index b7c3ce5..b1e07af 100644 --- a/scripts/deploy-docs.mjs +++ b/scripts/deploy-docs.mjs @@ -11,7 +11,13 @@ export function deploymentCommands({ receipt, input, head = 'HEAD', url = 'https { label: 'reviewed receipt', executable: process.execPath, - args: ['scripts/verification-runner.mjs', 'verify', '--receipt', receipt, '--input', input, '--head', head], + args: [ + 'scripts/verification-runner.mjs', 'verify', + '--receipt', receipt, + '--input', input, + '--head', head, + '--gates', 'docs-content,docs-typecheck,docs-lint,docs-build,docs-artifact', + ], }, { label: 'Alchemy production deployment', diff --git a/scripts/test-docs-deployment.mjs b/scripts/test-docs-deployment.mjs index 3e60e90..224adff 100644 --- a/scripts/test-docs-deployment.mjs +++ b/scripts/test-docs-deployment.mjs @@ -11,6 +11,11 @@ const options = { const commands = deploymentCommands(options); assert.equal(commands.filter(({ args }) => args.includes('deploy')).length, 1, 'promotion performs exactly one deployment'); assert.equal(commands.filter(({ args }) => args.includes('build')).length, 0, 'promotion never starts another build'); +assert.equal( + commands[0].args[commands[0].args.indexOf('--gates') + 1], + 'docs-content,docs-typecheck,docs-lint,docs-build,docs-artifact', + 'promotion verifies the same job-scoped docs receipt produced by CI', +); assert.equal(commands.find(({ args }) => args.includes('deploy')).env.PLANR_DOCS_RECEIPT_VALIDATED, '1'); assert.deepEqual(commands.map(({ label }) => label), ['reviewed receipt', 'Alchemy production deployment', 'bounded live oracle']); diff --git a/scripts/test-verification-runner.mjs b/scripts/test-verification-runner.mjs index 7616b3a..1ef7813 100644 --- a/scripts/test-verification-runner.mjs +++ b/scripts/test-verification-runner.mjs @@ -9,6 +9,7 @@ import { linuxTargetCommandPlan, runLinuxTargetVerification, runVerification, + selectVerificationGates, verifyLinuxTargetReceipt, verifyReceipt, } from "./verification-runner.mjs"; @@ -142,6 +143,23 @@ assert.equal(plan.filter(({ executable, args }) => [executable, ...args].join(" assert.equal(plan.filter(({ executable, args }) => [executable, ...args].join(" ").includes("cargo install")).length, 0); assert.equal(new Set(plan.map((entry) => JSON.stringify([entry.executable, entry.args]))).size, plan.length); +const fullDocsSelection = selectVerificationGates(fullSelection, [ + "docs-content", "docs-typecheck", "docs-lint", "docs-build", "docs-artifact", +]); +const fullDocsReceipt = runVerification({ + selection: fullDocsSelection, + repoRoot: root, + execute: () => ({ status: 0 }), +}); +assert.equal(fullDocsReceipt.verdict, "pass", "a full-profile docs job runs without parallel Linux receipts"); +assert.equal(fullDocsReceipt.commands.every(({ gate }) => gate.startsWith("docs-")), true); +assert.equal(fullDocsReceipt.commands.filter(({ gate }) => gate === "docs-build").length, 1); +assert.throws( + () => selectVerificationGates(docsSelection, ["linux-portability"]), + /verification gate was not selected/, + "a job cannot claim a gate excluded by the exact change selection", +); + const releaseSelection = classifyChanges([{ status: "M", path: "scripts/release.sh" }]); const releasePlan = commandPlanFor(releaseSelection); for (const candidatePlan of [releasePlan, plan]) { diff --git a/scripts/verification-runner.mjs b/scripts/verification-runner.mjs index 1c3cc1b..d63b0c1 100644 --- a/scripts/verification-runner.mjs +++ b/scripts/verification-runner.mjs @@ -17,7 +17,7 @@ import { classifyChanges, parseGitNameStatus, POLICY_DIGEST, POLICY_VERSION } fr export const RECEIPT_SCHEMA = "planr.verification-receipt.v3"; export const LINUX_TARGET_RECEIPT_SCHEMA = "planr.linux-target-receipt.v1"; -export const RUNNER_VERSION = "1.2.0"; +export const RUNNER_VERSION = "1.2.1"; const LINUX_TARGETS = deepFreeze({ "linux-x86_64": { cargoTarget: "x86_64-unknown-linux-musl", hostArchitecture: "x64" }, @@ -82,6 +82,26 @@ export function commandPlanFor(selection) { return plan; } +export function selectVerificationGates(selection, requestedGates) { + assertSelection(selection); + if (!Array.isArray(requestedGates) || requestedGates.length === 0) { + throw new Error("at least one verification gate is required"); + } + if (new Set(requestedGates).size !== requestedGates.length) { + throw new Error("verification gates must be unique"); + } + for (const gate of requestedGates) { + if (!selection.selectedGates.includes(gate)) { + throw new Error(`verification gate was not selected for this change: ${gate}`); + } + } + return { + ...selection, + selectedGates: [...requestedGates], + reasons: selection.reasons.filter(({ gate }) => requestedGates.includes(gate)), + }; +} + export function linuxTargetCommandPlan(target) { const config = LINUX_TARGETS[target]; if (!config) throw new Error(`unknown Linux release target: ${target}`); @@ -598,10 +618,12 @@ function selectionFromCli(values, root) { } else { throw new Error("--input, --base, or --profile is required"); } - const selection = classifyChanges(changes, { baseRevision: base ?? null, headRevision: head }); + let selection = classifyChanges(changes, { baseRevision: base ?? null, headRevision: head }); if (explicitProfile && explicitProfile !== selection.profile) { throw new Error(`explicit profile ${explicitProfile} does not match classified profile ${selection.profile}`); } + const requestedGates = values.get("--gates"); + if (requestedGates) selection = selectVerificationGates(selection, requestedGates.split(",")); return selection; } From ccc548a1f66d7b35c0b6ec89fe69c5fae7526389 Mon Sep 17 00:00:00 2001 From: Kevin Kern Date: Tue, 28 Jul 2026 13:44:41 +0200 Subject: [PATCH 3/5] test: align verification with current docs and CI --- apps/docs/scripts/verify-static-deployment.mjs | 2 -- tests/grok_contract.rs | 1 - 2 files changed, 3 deletions(-) diff --git a/apps/docs/scripts/verify-static-deployment.mjs b/apps/docs/scripts/verify-static-deployment.mjs index 2f43ec8..6c62742 100644 --- a/apps/docs/scripts/verify-static-deployment.mjs +++ b/apps/docs/scripts/verify-static-deployment.mjs @@ -45,8 +45,6 @@ for (const route of [ '/docs/integrations/cursor', '/docs/integrations/grok-build', '/docs/integrations/pi', - '/docs/integrations/generic-mcp', - '/docs/integrations/cli-only', ]) { assert.ok(agentQuickstart.includes(route), `static agent quickstart omits ${route}`); } diff --git a/tests/grok_contract.rs b/tests/grok_contract.rs index 42e8fa2..d766323 100644 --- a/tests/grok_contract.rs +++ b/tests/grok_contract.rs @@ -174,7 +174,6 @@ fn grok_release_boundary_has_no_credentials_live_calls_or_runtime_dependency() { let production_release_files = [ ".github/workflows/ci.yml", ".github/workflows/release.yml", - ".github/workflows/security.yml", "scripts/build-release.sh", "scripts/build-linux-release.sh", "scripts/prepare-release-candidate.sh", From 4100a2521a92c5451b11988b123719fc1a5cb084 Mon Sep 17 00:00:00 2001 From: Kevin Kern Date: Tue, 28 Jul 2026 13:48:06 +0200 Subject: [PATCH 4/5] test: remove stale release assumptions --- apps/docs/scripts/verify-static-deployment.mjs | 6 +++++- tests/pi_contract.rs | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/docs/scripts/verify-static-deployment.mjs b/apps/docs/scripts/verify-static-deployment.mjs index 6c62742..f249871 100644 --- a/apps/docs/scripts/verify-static-deployment.mjs +++ b/apps/docs/scripts/verify-static-deployment.mjs @@ -51,7 +51,11 @@ for (const route of [ assert.ok(!/planr install (codex|claude|cursor|grok)/.test(agentQuickstart), 'static agent quickstart duplicated a runtime setup recipe'); const grokGuide = await readFile(path.join(outputRoot, 'docs', 'integrations', 'grok-build.md'), 'utf8'); -for (const marker of ['Planr 1.8.0', 'planr install grok', 'no xAI credential is required']) { +for (const marker of [ + 'planr install grok', + 'no xAI credential is required', + 'Use $planr. Inspect this repository and tell me the verified next step.', +]) { assert.ok(grokGuide.includes(marker), `static Grok guide omits ${marker}`); } diff --git a/tests/pi_contract.rs b/tests/pi_contract.rs index bd201e5..0d6fb49 100644 --- a/tests/pi_contract.rs +++ b/tests/pi_contract.rs @@ -158,7 +158,6 @@ fn pi_release_boundary_has_no_live_runtime_or_dependency() { let production_release_files = [ ".github/workflows/ci.yml", ".github/workflows/release.yml", - ".github/workflows/security.yml", "scripts/build-release.sh", "scripts/build-linux-release.sh", "scripts/prepare-release-candidate.sh", From 71861725def8db352e0c9e54d8f8bf796b8bcd5f Mon Sep 17 00:00:00 2001 From: Kevin Kern Date: Tue, 28 Jul 2026 13:51:30 +0200 Subject: [PATCH 5/5] test: bind Grok artifact marker to current copy --- apps/docs/scripts/verify-static-deployment.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/docs/scripts/verify-static-deployment.mjs b/apps/docs/scripts/verify-static-deployment.mjs index f249871..6c364c7 100644 --- a/apps/docs/scripts/verify-static-deployment.mjs +++ b/apps/docs/scripts/verify-static-deployment.mjs @@ -53,7 +53,7 @@ assert.ok(!/planr install (codex|claude|cursor|grok)/.test(agentQuickstart), 'st const grokGuide = await readFile(path.join(outputRoot, 'docs', 'integrations', 'grok-build.md'), 'utf8'); for (const marker of [ 'planr install grok', - 'no xAI credential is required', + 'contains no xAI credential', 'Use $planr. Inspect this repository and tell me the verified next step.', ]) { assert.ok(grokGuide.includes(marker), `static Grok guide omits ${marker}`);