diff --git a/.github/workflows/assurance-assessment.yml b/.github/workflows/assurance-assessment.yml new file mode 100644 index 00000000..c908acfe --- /dev/null +++ b/.github/workflows/assurance-assessment.yml @@ -0,0 +1,657 @@ +# Stage 3 of Bomly's release assurance framework: the exhaustive pass, run +# against the binaries users actually download. It cannot change the release — +# published releases are immutable — so its job is to measure and to publish an +# honest report, and to open a tracking issue when something is wrong. +# +# The report it writes (docs/assurance/reports/.json) plus the index beside +# it are the only data source for bomly.dev/assurance. +name: Release assessment + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: Release tag to assess, such as v0.24.0. + type: string + required: true + +permissions: + contents: read + +concurrency: + group: release-assessment-${{ github.event.release.tag_name || inputs.tag }} + cancel-in-progress: false + +jobs: + resolve: + name: Resolve the release + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + tag: ${{ steps.release.outputs.tag }} + version: ${{ steps.release.outputs.version }} + commit: ${{ steps.release.outputs.commit }} + url: ${{ steps.release.outputs.url }} + published_at: ${{ steps.release.outputs.published_at }} + steps: + - name: Look up the release + id: release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name || inputs.tag }} + run: | + set -euo pipefail + release_json="$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}")" + published_at="$(echo "${release_json}" | jq -r '.published_at // ""')" + url="$(echo "${release_json}" | jq -r '.html_url')" + draft="$(echo "${release_json}" | jq -r '.draft')" + if [ "${draft}" = "true" ]; then + echo "::error::${TAG} is still a draft. The assessment describes published releases." + exit 1 + fi + commit="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${TAG}" --jq '.sha')" + { + echo "tag=${TAG}" + echo "version=${TAG#v}" + echo "commit=${commit}" + echo "url=${url}" + echo "published_at=${published_at}" + } >> "${GITHUB_OUTPUT}" + + install-scripts: + name: Install script (${{ matrix.platform.name }}) + needs: resolve + strategy: + fail-fast: false + matrix: + platform: + - name: ubuntu + os: ubuntu-latest + - name: macos + os: macos-latest + - name: windows + os: windows-latest + runs-on: ${{ matrix.platform.os }} + timeout-minutes: 20 + steps: + - name: Check out the released source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.tag }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Run the published install script + id: install + continue-on-error: true + shell: bash + env: + BOMLY_VERSION: ${{ needs.resolve.outputs.tag }} + BOMLY_INSTALL_DIR: ${{ runner.temp }}/bomly-install + run: | + set -euo pipefail + mkdir -p "${BOMLY_INSTALL_DIR}" + if [ "${RUNNER_OS}" = "Windows" ]; then + pwsh -NoProfile -File scripts/install.ps1 \ + -Version "${BOMLY_VERSION}" -InstallDir "${BOMLY_INSTALL_DIR}" + "${BOMLY_INSTALL_DIR}/bomly.exe" version | tee "${RUNNER_TEMP}/version.txt" + else + sh scripts/install.sh + "${BOMLY_INSTALL_DIR}/bomly" version | tee "${RUNNER_TEMP}/version.txt" + fi + + - name: Record the install result + if: always() + shell: bash + env: + BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} + INSTALL_OUTCOME: ${{ steps.install.outcome }} + VERSION: ${{ needs.resolve.outputs.version }} + run: | + set -euo pipefail + exit_code=0 + reported="$(head -n 1 "${RUNNER_TEMP}/version.txt" 2>/dev/null || echo "")" + summary="The published install script installed Bomly ${VERSION} on ${RUNNER_OS} and the binary reported: ${reported}" + if [ "${INSTALL_OUTCOME}" != "success" ]; then + exit_code=1 + summary="The published install script failed on ${RUNNER_OS}. Open the install step for the exact command that failed." + elif ! printf '%s' "${reported}" | grep -q "${VERSION}"; then + exit_code=1 + summary="The install script ran on ${RUNNER_OS} but the installed binary reported '${reported}' instead of ${VERSION}." + fi + go run ./internal/assurance/cmd emit \ + --id install-script --instance '${{ matrix.platform.name }}' \ + --exit-code "${exit_code}" --summary "${summary}" \ + --out assurance-results --step-summary + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-install-${{ matrix.platform.name }} + path: assurance-results + if-no-files-found: error + retention-days: 30 + + public-download: + name: Public download + needs: resolve + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out the released source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.tag }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + # Unauthenticated, from the public URL, exactly as a user would. + - name: Download every published file + id: download + continue-on-error: true + env: + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/assets" + base="https://github.com/${GITHUB_REPOSITORY}/releases/download/${TAG}" + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/SHA256SUMS" "${base}/SHA256SUMS" + while read -r _ name; do + name="${name#\*}" + echo "downloading ${name}" + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/${name}" "${base}/${name}" + done < "${RUNNER_TEMP}/assets/SHA256SUMS" + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/SHA256SUMS.sigstore.json" "${base}/SHA256SUMS.sigstore.json" + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/multiple.intoto.jsonl" "${base}/multiple.intoto.jsonl" + + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + # Only the checksums are re-checked here: the pre-release stage already + # extracted and ran the binaries, so what is new after publication is + # whether the public copies are complete and unaltered. + # + # The signature is verified first. Checking assets against a checksum + # list downloaded beside them only proves the two agree; the signature is + # what ties that list back to this repository's release workflow, so a + # replacement of both files cannot pass. + - name: Check the public copies against the signed checksum list + id: verify + continue-on-error: true + run: | + set -euo pipefail + cd "${RUNNER_TEMP}/assets" + cosign verify-blob SHA256SUMS \ + --bundle SHA256SUMS.sigstore.json \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + --certificate-identity-regexp '^https://github.com/bomly-dev/bomly-cli/\.github/workflows/release\.yml@refs/tags/' + sha256sum --check --strict SHA256SUMS + + - name: Record the public download result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} + DOWNLOAD_OUTCOME: ${{ steps.download.outcome }} + VERIFY_OUTCOME: ${{ steps.verify.outcome }} + run: | + set -euo pipefail + exit_code=0 + count="$(find "${RUNNER_TEMP}/assets" -type f | wc -l | tr -d ' ')" + summary="All ${count} published files downloaded from their public URLs without credentials and match the signed checksum list." + if [ "${DOWNLOAD_OUTCOME}" != "success" ]; then + exit_code=1 + summary="At least one published file could not be downloaded from its public URL." + elif [ "${VERIFY_OUTCOME}" != "success" ]; then + exit_code=1 + summary="The published files downloaded, but the checksum list signature or at least one file did not verify." + fi + go run ./internal/assurance/cmd emit \ + --id public-download --exit-code "${exit_code}" --summary "${summary}" \ + --metric files="${count}" --out assurance-results --step-summary + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-public-download + path: assurance-results + if-no-files-found: error + retention-days: 30 + + released-scan: + name: Released binary scan (${{ matrix.slice.name }}) + needs: resolve + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + slice: + - name: go + run: 'TestScan$/scan-go$|TestDiff/diff-go|TestExplain/explain-go' + - name: node + run: 'TestScan$/scan-npm$' + - name: sbom + run: 'TestScan$/scan-sbom' + steps: + - name: Check out the released source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.tag }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Download the released binaries + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.resolve.outputs.tag }} + VERSION: ${{ needs.resolve.outputs.version }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/release" + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" --dir "${RUNNER_TEMP}/release" \ + --pattern "bomly_${VERSION}_linux_amd64.tar.gz" \ + --pattern "bomly-lite_${VERSION}_linux_amd64.tar.gz" \ + --pattern SHA256SUMS + (cd "${RUNNER_TEMP}/release" && sha256sum --check --ignore-missing SHA256SUMS) + tar -xzf "${RUNNER_TEMP}/release/bomly_${VERSION}_linux_amd64.tar.gz" -C "${RUNNER_TEMP}/release" bomly + tar -xzf "${RUNNER_TEMP}/release/bomly-lite_${VERSION}_linux_amd64.tar.gz" -C "${RUNNER_TEMP}/release" bomly-lite + chmod +x "${RUNNER_TEMP}/release/bomly" "${RUNNER_TEMP}/release/bomly-lite" + + # The same golden files the source tree is checked against, driven by the + # binary the release shipped. + - name: Scan real projects with the released binary + id: scan + continue-on-error: true + env: + BOMLY_SMOKE_BINARY: ${{ runner.temp }}/release/bomly + BOMLY_SMOKE_LITE_BINARY: ${{ runner.temp }}/release/bomly-lite + run: | + set -o pipefail + go test -tags smoke ./test/smoke/ -json -count=1 -timeout 30m \ + -run '${{ matrix.slice.run }}' 2>&1 | tee "${RUNNER_TEMP}/scan.jsonl" > /dev/null + + - name: Record the scan result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} + SCAN_OUTCOME: ${{ steps.scan.outcome }} + run: | + exit_code=0 + if [ "${SCAN_OUTCOME}" != "success" ]; then + exit_code=1 + fi + go run ./internal/assurance/cmd gotest \ + --id released-scan --instance '${{ matrix.slice.name }}' \ + --input "${RUNNER_TEMP}/scan.jsonl" --exit-code "${exit_code}" \ + --echo --out assurance-results --step-summary + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-released-scan-${{ matrix.slice.name }} + path: assurance-results + if-no-files-found: error + retention-days: 30 + + sbom: + name: SBOM interoperability + needs: resolve + uses: ./.github/workflows/sbom-interoperability.yml + with: + ref: ${{ needs.resolve.outputs.tag }} + release_tag: ${{ needs.resolve.outputs.tag }} + assurance: true + + perf-samples: + name: Repeated scan timing + needs: resolve + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out the released source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.tag }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Download the released lite binary + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.resolve.outputs.tag }} + VERSION: ${{ needs.resolve.outputs.version }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/release" bin + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" --dir "${RUNNER_TEMP}/release" \ + --pattern "bomly-lite_${VERSION}_linux_amd64.tar.gz" --pattern SHA256SUMS + (cd "${RUNNER_TEMP}/release" && sha256sum --check --ignore-missing SHA256SUMS) + tar -xzf "${RUNNER_TEMP}/release/bomly-lite_${VERSION}_linux_amd64.tar.gz" \ + -C "${RUNNER_TEMP}/release" bomly-lite + install -m 0755 "${RUNNER_TEMP}/release/bomly-lite" bin/bomly-lite + + - name: Measure repeated scans + id: perf + continue-on-error: true + run: | + go run ./internal/assurance/perfrun -output .benchmark-runs/performance \ + -case canonical-sbom-scan -samples 5 -network-state offline -- \ + ./bin/bomly-lite scan --sbom --path test/smoke/testdata/sboms/go.spdx.json \ + --detectors sbom --format json + + - name: Record the timing result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + manifest=".benchmark-runs/performance/run-manifest.json" + if [ -f "${manifest}" ]; then + go run ./internal/assurance/cmd convert benchmark-run \ + --id perf-samples --input "${manifest}" \ + --out assurance-results --step-summary + else + go run ./internal/assurance/cmd emit \ + --id perf-samples --status fail \ + --summary "The performance run stopped before it could write a manifest." \ + --out assurance-results --step-summary + fi + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-perf-samples + path: assurance-results + if-no-files-found: error + retention-days: 30 + + report: + name: Publish the release report + # always(), so a failed check still gets a published report — but only when + # the release itself was resolved, since everything below needs its tag. + if: always() && needs.resolve.result == 'success' + needs: [resolve, install-scripts, public-download, released-scan, sbom, perf-samples] + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Mint release bot token + # Commits the report to main and dispatches to the landing page. The + # same app already pushes the version-bump commits. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: app-token + with: + client-id: ${{ vars.RELEASE_BOT_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} + owner: bomly-dev + repositories: | + bomly-cli + bomly-landing-page + permission-contents: write + + - name: Mint issue token + # Issues is a newer grant on the release app than contents. Minting it + # separately means an app that has not been granted it yet loses the + # tracking issue, not the whole report. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: issue-token + continue-on-error: true + with: + client-id: ${{ vars.RELEASE_BOT_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} + owner: bomly-dev + repositories: bomly-cli + permission-issues: write + + - name: Check out main + # The report is committed to main, and the report tooling comes from + # main so a fix can be applied to an old release's report. The catalog, + # though, must be the one the checks actually ran against — see below. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + persist-credentials: true + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Collect this stage's check results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: assurance-* + merge-multiple: true + path: assurance-results + + # Stages 1 and 2 ran in other workflow runs. Their results are pulled in + # so the report covers the whole release, and anything that cannot be + # found stays absent — the report then shows those checks as missing + # rather than pretending they passed. + - name: Collect the earlier stages' check results + continue-on-error: true + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + TAG: ${{ needs.resolve.outputs.tag }} + COMMIT: ${{ needs.resolve.outputs.commit }} + run: | + set -uo pipefail + parent="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${COMMIT}" --jq '.parents[0].sha // ""')" + + # A workflow called with `uses:` produces no run of its own — the + # jobs live in the caller's run (Auto Version). So the search is over + # every run for the commit, looking for the verdict job by name. + find_verdict_run() { + local sha="$1" + [ -n "${sha}" ] || return 1 + local run_id + for run_id in $(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${sha}&per_page=50" \ + --jq '.workflow_runs[].id'); do + if gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?per_page=100" \ + --jq '.jobs[] | select(.conclusion == "success") | .name' \ + | grep -q 'Prerequisites verdict$'; then + echo "${run_id}" + return 0 + fi + done + return 1 + } + + prerequisites_run="" + for candidate in "${COMMIT}" "${parent}"; do + if prerequisites_run="$(find_verdict_run "${candidate}")"; then + break + fi + prerequisites_run="" + done + if [ -n "${prerequisites_run}" ]; then + gh run download "${prerequisites_run}" --repo "${GITHUB_REPOSITORY}" \ + --name assurance-stage-prerequisites --dir assurance-results || true + echo "prerequisites_run_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${prerequisites_run}" >> "${GITHUB_ENV}" + fi + + release_run="$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/release.yml/runs?per_page=50" \ + | jq -r --arg tag "${TAG}" '[.workflow_runs[] | select(.head_branch == $tag)][0].id // ""')" + if [ -n "${release_run}" ]; then + gh run download "${release_run}" --repo "${GITHUB_REPOSITORY}" \ + --name assurance-stage-pre-release --dir assurance-results || true + echo "pre_release_run_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${release_run}" >> "${GITHUB_ENV}" + fi + + # A release is judged against the catalog that existed when it was made, + # so a check added later is not counted as missing from an older release. + - name: Take the catalog from the released tag + env: + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + git fetch --quiet origin "refs/tags/${TAG}:refs/tags/${TAG}" || true + if git show "${TAG}:docs/assurance/catalog.json" > "${RUNNER_TEMP}/catalog.json" 2>/dev/null; then + echo "catalog=${RUNNER_TEMP}/catalog.json" >> "${GITHUB_ENV}" + else + echo "${TAG} predates the assurance catalog; using the one on main." + echo "catalog=docs/assurance/catalog.json" >> "${GITHUB_ENV}" + fi + + - name: Build the release report + id: report + continue-on-error: true + env: + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -eo pipefail + go run ./internal/assurance/cmd report \ + --results assurance-results \ + --catalog "${catalog}" --out docs/assurance \ + --tag "${TAG}" \ + --commit '${{ needs.resolve.outputs.commit }}' \ + --url '${{ needs.resolve.outputs.url }}' \ + --published-at '${{ needs.resolve.outputs.published_at }}' \ + --prerequisites-run "${prerequisites_run_url:-}" \ + --pre-release-run "${pre_release_run_url:-}" \ + --assessment-run "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --summary-out "${RUNNER_TEMP}/assurance-summary.md" \ + --step-summary --allow-unknown + echo "verdict=$(jq -r '.verdict.overall' "docs/assurance/reports/${TAG}.json")" >> "${GITHUB_OUTPUT}" + + - name: Commit the report to main + if: steps.report.outcome == 'success' + env: + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + git config user.name "${{ steps.app-token.outputs.app-slug }}[bot]" + git config user.email "${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com" + git add "docs/assurance/reports/${TAG}.json" docs/assurance/index.json + if git diff --cached --quiet; then + echo "The report is already committed and unchanged." + exit 0 + fi + git commit -m "docs(assurance): release report for ${TAG} [skip ci]" + # main moves while this runs; rebase onto whatever landed and retry. + for attempt in 1 2 3 4 5; do + if git push origin HEAD:main; then + exit 0 + fi + echo "push attempt ${attempt} failed; rebasing onto origin/main" + git fetch origin main + git rebase origin/main + done + echo "::error::Could not push the assurance report to main after five attempts." + exit 1 + + - name: Tell the landing page a report is available + if: steps.report.outcome == 'success' + continue-on-error: true + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + gh api repos/bomly-dev/bomly-landing-page/dispatches \ + --method POST \ + --input - </dev/null 2>&1 || true + + existing="$(gh issue list --repo "${GITHUB_REPOSITORY}" --state all \ + --label release-assurance --search "\"${title}\" in:title" \ + --json number,title,state --jq \ + ".[] | select(.title == \"${title}\") | \"\(.number) \(.state)\"" | head -n1)" + number="${existing%% *}" + state="${existing##* }" + + body="${RUNNER_TEMP}/assurance-summary.md" + if [ ! -f "${body}" ]; then + printf '# %s\n\nThe assessment could not build a report for this release. Open the run for the cause: %s/%s/actions/runs/%s\n' \ + "${title}" "${GITHUB_SERVER_URL}" "${GITHUB_REPOSITORY}" "${GITHUB_RUN_ID}" > "${body}" + fi + + if [ "${REPORT_OUTCOME}" = "success" ] && [ "${VERDICT}" = "pass" ]; then + if [ -n "${number}" ] && [ "${state}" = "OPEN" ]; then + gh issue comment "${number}" --repo "${GITHUB_REPOSITORY}" \ + --body "A later assessment run of ${TAG} passed every check. Closing." + gh issue close "${number}" --repo "${GITHUB_REPOSITORY}" + fi + exit 0 + fi + + if [ -n "${number}" ]; then + gh issue edit "${number}" --repo "${GITHUB_REPOSITORY}" --body-file "${body}" + if [ "${state}" != "OPEN" ]; then + gh issue reopen "${number}" --repo "${GITHUB_REPOSITORY}" + fi + gh issue comment "${number}" --repo "${GITHUB_REPOSITORY}" \ + --body "The assessment ran again and still reports problems. See the updated description." + else + gh issue create --repo "${GITHUB_REPOSITORY}" --title "${title}" \ + --label release-assurance --body-file "${body}" + fi + + - name: Fail when the release did not pass + if: always() && (steps.report.outcome != 'success' || steps.report.outputs.verdict != 'pass') + run: | + echo "::error::The post-release assessment did not pass. The release is already public; follow the tracking issue." + exit 1 diff --git a/.github/workflows/assurance-prerequisites.yml b/.github/workflows/assurance-prerequisites.yml new file mode 100644 index 00000000..af64c556 --- /dev/null +++ b/.github/workflows/assurance-prerequisites.yml @@ -0,0 +1,170 @@ +# Stage 1 of Bomly's release assurance framework: everything that must hold +# before a version is tagged. It runs on the source tree, so a failure here is +# fixed by an ordinary pull request rather than by a broken release. +# +# Auto Version calls this workflow and only tags when it passes. Release +# preflight then looks for a successful run against the commit being released, +# so a hand-made tag cannot skip it. +# +# The checks it collects are declared in docs/assurance/catalog.json. +name: Release prerequisites + +on: + workflow_dispatch: + inputs: + ref: + description: Commit, branch, or tag to check (defaults to this ref). + type: string + required: false + default: "" + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check. + type: string + required: false + default: "" + assurance_tag: + description: Release tag recorded in the check results. + type: string + required: false + default: "" + +permissions: + contents: read + +concurrency: + group: assurance-prerequisites-${{ inputs.ref || github.ref }} + cancel-in-progress: false + +jobs: + smoke: + name: End-to-end scans + uses: ./.github/workflows/smoke.yml + with: + ref: ${{ inputs.ref || github.ref }} + assurance: true + assurance_tag: ${{ inputs.assurance_tag }} + + portable: + name: Platform stability + uses: ./.github/workflows/portable-assurance.yml + with: + ref: ${{ inputs.ref || github.ref }} + assurance: true + assurance_tag: ${{ inputs.assurance_tag }} + + # Fuzzing is advisory: a finding is recorded in the report and does not stop a + # release. `continue-on-error` is not allowed on a job that calls a reusable + # workflow, so fuzz.yml itself skips its failing step when it is called in + # assurance mode; the verdict job below is what decides the stage. + fuzz: + name: Parser fuzzing + uses: ./.github/workflows/fuzz.yml + with: + ref: ${{ inputs.ref || github.ref }} + fuzztime: 45s + assurance: true + assurance_tag: ${{ inputs.assurance_tag }} + + catalog: + name: Assurance catalog + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Validate the assurance catalog + id: catalog + continue-on-error: true + run: make assurance-catalog + + - name: Record the catalog result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + CATALOG_OUTCOME: ${{ steps.catalog.outcome }} + run: | + set -euo pipefail + exit_code=0 + summary="The assurance catalog is valid and every fixture and expected-result file it names still matches its recorded checksum." + if [ "${CATALOG_OUTCOME}" != "success" ]; then + exit_code=1 + summary="The assurance catalog is invalid or one of the files it names has changed. Open the validation step for the exact entry." + fi + checks="$(grep -c '"stage":' docs/assurance/catalog.json || true)" + # --stage and --level are explicit: when the catalog is the thing + # that is broken, the emitter cannot look them up in it. + go run ./internal/assurance/cmd emit \ + --id catalog-valid --stage prerequisites --level gate \ + --exit-code "${exit_code}" --summary "${summary}" \ + --metric checks="${checks}" --out assurance-results --step-summary + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-catalog + path: assurance-results + if-no-files-found: error + retention-days: 7 + + - name: Fail the job when the catalog is invalid + if: steps.catalog.outcome == 'failure' + run: exit 1 + + verdict: + name: Prerequisites verdict + if: always() + needs: [smoke, portable, fuzz, catalog] + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Collect check results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: assurance-* + merge-multiple: true + path: assurance-results + + - name: Judge the prerequisites stage + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + run: | + go run ./internal/assurance/cmd verdict \ + --results assurance-results --stage prerequisites \ + --tag "${BOMLY_ASSURANCE_TAG}" \ + --json assurance-results/stage-prerequisites.json --step-summary + + - name: Upload the stage report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-stage-prerequisites + path: assurance-results + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/auto-version.yml b/.github/workflows/auto-version.yml index e6c7c051..e2d9e207 100644 --- a/.github/workflows/auto-version.yml +++ b/.github/workflows/auto-version.yml @@ -11,6 +11,11 @@ on: - patch - minor - major + skip_prerequisites: + description: Tag without running the release prerequisites stage (emergency use). + required: false + type: boolean + default: false permissions: contents: read @@ -20,7 +25,20 @@ concurrency: cancel-in-progress: false jobs: + # Stage 1 of release assurance, on the commit that is about to be tagged. + # Running it before the tag exists means a stale golden file or an + # intermittent failure is fixed by a normal pull request instead of leaving a + # broken release behind. + prerequisites: + name: Release prerequisites + if: ${{ !inputs.skip_prerequisites }} + uses: ./.github/workflows/assurance-prerequisites.yml + with: + ref: ${{ github.sha }} + bump-tag-and-release: + needs: prerequisites + if: ${{ always() && (needs.prerequisites.result == 'success' || inputs.skip_prerequisites) }} runs-on: ubuntu-latest environment: release @@ -41,6 +59,16 @@ jobs: token: ${{ steps.app-token.outputs.token }} persist-credentials: true + - name: Require the prerequisites stage to have run on this commit + shell: bash + env: + SKIPPED: ${{ inputs.skip_prerequisites }} + run: | + set -euo pipefail + if [[ "${SKIPPED}" == "true" ]]; then + echo "::warning::Release prerequisites were skipped for this tag. The release report will show every prerequisite check as missing." + fi + - name: Require main branch shell: bash run: | diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 4727dfde..4fe2c268 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -8,6 +8,28 @@ on: workflow_dispatch: schedule: - cron: "0 8 * * *" + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check out. + type: string + required: false + default: "" + fuzztime: + description: Time budget per fuzz target. + type: string + required: false + default: 2m + assurance: + description: Upload a release assurance check result. + type: boolean + required: false + default: false + assurance_tag: + description: Release tag recorded in the check result. + type: string + required: false + default: "" permissions: contents: read @@ -24,6 +46,7 @@ jobs: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ inputs.ref || github.ref }} persist-credentials: false - name: Set up Go @@ -34,10 +57,58 @@ jobs: cache-dependency-path: go.sum - name: Run native Go fuzz targets - run: make fuzz FUZZTIME=2m + id: fuzz + continue-on-error: true + env: + FUZZ_RESULTS_JSONL: ${{ runner.temp }}/fuzz-details.jsonl + FUZZTIME: ${{ inputs.fuzztime || '2m' }} + run: make fuzz FUZZTIME="${FUZZTIME}" + + - name: Record the fuzz result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + FUZZ_OUTCOME: ${{ steps.fuzz.outcome }} + FUZZTIME: ${{ inputs.fuzztime || '2m' }} + run: | + set -euo pipefail + details="${RUNNER_TEMP}/fuzz-details.jsonl" + targets=0 + failed=0 + if [ -f "${details}" ]; then + targets="$(wc -l < "${details}" | tr -d ' ')" + failed="$(grep -c '"exit_code":[^0]' "${details}" || true)" + fi + exit_code=0 + summary="${targets} fuzz targets ran for ${FUZZTIME} each against their seed corpus without finding a crash." + if [ "${FUZZ_OUTCOME}" != "success" ]; then + exit_code=1 + summary="${failed} of ${targets} fuzz targets found a failing input. Reproducers are attached to this run." + fi + # The details file is absent when the fuzz step died before writing + # it; the check still has to report, so the flag is conditional. + details_arg="" + if [ -f "${details}" ]; then + details_arg="--details-jsonl ${details}" + fi + # shellcheck disable=SC2086 # details_arg is a flag pair or empty + go run ./internal/assurance/cmd emit \ + --id fuzz --exit-code "${exit_code}" --summary "${summary}" \ + --metric targets="${targets}" --metric targets_failed="${failed}" \ + ${details_arg} \ + --out assurance-results --step-summary + + - name: Upload check result + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-fuzz + path: assurance-results + if-no-files-found: error + retention-days: 7 - name: Upload fuzz failures - if: failure() + if: steps.fuzz.outcome == 'failure' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: go-fuzz-failures @@ -45,3 +116,10 @@ jobs: **/testdata/fuzz/** if-no-files-found: ignore retention-days: 7 + + # In the release prerequisites stage fuzzing is advisory: the finding is + # recorded in the check result and the stage verdict decides what happens. + # On the nightly and manual runs a finding still fails the job. + - name: Fail the job when a fuzz target failed + if: steps.fuzz.outcome == 'failure' && !inputs.assurance + run: exit 1 diff --git a/.github/workflows/notify-landing-yank.yml b/.github/workflows/notify-landing-yank.yml index 3b44fcd2..0a0d89f1 100644 --- a/.github/workflows/notify-landing-yank.yml +++ b/.github/workflows/notify-landing-yank.yml @@ -32,7 +32,7 @@ jobs: permission-contents: write - name: Trigger docs sync on landing page (published) - if: github.event.action == 'published' + if: github.event.action == 'published' && github.event.release.draft == false env: GH_TOKEN: ${{ steps.landing-token.outputs.token }} TAG: ${{ github.event.release.tag_name }} @@ -45,8 +45,15 @@ jobs: {"event_type":"bomly-release","client_payload":{"version":"${TAG}","publishedAt":"${PUBLISHED_AT}"}} EOF + # A failed pre-release assurance gate leaves a draft release that a + # maintainer deletes. Deleting a draft fires the same `deleted` event as + # yanking a live release, so deletions of drafts are ignored: nothing was + # ever published for that version. `unpublished` always refers to a live + # release (it is what turns one back into a draft), so it is not guarded. - name: Trigger docs removal on landing page (yanked) - if: github.event.action == 'deleted' || github.event.action == 'unpublished' + if: >- + github.event.action == 'unpublished' || + (github.event.action == 'deleted' && github.event.release.draft == false) env: GH_TOKEN: ${{ steps.landing-token.outputs.token }} TAG: ${{ github.event.release.tag_name }} @@ -59,7 +66,9 @@ jobs: EOF winget-yank: - if: github.event.action == 'deleted' || github.event.action == 'unpublished' + if: >- + github.event.action == 'unpublished' || + (github.event.action == 'deleted' && github.event.release.draft == false) runs-on: ubuntu-latest timeout-minutes: 15 steps: diff --git a/.github/workflows/portable-assurance.yml b/.github/workflows/portable-assurance.yml index 3522b404..23811733 100644 --- a/.github/workflows/portable-assurance.yml +++ b/.github/workflows/portable-assurance.yml @@ -1,7 +1,33 @@ +# Repeated unit tests across supported platforms and cross-builds of every +# release binary. The full suite is repeated by the `portable` job (twice on +# each of three platforms); this job repeats only the Java detector suites, +# which is where intermittent failures have actually appeared. Runs on its own when a maintainer starts it, and as part of +# the "Release prerequisites" stage before a version is tagged. +# +# Every step ends by writing a release assurance check result; the job summary +# is rendered from that same result, so the summary and the report can never +# disagree. Check ids used here must stay declared in docs/assurance/catalog.json. name: Portable stability assurance on: workflow_dispatch: + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check out. + type: string + required: false + default: "" + assurance: + description: Upload release assurance check results. + type: boolean + required: false + default: false + assurance_tag: + description: Release tag recorded in the check results. + type: string + required: false + default: "" permissions: contents: read @@ -17,6 +43,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -27,17 +55,23 @@ jobs: - name: Repeat portable suite twice id: portable-tests + continue-on-error: true shell: bash run: | - set -euo pipefail + set -uo pipefail completed=0 failed=0 + : > "${RUNNER_TEMP}/portable-details.jsonl" for iteration in 1 2; do echo "portable suite iteration ${iteration}" if go test ./... -count=1; then completed="${iteration}" + printf '{"name":"iteration %s","status":"pass"}\n' "${iteration}" \ + >> "${RUNNER_TEMP}/portable-details.jsonl" else failed="${iteration}" + printf '{"name":"iteration %s","status":"fail"}\n' "${iteration}" \ + >> "${RUNNER_TEMP}/portable-details.jsonl" break fi done @@ -47,10 +81,11 @@ jobs: exit 1 fi - - name: Explain this platform result + - name: Record the platform result if: always() shell: bash env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} TEST_RESULT: ${{ steps.portable-tests.outcome }} COMPLETED: ${{ steps.portable-tests.outputs.completed }} FAILED: ${{ steps.portable-tests.outputs.failed }} @@ -58,35 +93,32 @@ jobs: set -euo pipefail completed="${COMPLETED:-0}" failed="${FAILED:-0}" - if [[ "${TEST_RESULT}" == "success" ]]; then - result="Passed" - icon="✅" - else - result="Failed" - icon="❌" + exit_code=0 + summary="The complete Go test suite passed twice on ${RUNNER_OS}/${RUNNER_ARCH}, with no reused results." + if [[ "${TEST_RESULT}" != "success" ]]; then + exit_code=1 + summary="The complete Go test suite failed on run ${failed} of 2 on ${RUNNER_OS}/${RUNNER_ARCH}. Open the Repeat portable suite twice step for the failing package and test." fi - if [[ "${failed}" == "0" ]]; then - failed_display="—" - else - failed_display="${failed}" - fi - { - echo "## ${icon} ${RUNNER_OS}/${RUNNER_ARCH}: ${result}" - echo - echo "The complete Go test suite was scheduled twice on this platform. Each run used \`-count=1\` so Go did not reuse test results." - echo - echo "| Planned runs | Completed runs | Failed run |" - echo "|---:|---:|---:|" - echo "| 2 | ${completed} | ${failed_display} |" - echo - echo "### Need more detail?" - echo - echo "Open the **Repeat portable suite twice** step to find the failing package and test. To reproduce the same check locally:" - echo - echo '```sh' - echo "go test ./... -count=1" - echo '```' - } >> "${GITHUB_STEP_SUMMARY}" + go run ./internal/assurance/cmd emit \ + --id unit-portable --instance '${{ matrix.os }}' \ + --exit-code "${exit_code}" --summary "${summary}" \ + --metric planned_runs=2 --metric completed_runs="${completed}" \ + --details-jsonl "${RUNNER_TEMP}/portable-details.jsonl" \ + --out assurance-results --step-summary + + - name: Upload check result + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-portable-${{ matrix.os }} + path: assurance-results + if-no-files-found: error + retention-days: 7 + + - name: Fail the job when the suite failed + if: steps.portable-tests.outcome == 'failure' + shell: bash + run: exit 1 linux-stability: name: Linux repeated stability @@ -94,6 +126,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -110,16 +144,20 @@ jobs: - name: Repeat Java suites ten times id: java-tests + continue-on-error: true run: | - set -euo pipefail + set -uo pipefail completed=0 failed=0 + : > "${RUNNER_TEMP}/java-details.jsonl" for iteration in $(seq 1 10); do echo "Java suite iteration ${iteration}" if go test ./internal/detectors/gradle ./internal/detectors/maven ./internal/detectors/sbt -count=1; then completed="${iteration}" + printf '{"name":"iteration %s","status":"pass"}\n' "${iteration}" >> "${RUNNER_TEMP}/java-details.jsonl" else failed="${iteration}" + printf '{"name":"iteration %s","status":"fail"}\n' "${iteration}" >> "${RUNNER_TEMP}/java-details.jsonl" break fi done @@ -129,33 +167,38 @@ jobs: exit 1 fi - - name: Repeat complete suite five times - id: complete-tests + - name: Record the Java stability result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + JAVA_RESULT: ${{ steps.java-tests.outcome }} + COMPLETED: ${{ steps.java-tests.outputs.completed }} + FAILED: ${{ steps.java-tests.outputs.failed }} run: | set -euo pipefail - completed=0 - failed=0 - for iteration in $(seq 1 5); do - echo "complete suite iteration ${iteration}" - if go test ./... -count=1; then - completed="${iteration}" - else - failed="${iteration}" - break - fi - done - echo "completed=${completed}" >> "${GITHUB_OUTPUT}" - echo "failed=${failed}" >> "${GITHUB_OUTPUT}" - if [[ "${failed}" != "0" ]]; then - exit 1 + completed="${COMPLETED:-0}" + exit_code=0 + summary="The Gradle, Maven, and sbt detector suites passed ten times in a row." + if [[ "${JAVA_RESULT}" != "success" ]]; then + exit_code=1 + summary="The Java detector suites failed on run ${FAILED:-0} of 10, which points at an intermittent failure." fi + go run ./internal/assurance/cmd emit \ + --id unit-repeat-java --exit-code "${exit_code}" --summary "${summary}" \ + --metric planned_runs=10 --metric completed_runs="${completed}" \ + --details-jsonl "${RUNNER_TEMP}/java-details.jsonl" \ + --out assurance-results --step-summary + # Every target is attempted, even after one fails, so the report shows the + # complete build matrix instead of stopping at the first problem. - name: Cross-build release targets id: release-builds + continue-on-error: true run: | - set -euo pipefail + set -uo pipefail completed=0 - failed="" + planned=0 + : > "${RUNNER_TEMP}/build-details.jsonl" targets=( "linux amd64" "linux arm64" @@ -164,165 +207,77 @@ jobs: "windows amd64" "windows arm64" ) + record() { + printf '{"name":"%s","status":"%s"}\n' "${1}" "${2}" >> "${RUNNER_TEMP}/build-details.jsonl" + } for target in "${targets[@]}"; do read -r target_os target_arch <<<"${target}" suffix="" if [[ "${target_os}" == "windows" ]]; then suffix=".exe" fi - echo "cross-build ${target_os}/${target_arch}" - if GOOS="${target_os}" GOARCH="${target_arch}" CGO_ENABLED=0 \ - go build -o "bin/bomly-${target_os}-${target_arch}${suffix}" ./cmd/bomly; then - completed=$((completed + 1)) - else - failed="${target_os}/${target_arch} full" - break - fi - if GOOS="${target_os}" GOARCH="${target_arch}" CGO_ENABLED=0 \ - go build -tags "bomly_external_syft,bomly_external_grype" \ - -o "bin/bomly-lite-${target_os}-${target_arch}${suffix}" ./cmd/bomly; then - completed=$((completed + 1)) - else - failed="${target_os}/${target_arch} lite" - break - fi + for variant in full lite; do + planned=$((planned + 1)) + echo "cross-build ${target_os}/${target_arch} ${variant}" + status=fail + if [[ "${variant}" == "full" ]]; then + if GOOS="${target_os}" GOARCH="${target_arch}" CGO_ENABLED=0 \ + go build -o "bin/bomly-${target_os}-${target_arch}${suffix}" ./cmd/bomly; then + status=pass + fi + else + if GOOS="${target_os}" GOARCH="${target_arch}" CGO_ENABLED=0 \ + go build -tags "bomly_external_syft,bomly_external_grype" \ + -o "bin/bomly-lite-${target_os}-${target_arch}${suffix}" ./cmd/bomly; then + status=pass + fi + fi + if [[ "${status}" == "pass" ]]; then + completed=$((completed + 1)) + fi + record "${target_os}/${target_arch} ${variant}" "${status}" + done done echo "completed=${completed}" >> "${GITHUB_OUTPUT}" - echo "failed=${failed}" >> "${GITHUB_OUTPUT}" - if [[ -n "${failed}" ]]; then + echo "planned=${planned}" >> "${GITHUB_OUTPUT}" + if [[ "${completed}" != "${planned}" ]]; then exit 1 fi - - name: Explain Linux stability result + - name: Record the cross-build result if: always() env: - JAVA_RESULT: ${{ steps.java-tests.outcome }} - JAVA_COMPLETED: ${{ steps.java-tests.outputs.completed }} - JAVA_FAILED: ${{ steps.java-tests.outputs.failed }} - COMPLETE_RESULT: ${{ steps.complete-tests.outcome }} - COMPLETE_COMPLETED: ${{ steps.complete-tests.outputs.completed }} - COMPLETE_FAILED: ${{ steps.complete-tests.outputs.failed }} + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} BUILD_RESULT: ${{ steps.release-builds.outcome }} - BUILD_COMPLETED: ${{ steps.release-builds.outputs.completed }} - BUILD_FAILED: ${{ steps.release-builds.outputs.failed }} + COMPLETED: ${{ steps.release-builds.outputs.completed }} + PLANNED: ${{ steps.release-builds.outputs.planned }} run: | set -euo pipefail - if [[ "${JAVA_RESULT}" == "success" && "${COMPLETE_RESULT}" == "success" && "${BUILD_RESULT}" == "success" ]]; then - result="Passed" - icon="✅" - else - result="Failed" - icon="❌" + completed="${COMPLETED:-0}" + planned="${PLANNED:-12}" + exit_code=0 + summary="All ${planned} release binaries built: full and lite for Linux, macOS, and Windows on amd64 and arm64." + if [[ "${BUILD_RESULT}" != "success" ]]; then + exit_code=1 + summary="${completed} of ${planned} release binaries built. The failed targets are listed below." fi - show_result() { - case "${1}" in - success) printf "Passed" ;; - failure) printf "Failed" ;; - cancelled) printf "Cancelled" ;; - skipped) printf "Skipped" ;; - *) printf "Not run" ;; - esac - } - show_completed() { - case "${1}" in - success|failure) printf "%s" "${2:-0}" ;; - *) printf "—" ;; - esac - } - show_failure() { - case "${1}" in - success) printf "—" ;; - failure) - if [[ -z "${2:-}" || "${2}" == "0" ]]; then - printf "See step log" - else - printf "%s" "${2}" - fi - ;; - cancelled) printf "Cancelled" ;; - skipped) printf "Skipped" ;; - *) printf "Not run" ;; - esac - } - { - echo "## ${icon} Linux stability and release builds: ${result}" - echo - echo "This job repeats the areas most likely to reveal intermittent failures, then checks that every release binary can be built." - echo - echo "| Check | Result | Planned | Completed | Failed at |" - echo "|---|---|---:|---:|---|" - echo "| Java detector tests | $(show_result "${JAVA_RESULT}") | 10 runs | $(show_completed "${JAVA_RESULT}" "${JAVA_COMPLETED:-}") | $(show_failure "${JAVA_RESULT}" "${JAVA_FAILED:-}") |" - echo "| Complete test suite | $(show_result "${COMPLETE_RESULT}") | 5 runs | $(show_completed "${COMPLETE_RESULT}" "${COMPLETE_COMPLETED:-}") | $(show_failure "${COMPLETE_RESULT}" "${COMPLETE_FAILED:-}") |" - echo "| Release binaries | $(show_result "${BUILD_RESULT}") | 12 builds | $(show_completed "${BUILD_RESULT}" "${BUILD_COMPLETED:-}") | $(show_failure "${BUILD_RESULT}" "${BUILD_FAILED:-}") |" - echo - echo "The release check builds full and lite binaries for Linux, macOS, and Windows on amd64 and arm64." - echo - echo "### Need more detail?" - echo - echo "Open the failed step for the test name or release target. Reproduce the test checks with:" - echo - echo '```sh' - echo "go test ./internal/detectors/gradle ./internal/detectors/maven ./internal/detectors/sbt -count=1" - echo "go test ./... -count=1" - echo '```' - echo - echo "For a failed release build, use the operating system, architecture, and variant shown in the table with the build command from **Cross-build release targets**." - } >> "${GITHUB_STEP_SUMMARY}" + go run ./internal/assurance/cmd emit \ + --id cross-build --exit-code "${exit_code}" --summary "${summary}" \ + --metric builds_planned="${planned}" --metric builds_completed="${completed}" \ + --details-jsonl "${RUNNER_TEMP}/build-details.jsonl" \ + --out assurance-results --step-summary - summary: - name: Explain overall result - if: always() - needs: - - portable - - linux-stability - runs-on: ubuntu-latest - steps: - - name: Summarize workflow - env: - PORTABLE_RESULT: ${{ needs.portable.result }} - STABILITY_RESULT: ${{ needs.linux-stability.result }} - run: | - set -euo pipefail - if [[ "${PORTABLE_RESULT}" == "success" && "${STABILITY_RESULT}" == "success" ]]; then - result="Passed" - icon="✅" - else - result="Failed" - icon="❌" - fi - if [[ "${PORTABLE_RESULT}" == "success" ]]; then - portable_label="Passed" - else - portable_label="Failed" - fi - if [[ "${STABILITY_RESULT}" == "success" ]]; then - stability_label="Passed" - else - stability_label="Failed" - fi - { - echo "# ${icon} Portable stability assurance: ${result}" - echo - echo "This workflow checked that the test suite is repeatable across supported development platforms and that release binaries build for every target." - echo - echo "| Area | Result | What ran |" - echo "|---|---|---|" - echo "| Linux, macOS, and Windows | ${portable_label} | Complete suite twice on each platform |" - echo "| Linux stability and release builds | ${stability_label} | Java tests 10 times, complete suite 5 times, and 12 release builds |" - echo - echo "Each job has its own summary with completed counts and the exact point of failure." - echo - echo "## Need more detail?" - echo - echo "Show only failed logs:" - echo - echo '```sh' - echo "gh run view ${GITHUB_RUN_ID} --log-failed" - echo '```' - echo - echo "After fixing a problem, rerun only the failed jobs:" - echo - echo '```sh' - echo "gh run rerun ${GITHUB_RUN_ID} --failed" - echo '```' - } >> "${GITHUB_STEP_SUMMARY}" + - name: Upload check results + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-linux-stability + path: assurance-results + if-no-files-found: error + retention-days: 7 + + - name: Fail the job when any check failed + if: >- + steps.java-tests.outcome == 'failure' || + steps.release-builds.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 35936efe..a8435aed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,10 @@ concurrency: jobs: preflight: + # Fail within seconds on the two things that strand a release: an expired + # publishing token, and a tag whose commit never passed the release + # prerequisites stage. + # # Fail within seconds if WINGET_GITHUB_TOKEN is expired or revoked, # instead of 14 minutes into GoReleaser — after it has already created # the draft release and opened the Homebrew/Scoop PRs — which strands @@ -25,6 +29,50 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: + # Stage 1 of release assurance runs on the source tree before a tag + # exists (see assurance-prerequisites.yml). Auto Version runs it and then + # commits the version bump, so the tagged commit is usually the child of + # the commit that was checked: accept either. + - name: Require a passing release prerequisites run + if: vars.RELEASE_ASSURANCE_ENFORCE != 'false' + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + + tag_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${RELEASE_TAG}" --jq '.sha')" + parent_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${RELEASE_TAG}" --jq '.parents[0].sha // ""')" + + # A workflow called with `uses:` produces no run of its own — the + # jobs live in the caller's run (Auto Version). So the search is over + # every run for the commit, looking for the verdict job by name. + find_verdict_run() { + local sha="$1" + [ -n "${sha}" ] || return 1 + local run_id + for run_id in $(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${sha}&per_page=50" \ + --jq '.workflow_runs[].id'); do + if gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?per_page=100" \ + --jq '.jobs[] | select(.conclusion == "success") | .name' \ + | grep -q 'Prerequisites verdict$'; then + echo "${run_id}" + return 0 + fi + done + return 1 + } + + for candidate in "${tag_sha}" "${parent_sha}"; do + if run_id="$(find_verdict_run "${candidate}")"; then + echo "Release prerequisites passed for ${candidate} in run ${run_id}." + exit 0 + fi + done + + echo "::error::No successful \"Release prerequisites\" run exists for ${RELEASE_TAG} (${tag_sha}) or its parent. Run it first with: gh workflow run assurance-prerequisites.yml -f ref=${tag_sha}. Set the repository variable RELEASE_ASSURANCE_ENFORCE=false to release without it." + exit 1 + - name: Check winget token validity and expiry env: WINGET_GITHUB_TOKEN: ${{ secrets.WINGET_GITHUB_TOKEN }} @@ -168,9 +216,264 @@ jobs: base64-subjects: ${{ needs.release.outputs.hashes }} upload-assets: false - publish: + # Stage 2 of release assurance: the release exists as a draft, so its files + # can be checked before anyone can download them. Published releases are + # immutable, so this is the last point where a bad artifact can be stopped. + verify-draft: needs: [release, provenance] if: startsWith(github.ref, 'refs/tags/') + name: Verify draft release (${{ matrix.platform.os }}) + runs-on: ${{ matrix.platform.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + platform: + - os: ubuntu-latest + scope: full + - os: macos-latest + scope: native + - os: windows-latest + scope: native + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Mint release token + # Draft release assets are not publicly downloadable, so this job needs + # the same app token the publish job uses to read them by release id. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: release-token + with: + client-id: ${{ vars.RELEASE_BOT_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} + owner: bomly-dev + repositories: bomly-cli + # Draft releases are only listed to identities with push access, so + # read is not enough here even though this job only reads. + permission-contents: write + + - name: Download draft release assets + shell: bash + env: + GH_TOKEN: ${{ steps.release-token.outputs.token }} + RELEASE_TAG: ${{ github.ref_name }} + SCOPE: ${{ matrix.platform.scope }} + run: | + set -euo pipefail + version="${RELEASE_TAG#v}" + mkdir -p "${RUNNER_TEMP}/assets" + + release_id="$(gh api "repos/bomly-dev/bomly-cli/releases?per_page=100" | \ + jq -r --arg tag "$RELEASE_TAG" '.[] | select(.tag_name == $tag and .draft == true) | .id' | head -n1)" + if [ -z "${release_id}" ]; then + echo "::error::No draft release found for tag ${RELEASE_TAG}" + exit 1 + fi + + case "${RUNNER_OS}" in + Linux) native_os=linux ;; + macOS) native_os=darwin ;; + Windows) native_os=windows ;; + *) echo "::error::Unsupported runner ${RUNNER_OS}"; exit 1 ;; + esac + case "${RUNNER_ARCH}" in + X64) native_arch=amd64 ;; + ARM64) native_arch=arm64 ;; + *) echo "::error::Unsupported runner architecture ${RUNNER_ARCH}"; exit 1 ;; + esac + extension="tar.gz" + if [ "${native_os}" = "windows" ]; then + extension="zip" + fi + + gh api "repos/bomly-dev/bomly-cli/releases/${release_id}/assets?per_page=100" \ + --jq '.[] | [.id, .name] | @tsv' > "${RUNNER_TEMP}/assets.tsv" + + while IFS=$'\t' read -r asset_id asset_name; do + case "${SCOPE}" in + full) ;; + native) + case "${asset_name}" in + SHA256SUMS|"bomly_${version}_${native_os}_${native_arch}.${extension}"|"bomly-lite_${version}_${native_os}_${native_arch}.${extension}") ;; + *) continue ;; + esac + ;; + esac + echo "downloading ${asset_name}" + curl -fsSL -H "Authorization: Bearer ${GH_TOKEN}" -H "Accept: application/octet-stream" \ + -o "${RUNNER_TEMP}/assets/${asset_name}" \ + "https://api.github.com/repos/bomly-dev/bomly-cli/releases/assets/${asset_id}" + done < "${RUNNER_TEMP}/assets.tsv" + + - name: Verify release files and binaries + id: verify + continue-on-error: true + shell: bash + env: + BOMLY_ASSURANCE_TAG: ${{ github.ref_name }} + run: | + go run ./internal/assurance/cmd verify-release \ + --dir "${RUNNER_TEMP}/assets" --version "${GITHUB_REF_NAME#v}" \ + --scope '${{ matrix.platform.scope }}' \ + --out assurance-results --step-summary + + - name: Install cosign + if: matrix.platform.scope == 'full' + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: Verify the checksum signature + id: signature + if: matrix.platform.scope == 'full' + continue-on-error: true + shell: bash + run: | + set -euo pipefail + cosign verify-blob "${RUNNER_TEMP}/assets/SHA256SUMS" \ + --bundle "${RUNNER_TEMP}/assets/SHA256SUMS.sigstore.json" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + --certificate-identity-regexp '^https://github.com/bomly-dev/bomly-cli/\.github/workflows/release\.yml@refs/tags/' + + - name: Record the signature result + if: matrix.platform.scope == 'full' && always() + shell: bash + env: + BOMLY_ASSURANCE_TAG: ${{ github.ref_name }} + SIGNATURE_OUTCOME: ${{ steps.signature.outcome }} + run: | + set -euo pipefail + exit_code=0 + summary="The checksum list carries a valid Sigstore signature issued to this repository's release workflow." + if [ "${SIGNATURE_OUTCOME}" != "success" ]; then + exit_code=1 + summary="cosign could not verify the signature over SHA256SUMS. Nothing about this release's files can be trusted until that is explained." + fi + go run ./internal/assurance/cmd emit \ + --id release-signature --exit-code "${exit_code}" --summary "${summary}" \ + --out assurance-results --step-summary + + - name: Install slsa-verifier + if: matrix.platform.scope == 'full' + uses: slsa-framework/slsa-verifier/actions/installer@ea584f4502babc6f60d9bc799dbbb13c1caa9ee6 # v2.7.1 + + - name: Verify build provenance + id: provenance + if: matrix.platform.scope == 'full' + continue-on-error: true + shell: bash + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#v}" + for archive in "bomly_${version}_linux_amd64.tar.gz" "bomly-lite_${version}_linux_amd64.tar.gz"; do + slsa-verifier verify-artifact "${RUNNER_TEMP}/assets/${archive}" \ + --provenance-path "${RUNNER_TEMP}/assets/multiple.intoto.jsonl" \ + --source-uri github.com/bomly-dev/bomly-cli \ + --source-tag "${GITHUB_REF_NAME}" + done + + - name: Record the provenance result + if: matrix.platform.scope == 'full' && always() + shell: bash + env: + BOMLY_ASSURANCE_TAG: ${{ github.ref_name }} + PROVENANCE_OUTCOME: ${{ steps.provenance.outcome }} + run: | + set -euo pipefail + exit_code=0 + summary="SLSA build provenance verifies the linux/amd64 archives against this repository and tag." + if [ "${PROVENANCE_OUTCOME}" != "success" ]; then + exit_code=1 + summary="slsa-verifier could not verify the build provenance attached to this release." + fi + go run ./internal/assurance/cmd emit \ + --id release-provenance --exit-code "${exit_code}" --summary "${summary}" \ + --out assurance-results --step-summary + + - name: Upload check results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-release-${{ matrix.platform.os }} + path: assurance-results + if-no-files-found: error + retention-days: 30 + + # The gate that decides whether the draft may be published. It reads the + # check results rather than job statuses, so an advisory check can fail + # without stopping a release and a missing result can never look like a pass. + gate: + needs: verify-draft + if: always() && startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + verdict: ${{ steps.verdict.outputs.verdict }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Collect check results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: assurance-release-* + merge-multiple: true + path: assurance-results + + - name: Judge the pre-release stage + id: verdict + continue-on-error: true + env: + BOMLY_ASSURANCE_TAG: ${{ github.ref_name }} + run: | + set -o pipefail + if go run ./internal/assurance/cmd verdict \ + --results assurance-results --stage pre-release \ + --tag "${GITHUB_REF_NAME}" --step-summary; then + echo "verdict=pass" >> "${GITHUB_OUTPUT}" + else + echo "verdict=fail" >> "${GITHUB_OUTPUT}" + fi + + - name: Upload the stage report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-stage-pre-release + path: assurance-results + # warn, not error: when nothing was collected the gate step's message + # is the useful one, and an upload failure would hide it. + if-no-files-found: warn + retention-days: 30 + + - name: Fail when the stage did not pass + if: steps.verdict.outputs.verdict != 'pass' && vars.RELEASE_ASSURANCE_ENFORCE != 'false' + run: | + echo "::error::The pre-release stage did not pass. The draft release was left unpublished; fix the cause, delete the tag and the draft, then tag again." + exit 1 + + publish: + needs: [release, provenance, gate] + if: | + startsWith(github.ref, 'refs/tags/') && + (needs.gate.outputs.verdict == 'pass' || vars.RELEASE_ASSURANCE_ENFORCE == 'false') runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/sbom-interoperability.yml b/.github/workflows/sbom-interoperability.yml index 7d3af05e..5910bd96 100644 --- a/.github/workflows/sbom-interoperability.yml +++ b/.github/workflows/sbom-interoperability.yml @@ -1,21 +1,39 @@ +# Generates canonical SBOMs and validates them with the official SPDX and +# CycloneDX tools, pinned by checksum. Runs weekly on its own, and as part of +# the post-release assessment against the binary the release actually shipped. +# +# The download, generation, and validation logic lives in +# internal/assurance/sbominterop, which records every command, exit code, and +# checksum in a bomly.sbom-assurance-run/v1 manifest. This workflow only +# arranges the inputs and turns that manifest into a check result. name: SBOM interoperability assurance on: schedule: - cron: "41 5 * * 1" workflow_dispatch: + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check out. + type: string + required: false + default: "" + release_tag: + description: Validate the binary published for this release tag instead of a freshly built one. + type: string + required: false + default: "" + assurance: + description: Upload a release assurance check result. + type: boolean + required: false + default: false permissions: contents: read env: - SPDX_TOOLS_VERSION: "2.0.7" - SPDX_TOOLS_URL: https://github.com/spdx/tools-java/releases/download/v2.0.7/tools-java-2.0.7.zip - SPDX_TOOLS_SHA256: 2dc63c3399c5178058b1be8a3de6f13b9f24981cd86c4292ef98f4a7e90de36d - SPDX_TOOLS_JAR: tools-java-2.0.7-jar-with-dependencies.jar - CDX_CLI_VERSION: "0.32.0" - CDX_CLI_URL: https://github.com/CycloneDX/cyclonedx-cli/releases/download/v0.32.0/cyclonedx-linux-x64 - CDX_CLI_SHA256: 454879e6a4a405c8a13bff49b8982adcb0596f3019b26b0811c66e4d7f0783e1 FIXTURE_SBOM: test/smoke/testdata/sboms/go.spdx.json OUTPUT_DIR: sbom-assurance-artifacts @@ -23,9 +41,13 @@ jobs: validate: name: Validate generated SBOMs runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -41,58 +63,62 @@ jobs: java-version: "21" - name: Build the Bomly binary + if: inputs.release_tag == '' run: make build-full - - name: Download checksum-pinned validators - run: | - set -euo pipefail - mkdir -p "${OUTPUT_DIR}/validators" - curl -fsSL --retry 3 -o "${OUTPUT_DIR}/validators/tools-java.zip" "${SPDX_TOOLS_URL}" - echo "${SPDX_TOOLS_SHA256} ${OUTPUT_DIR}/validators/tools-java.zip" | sha256sum -c - - unzip -j -o "${OUTPUT_DIR}/validators/tools-java.zip" "${SPDX_TOOLS_JAR}" -d "${OUTPUT_DIR}/validators" - curl -fsSL --retry 3 -o "${OUTPUT_DIR}/validators/cyclonedx-cli" "${CDX_CLI_URL}" - echo "${CDX_CLI_SHA256} ${OUTPUT_DIR}/validators/cyclonedx-cli" | sha256sum -c - - chmod +x "${OUTPUT_DIR}/validators/cyclonedx-cli" - - - name: Generate SBOMs with the built binary - run: | - set -euo pipefail - ./bin/bomly scan --sbom --path "${FIXTURE_SBOM}" --detectors sbom \ - --format spdx > "${OUTPUT_DIR}/bomly.spdx.json" - ./bin/bomly scan --sbom --path "${FIXTURE_SBOM}" --detectors sbom \ - --format cyclonedx > "${OUTPUT_DIR}/bomly.cdx.json" - sha256sum "${OUTPUT_DIR}/bomly.spdx.json" "${OUTPUT_DIR}/bomly.cdx.json" \ - | tee "${OUTPUT_DIR}/checksums.txt" - - - name: Validate SPDX 2.3 output (spdx/tools-java) + # The post-release assessment validates what users downloaded, not a + # rebuild of it, so the released archive is verified against the published + # checksum list before it is used. + - name: Download the released binary + if: inputs.release_tag != '' + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.release_tag }} run: | set -euo pipefail - java -jar "${OUTPUT_DIR}/validators/${SPDX_TOOLS_JAR}" Verify \ - "${OUTPUT_DIR}/bomly.spdx.json" 2>&1 | tee "${OUTPUT_DIR}/spdx-validation.log" + version="${RELEASE_TAG#v}" + archive="bomly_${version}_linux_amd64.tar.gz" + mkdir -p "${RUNNER_TEMP}/release" bin + gh release download "${RELEASE_TAG}" --repo bomly-dev/bomly-cli \ + --pattern "${archive}" --pattern SHA256SUMS --dir "${RUNNER_TEMP}/release" + (cd "${RUNNER_TEMP}/release" && sha256sum --check --ignore-missing SHA256SUMS) + tar -xzf "${RUNNER_TEMP}/release/${archive}" -C "${RUNNER_TEMP}/release" bomly + install -m 0755 "${RUNNER_TEMP}/release/bomly" bin/bomly + ./bin/bomly version - - name: Validate CycloneDX 1.7 output (cyclonedx-cli) + - name: Generate and validate SBOMs + id: sbom + continue-on-error: true run: | - set -euo pipefail - "${OUTPUT_DIR}/validators/cyclonedx-cli" validate \ - --input-file "${OUTPUT_DIR}/bomly.cdx.json" \ - --input-format json --input-version v1_7 --fail-on-errors \ - 2>&1 | tee "${OUTPUT_DIR}/cyclonedx-validation.log" + go run ./internal/assurance/sbominterop \ + -bomly ./bin/bomly -input "${FIXTURE_SBOM}" -output "${OUTPUT_DIR}" - - name: Summarize the result + - name: Record the interoperability result if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.release_tag }} run: | set -euo pipefail - { - printf '# SBOM interoperability assurance\n\n' - printf 'The built binary generated SPDX 2.3 and CycloneDX 1.7 SBOMs from a pinned fixture, and the official validators checked them.\n\n' - printf -- '- **Revision:** `%s`\n' "${GITHUB_SHA}" - printf -- '- **Validators:** spdx/tools-java `%s`, cyclonedx-cli `%s` (both checksum-pinned)\n\n' "${SPDX_TOOLS_VERSION}" "${CDX_CLI_VERSION}" - if [[ -f "${OUTPUT_DIR}/checksums.txt" ]]; then - printf '## Generated file checksums\n\n```\n' - cat "${OUTPUT_DIR}/checksums.txt" - printf '```\n' - fi - } >> "${GITHUB_STEP_SUMMARY}" + manifest="${OUTPUT_DIR}/run-manifest.json" + if [ -f "${manifest}" ]; then + go run ./internal/assurance/cmd convert sbom-assurance \ + --id sbom-interoperability --input "${manifest}" \ + --out assurance-results --step-summary + else + go run ./internal/assurance/cmd emit \ + --id sbom-interoperability --status fail \ + --summary "The SBOM interoperability run stopped before it could write a manifest. Open the generate step for the cause." \ + --out assurance-results --step-summary + fi + + - name: Upload check result + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-sbom-interoperability + path: assurance-results + if-no-files-found: error + retention-days: 30 - name: Upload assurance evidence if: always() @@ -102,8 +128,10 @@ jobs: path: | sbom-assurance-artifacts/bomly.spdx.json sbom-assurance-artifacts/bomly.cdx.json - sbom-assurance-artifacts/checksums.txt - sbom-assurance-artifacts/spdx-validation.log - sbom-assurance-artifacts/cyclonedx-validation.log + sbom-assurance-artifacts/run-manifest.json if-no-files-found: warn retention-days: 14 + + - name: Fail the job when validation failed + if: steps.sbom.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 4eacfc56..b00fed7e 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -7,6 +7,26 @@ on: schedule: - cron: "0 8 * * *" workflow_dispatch: + # Called by "Release prerequisites" (assurance-prerequisites.yml). The + # `assurance` input turns on per-slice check results, which that stage + # collects to decide whether the code is fit to be tagged. + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check out. + type: string + required: false + default: "" + assurance: + description: Emit release assurance check results for each slice. + type: boolean + required: false + default: false + assurance_tag: + description: Release tag recorded in the check results. + type: string + required: false + default: "" permissions: contents: read @@ -26,6 +46,9 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -122,6 +145,9 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -254,7 +280,37 @@ jobs: env: BOMLY_SMOKE_BINARY: ${{ runner.temp }}/bomly-bin/bomly BOMLY_SMOKE_LITE_BINARY: ${{ runner.temp }}/bomly-bin/bomly-lite - run: go test -tags smoke ./test/smoke/ -v -count=1 -timeout 15m -run '${{ matrix.slice.run }}' + # -json instead of -v so the run can be summarised into an assurance + # check result; `assurance gotest --echo` below replays the same + # output, so the logs read exactly as they did before. + run: | + set -o pipefail + go test -tags smoke ./test/smoke/ -json -count=1 -timeout 15m \ + -run '${{ matrix.slice.run }}' 2>&1 | tee "${RUNNER_TEMP}/smoke.jsonl" > /dev/null + + - name: Summarise this slice + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + SMOKE_OUTCOME: ${{ steps.smoke_tests.outcome }} + run: | + exit_code=0 + if [ "${SMOKE_OUTCOME}" != "success" ]; then + exit_code=1 + fi + go run ./internal/assurance/cmd gotest \ + --id smoke --instance '${{ matrix.slice.name }}' \ + --input "${RUNNER_TEMP}/smoke.jsonl" --exit-code "${exit_code}" \ + --echo --out assurance-results + + - name: Upload check result + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-smoke-${{ matrix.slice.name }} + path: assurance-results + if-no-files-found: error + retention-days: 7 - name: Fail job if smoke tests failed if: steps.smoke_tests.outcome == 'failure' diff --git a/.github/workflows/update-smoke-goldens.yml b/.github/workflows/update-smoke-goldens.yml index a5da7fa6..30dbde3c 100644 --- a/.github/workflows/update-smoke-goldens.yml +++ b/.github/workflows/update-smoke-goldens.yml @@ -319,6 +319,14 @@ jobs: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + # No cache writes: this job runs with write permissions, and a + # poisoned cache would reach the jobs that trust it. + cache: false + - name: Download regenerated goldens uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -349,6 +357,16 @@ jobs: --jq '[.jobs[] | select(.name | startswith("Regenerate (")) | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out") | .name] | join(", ")')" echo "failed=${failed}" >> "$GITHUB_OUTPUT" + - name: Refresh the catalog checksums + # Public evidence claims record the checksum of the golden file that + # backs them. Regenerating a golden without refreshing the catalog + # would fail the catalog-valid gate and block the next release. + if: steps.changes.outputs.changed == 'true' + shell: bash + run: | + set -euo pipefail + go run ./internal/assurance/cmd catalog-validate --refresh + - name: Create pull request with updated goldens if: steps.changes.outputs.changed == 'true' env: @@ -368,7 +386,8 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git checkout -b "${branch_name}" - git add test/smoke/testdata/golden + + git add test/smoke/testdata/golden docs/assurance/catalog.json git commit -m "test: update smoke golden files" git push --set-upstream origin "${branch_name}" diff --git a/.gitignore b/.gitignore index 55c34326..e749bd48 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ # Ignore generated SBOM files /sboms +# Ignore local assurance report previews +/.assurance + # Ignore local dependency graph benchmark artifacts /.benchmark-runs diff --git a/AGENTS.md b/AGENTS.md index 909030c6..b06c9a04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,8 @@ make smoke ARGS="-update" # regenerate smoke golden files make fuzz FUZZTIME=5s # run every registered fuzz target with a short per-target budget make benchmark # run the hidden local dependency-graph benchmark make benchmark-report # analyze local benchmark artifacts with Copilot CLI -make evidence # verify the public evidence catalog (test/evidence/cases.json) +make assurance-catalog # validate the release assurance catalog (docs/assurance/catalog.json) +make assurance-report # render a fixture assurance report into .assurance/ for local preview make run ARGS="scan" # go run ./cmd/bomly make generate # regenerate config reference, JSON schemas, schema docs, support matrix, and component docs (binary-driven) ``` @@ -54,6 +55,7 @@ See [`dev-docs/ARCHITECTURE.md`](dev-docs/ARCHITECTURE.md) for full detail (the | `internal/baseline` | Portable package-finding baseline codec and audit-integrated policy-status resolver | | `internal/remediation` | Canonical vulnerability fix status, version, detector-hint validation, and occurrence suggestions | | `internal/sbom` | SBOM codec (SPDX 2.3, CycloneDX) | +| `internal/assurance` | Release assurance framework: check-result contract, catalog, report generation, release-asset verification, and the `sbominterop` and `perfrun` check tools | | `internal/licenseexpr` | SPDX license expression parsing and identifier classification (guards the parser's panics) | | `internal/benchmark` | Hidden local dependency-graph benchmark, baseline comparison, scoring, and embedded presets | | `internal/output` | Output rendering plus structured command payloads and schema generation for `scan`, `diff`, `explain`, JSON, and SARIF 2.1.0 | @@ -97,6 +99,7 @@ Runtime preparation is owned by `internal/engine`: build the filtered registry o - `internal/remediation` owns canonical vulnerability remediation decisions. Detectors may supply validated read-only strategy hints, but they do not choose final actions or versions. - `internal/licenseexpr` owns all SPDX license expression parsing. The underlying parser panics on some malformed input, and license strings come from untrusted lockfiles and registry APIs, so no other package under `internal/` may import `github.com/github/go-spdx` directly; `TestNoDirectSPDXExpressionUse` enforces this. - `internal/registry` owns package-manager discovery, support lookups, and built-in registry wiring in `internal/registry/builder.go`. Do not create or reintroduce a separate `registrybuilder` package. +- `internal/assurance` owns the release assurance framework and must not be imported by any package under `cmd/`, `internal/cli`, or `internal/engine`: it is repository tooling, not shipped CLI behavior. It may read repository files and run downloaded release binaries, which no shipped package may do. - `internal/engine` may import `internal/detectors` and `internal/registry`, but detector packages must not point back into `internal/engine`. Runtime planning, prepared subprojects, and detector-chain reuse belong in `internal/engine`. ## Non-Negotiable @@ -105,7 +108,7 @@ Runtime preparation is owned by `internal/engine`: build the filtered registry o - **Plugin protocol is versioned `v1`.** External plugins use the SDK/HashiCorp gRPC `Metadata` and role descriptor contract. - **No secrets or credentials in logs.** Ever. - **Matcher network calls require explicit enrichment.** Built-in matchers may contact OSV (`https://api.osv.dev`), CISA KEV, deps.dev (`https://api.deps.dev`), OpenSSF Scorecard (`https://api.scorecard.dev`), and Grype's database service (`https://grype.anchore.io/databases`, plus the archive URL it returns) only during `--enrich`. Installed external matcher plugins such as ClearlyDefined and endoflife.date may contact their documented services during `--enrich`. `--audit` evaluates existing package data and must not trigger matcher calls. Remote Git targets and build-tool detectors have separate, explicit network behavior. -- **Record architecture decisions as ADRs in [`dev-docs/adr/`](dev-docs/adr/README.md).** Copy [`dev-docs/adr/TEMPLATE.md`](dev-docs/adr/TEMPLATE.md), take the next number, and add a row to the index. `dev-docs/ARCHITECTURE.md` stays the architecture narrative; `docs/ARCHITECTURE.md` is the public, user-facing overview. +- **Record architecture decisions as architecture decision records (ADRs) in [`dev-docs/adr/`](dev-docs/adr/README.md).** Copy [`dev-docs/adr/TEMPLATE.md`](dev-docs/adr/TEMPLATE.md), take the next number, and add a row to the index. `dev-docs/ARCHITECTURE.md` stays the architecture narrative; `docs/ARCHITECTURE.md` is the public, user-facing overview. - **Prefer `internal/`.** Add new packages inside `internal/` unless there is a clear public API need; genuinely public contract surface belongs in the SDK module. - **Standard library + Cobra + existing deps only.** Do not add new dependencies without discussion. @@ -231,8 +234,28 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai - Scan cases come from `test/smoke/testdata/scan_targets.json`; keep it in sync with `internal/benchmark/testdata/scan_targets.json` (the benchmark target list) when cases change. - Pin every scan case's detectors with `--detectors`; normalize volatile fields in `helpers_test.go::normalizeJSON` before goldens. - Register new tests in both slice matrices (`smoke.yml` and exactly one slice in `update-smoke-goldens.yml`); `go test -run` elements are unanchored regexes — use `$` anchors to keep slice ownership exact. +- A new slice also needs an entry in the `smoke` check's `expected_instances` in `docs/assurance/catalog.json` (with its `ecosystems`, which is what puts an ecosystem on the report's coverage list). `TestCatalogSmokeInstancesMatchWorkflowMatrix` fails when the two drift. +- Regenerating goldens invalidates the checksums the catalog's claims pin. `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the catalog with them; do the same when refreshing by hand. - `TestExamplePluginFixtureCompiles` runs in `make test` and must keep compiling against the pinned `bomly-dev/bomly-sdk` release; update the fixture source when the SDK contract changes. +## Release assurance + +Every quality check belongs to one of three release stages and is declared in `docs/assurance/catalog.json` (schema `bomly.assurance-catalog/v1`): + +- **prerequisites** — run on the source tree before a tag exists (smoke, portable stability, cross-builds, fuzz, catalog validation). +- **pre-release** — run inside `release.yml` against the still-draft release (asset completeness, checksums, the Sigstore/cosign signature over the checksum list, SLSA build provenance, released binaries). +- **post-release** — run against the shipped binaries after publication (install scripts, public download, released-binary scans, SBOM interoperability, performance samples). + +Rules: + +- Every check writes one `bomly.assurance-check/v1` document per instance through `go run ./internal/assurance/cmd` (`emit`, `gotest`, `convert`, or `verify-release`) and uploads it as an `assurance-*` artifact. Never hand-write that JSON in a workflow. +- Adding a check means adding a catalog entry **and** emitting its result; a declared check with no result is reported as `missing` and blocks its stage when it is a gate. +- `proves` and `limitations` are mandatory, public, and written in plain language — they are rendered on bomly.dev/assurance. +- Public evidence claims live in the same catalog (`evidence[]`), keep their pinned Git revisions and checksummed artifacts, and name the check that backs them. `make assurance-catalog` re-hashes every file they reference. +- The per-release report (`docs/assurance/reports/.json`) and `docs/assurance/index.json` are written by the post-release assessment and are the only data source for the public page. + +See [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) for the contracts and how to add a check. + ## Feature Checklist When adding a new user-visible feature (new CLI flag, new component class, new pipeline stage, new analyzer, etc.), walk this checklist before requesting review. The surfaces forgotten most often are **MCP**, **plugin command**, and **smoke test**. @@ -298,6 +321,12 @@ If a new analyzer / matcher / detector produces deterministic output for a fixed Any new user-visible feature needs a smoke case under `test/smoke/` — follow the golden/normalizer/slice-matrix rules in the Smoke tests section above. +### Release assurance + +Ask whether the feature makes a claim worth publishing. If it does, add it to `docs/assurance/catalog.json` — a `check` when something new runs for every release, an `evidence` entry when a pinned input and a committed result file prove a specific behavior. Both carry the same fields (`title`, `description`, `proves`, `limitations`), because the public page renders one shape for every claim. + +A declared check with no result is reported as `missing` and blocks its stage, so add the catalog entry and the workflow step that emits it together. + ### Documentation - `make generate` regenerates `docs/CONFIG_REFERENCE.md`, `docs/schemas/*`, `docs/SUPPORT_MATRIX.md`, and the component docs through the built binary. Run it whenever `internal/config/config.go` or `internal/output/*` change, or when the pinned SDK version (catalog / support-matrix data) is bumped. @@ -318,6 +347,7 @@ Draft releases are created automatically after merges to `main` from commit pref | [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | Public, user-facing architecture overview | | [`dev-docs/MODELS.md`](dev-docs/MODELS.md) | Domain model reference: Dependency, Package, Vulnerability, Finding, PackageRegistry | | [`dev-docs/CI.md`](dev-docs/CI.md) | CI setup and workflow (GitHub Actions) | +| [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) | Release assurance framework: stages, check contract, catalog, reports | | [`docs/CONFIG_REFERENCE.md`](docs/CONFIG_REFERENCE.md) | Generated config reference (all keys, env vars, defaults) | | [`docs/SUPPORT_MATRIX.md`](docs/SUPPORT_MATRIX.md) | Ecosystem detector coverage | | `docs/schemas/*.json`, `docs/schemas/*.md` | Generated JSON schemas and human-readable output docs for `scan`, `diff`, and `explain` | diff --git a/CLAUDE.md b/CLAUDE.md index b3b8ca3f..6801fc8a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,8 @@ make smoke ARGS="-update" # regenerate smoke golden files make fuzz FUZZTIME=5s # run every registered fuzz target with a short per-target budget make benchmark # run the hidden local dependency-graph benchmark make benchmark-report # analyze local benchmark artifacts with Copilot CLI -make evidence # verify the public evidence catalog (test/evidence/cases.json) +make assurance-catalog # validate the release assurance catalog (docs/assurance/catalog.json) +make assurance-report # render a fixture assurance report into .assurance/ for local preview make run ARGS="scan" # go run ./cmd/bomly make generate # regenerate config reference, JSON schemas, schema docs, support matrix, and component docs (binary-driven) ``` @@ -54,6 +55,7 @@ See [`dev-docs/ARCHITECTURE.md`](dev-docs/ARCHITECTURE.md) for full detail (the | `internal/baseline` | Portable package-finding baseline codec and audit-integrated policy-status resolver | | `internal/remediation` | Canonical vulnerability fix status, version, detector-hint validation, and occurrence suggestions | | `internal/sbom` | SBOM codec (SPDX 2.3, CycloneDX) | +| `internal/assurance` | Release assurance framework: check-result contract, catalog, report generation, release-asset verification, and the `sbominterop` and `perfrun` check tools | | `internal/licenseexpr` | SPDX license expression parsing and identifier classification (guards the parser's panics) | | `internal/benchmark` | Hidden local dependency-graph benchmark, baseline comparison, scoring, and embedded presets | | `internal/output` | Output rendering plus structured command payloads and schema generation for `scan`, `diff`, `explain`, JSON, and SARIF 2.1.0 | @@ -97,6 +99,7 @@ Runtime preparation is owned by `internal/engine`: build the filtered registry o - `internal/remediation` owns canonical vulnerability remediation decisions. Detectors may supply validated read-only strategy hints, but they do not choose final actions or versions. - `internal/licenseexpr` owns all SPDX license expression parsing. The underlying parser panics on some malformed input, and license strings come from untrusted lockfiles and registry APIs, so no other package under `internal/` may import `github.com/github/go-spdx` directly; `TestNoDirectSPDXExpressionUse` enforces this. - `internal/registry` owns package-manager discovery, support lookups, and built-in registry wiring in `internal/registry/builder.go`. Do not create or reintroduce a separate `registrybuilder` package. +- `internal/assurance` owns the release assurance framework and must not be imported by any package under `cmd/`, `internal/cli`, or `internal/engine`: it is repository tooling, not shipped CLI behavior. It may read repository files and run downloaded release binaries, which no shipped package may do. - `internal/engine` may import `internal/detectors` and `internal/registry`, but detector packages must not point back into `internal/engine`. Runtime planning, prepared subprojects, and detector-chain reuse belong in `internal/engine`. ## Non-Negotiable @@ -105,7 +108,7 @@ Runtime preparation is owned by `internal/engine`: build the filtered registry o - **Plugin protocol is versioned `v1`.** External plugins use the SDK/HashiCorp gRPC `Metadata` and role descriptor contract. - **No secrets or credentials in logs.** Ever. - **Matcher network calls require explicit enrichment.** Built-in matchers may contact OSV (`https://api.osv.dev`), CISA KEV, deps.dev (`https://api.deps.dev`), OpenSSF Scorecard (`https://api.scorecard.dev`), and Grype's database service (`https://grype.anchore.io/databases`, plus the archive URL it returns) only during `--enrich`. Installed external matcher plugins such as ClearlyDefined and endoflife.date may contact their documented services during `--enrich`. `--audit` evaluates existing package data and must not trigger matcher calls. Remote Git targets and build-tool detectors have separate, explicit network behavior. -- **Record architecture decisions as ADRs in [`dev-docs/adr/`](dev-docs/adr/README.md).** Copy [`dev-docs/adr/TEMPLATE.md`](dev-docs/adr/TEMPLATE.md), take the next number, and add a row to the index. `dev-docs/ARCHITECTURE.md` stays the architecture narrative; `docs/ARCHITECTURE.md` is the public, user-facing overview. +- **Record architecture decisions as architecture decision records (ADRs) in [`dev-docs/adr/`](dev-docs/adr/README.md).** Copy [`dev-docs/adr/TEMPLATE.md`](dev-docs/adr/TEMPLATE.md), take the next number, and add a row to the index. `dev-docs/ARCHITECTURE.md` stays the architecture narrative; `docs/ARCHITECTURE.md` is the public, user-facing overview. - **Prefer `internal/`.** Add new packages inside `internal/` unless there is a clear public API need; genuinely public contract surface belongs in the SDK module. - **Standard library + Cobra + existing deps only.** Do not add new dependencies without discussion. @@ -231,8 +234,28 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai - Scan cases come from `test/smoke/testdata/scan_targets.json`; keep it in sync with `internal/benchmark/testdata/scan_targets.json` (the benchmark target list) when cases change. - Pin every scan case's detectors with `--detectors`; normalize volatile fields in `helpers_test.go::normalizeJSON` before goldens. - Register new tests in both slice matrices (`smoke.yml` and exactly one slice in `update-smoke-goldens.yml`); `go test -run` elements are unanchored regexes — use `$` anchors to keep slice ownership exact. +- A new slice also needs an entry in the `smoke` check's `expected_instances` in `docs/assurance/catalog.json` (with its `ecosystems`, which is what puts an ecosystem on the report's coverage list). `TestCatalogSmokeInstancesMatchWorkflowMatrix` fails when the two drift. +- Regenerating goldens invalidates the checksums the catalog's claims pin. `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the catalog with them; do the same when refreshing by hand. - `TestExamplePluginFixtureCompiles` runs in `make test` and must keep compiling against the pinned `bomly-dev/bomly-sdk` release; update the fixture source when the SDK contract changes. +## Release assurance + +Every quality check belongs to one of three release stages and is declared in `docs/assurance/catalog.json` (schema `bomly.assurance-catalog/v1`): + +- **prerequisites** — run on the source tree before a tag exists (smoke, portable stability, cross-builds, fuzz, catalog validation). +- **pre-release** — run inside `release.yml` against the still-draft release (asset completeness, checksums, the Sigstore/cosign signature over the checksum list, SLSA build provenance, released binaries). +- **post-release** — run against the shipped binaries after publication (install scripts, public download, released-binary scans, SBOM interoperability, performance samples). + +Rules: + +- Every check writes one `bomly.assurance-check/v1` document per instance through `go run ./internal/assurance/cmd` (`emit`, `gotest`, `convert`, or `verify-release`) and uploads it as an `assurance-*` artifact. Never hand-write that JSON in a workflow. +- Adding a check means adding a catalog entry **and** emitting its result; a declared check with no result is reported as `missing` and blocks its stage when it is a gate. +- `proves` and `limitations` are mandatory, public, and written in plain language — they are rendered on bomly.dev/assurance. +- Public evidence claims live in the same catalog (`evidence[]`), keep their pinned Git revisions and checksummed artifacts, and name the check that backs them. `make assurance-catalog` re-hashes every file they reference. +- The per-release report (`docs/assurance/reports/.json`) and `docs/assurance/index.json` are written by the post-release assessment and are the only data source for the public page. + +See [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) for the contracts and how to add a check. + ## Feature Checklist When adding a new user-visible feature (new CLI flag, new component class, new pipeline stage, new analyzer, etc.), walk this checklist before requesting review. The surfaces forgotten most often are **MCP**, **plugin command**, and **smoke test**. @@ -298,6 +321,12 @@ If a new analyzer / matcher / detector produces deterministic output for a fixed Any new user-visible feature needs a smoke case under `test/smoke/` — follow the golden/normalizer/slice-matrix rules in the Smoke tests section above. +### Release assurance + +Ask whether the feature makes a claim worth publishing. If it does, add it to `docs/assurance/catalog.json` — a `check` when something new runs for every release, an `evidence` entry when a pinned input and a committed result file prove a specific behavior. Both carry the same fields (`title`, `description`, `proves`, `limitations`), because the public page renders one shape for every claim. + +A declared check with no result is reported as `missing` and blocks its stage, so add the catalog entry and the workflow step that emits it together. + ### Documentation - `make generate` regenerates `docs/CONFIG_REFERENCE.md`, `docs/schemas/*`, `docs/SUPPORT_MATRIX.md`, and the component docs through the built binary. Run it whenever `internal/config/config.go` or `internal/output/*` change, or when the pinned SDK version (catalog / support-matrix data) is bumped. @@ -318,6 +347,7 @@ Draft releases are created automatically after merges to `main` from commit pref | [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | Public, user-facing architecture overview | | [`dev-docs/MODELS.md`](dev-docs/MODELS.md) | Domain model reference: Dependency, Package, Vulnerability, Finding, PackageRegistry | | [`dev-docs/CI.md`](dev-docs/CI.md) | CI setup and workflow (GitHub Actions) | +| [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) | Release assurance framework: stages, check contract, catalog, reports | | [`docs/CONFIG_REFERENCE.md`](docs/CONFIG_REFERENCE.md) | Generated config reference (all keys, env vars, defaults) | | [`docs/SUPPORT_MATRIX.md`](docs/SUPPORT_MATRIX.md) | Ecosystem detector coverage | | `docs/schemas/*.json`, `docs/schemas/*.md` | Generated JSON schemas and human-readable output docs for `scan`, `diff`, and `explain` | diff --git a/Makefile b/Makefile index cb45a1ce..078da8cc 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ EXE_SUFFIX=$(if $(filter Windows_NT,$(OS)),.exe,) GOLANGCI_LINT=$(GOPATH_BIN)/golangci-lint$(EXE_SUFFIX) FUZZTIME?=60s -.PHONY: build build-full build-lite fmt fmt-check lint install-hooks test smoke fuzz run generate evidence benchmark benchmark-report licenses +.PHONY: build build-full build-lite fmt fmt-check lint install-hooks test smoke fuzz run generate assurance-catalog assurance-report benchmark benchmark-samples benchmark-report licenses build: build-full build-lite @@ -41,14 +41,18 @@ smoke: fuzz: FUZZTIME="$(FUZZTIME)" scripts/run-fuzz.sh -evidence: - go run ./internal/tools/publicevidence $(if $(CASE),-case $(CASE),) +assurance-catalog: + go run ./internal/assurance/cmd catalog-validate $(if $(CHECK),--check $(CHECK),) $(if $(EVIDENCE),--evidence $(EVIDENCE),) + +assurance-report: + go run ./internal/assurance/cmd report --results internal/assurance/testdata/fixtures/mixed-failure/results \ + --catalog internal/assurance/testdata/catalog.json --out .assurance --tag v0.0.0-preview --previous none --allow-unknown benchmark: build-full bin/$(BINARY_NAME)$(EXE_SUFFIX) benchmark $(if $(ARGS),$(ARGS),) benchmark-samples: build-lite - go run ./internal/tools/benchmarkrun -output .benchmark-runs/performance -case canonical-sbom-scan -samples 5 -network-state offline -- \ + go run ./internal/assurance/perfrun -output .benchmark-runs/performance -case canonical-sbom-scan -samples 5 -network-state offline -- \ ./bin/$(BINARY_NAME)-lite$(EXE_SUFFIX) scan --sbom --path test/smoke/testdata/sboms/go.spdx.json --detectors sbom --format json benchmark-report: diff --git a/README.md b/README.md index 1845468d..c1e9ebc6 100644 --- a/README.md +++ b/README.md @@ -79,9 +79,9 @@ Bomly reads manifests, lockfiles, package-manager output, container layers, or e | Can I triage reachable findings first? | `bomly scan --enrich --audit --analyze --fail-on high --fail-on reachable` | For more recipes, see [Getting Started](docs/GETTING_STARTED.md) and -[Use Cases](docs/USE_CASES.md). To review the public inputs, commands, expected -results, and limitations behind important behavior claims, see -[Reproducible Evidence](docs/EVIDENCE.md). +[Use Cases](docs/USE_CASES.md). To see the checks every release goes through, +what they prove, and where to read the per-release report, see +[Release assurance](docs/ASSURANCE.md). ## Explore Interactively diff --git a/dev-docs/ARCHITECTURE.md b/dev-docs/ARCHITECTURE.md index 87fecb6d..95a5545b 100644 --- a/dev-docs/ARCHITECTURE.md +++ b/dev-docs/ARCHITECTURE.md @@ -237,6 +237,7 @@ Cache failures are non-fatal. The command should warn and continue rather than f | `internal/engine/scan` | Scan command pipeline API | | `internal/output` | Text, JSON, SARIF rendering, plus structured response payloads and schema generation | | `internal/sbom` | SPDX and CycloneDX codecs | +| `internal/assurance` | Release assurance framework: check-result contract, catalog, report and index generation, release-asset verification, plus the `sbominterop` and `perfrun` check tools and the `cmd` entry point | | `internal/licenseexpr` | SPDX license expression parsing and identifier classification | | `internal/benchmark` | Hidden local dependency-graph benchmark, baseline comparison, scoring, and embedded presets | | `sdk` | Shared domain types | diff --git a/dev-docs/CI.md b/dev-docs/CI.md index 7de425e0..13c79e5f 100644 --- a/dev-docs/CI.md +++ b/dev-docs/CI.md @@ -9,15 +9,22 @@ its own minimal CI in its own repository. | Workflow | Trigger | Purpose | |-------------------------|----------------------------------|----------------------------------------------------------------------| | `CI` | Pull requests, pushes to `main` | Lint, `go test ./...`, full and lite builds, npm wrapper tests, go.mod/go.sum tidy-drift and no-`replace` checks | -| `Smoke` | Pull requests (labeled), nightly | End-to-end smoke slices driving the built binary against pinned public repositories | +| `Release prerequisites` | Called by `Auto Version`, manual dispatch | Stage 1 of release assurance: calls `Smoke`, `Portable stability assurance`, and `Fuzz`, validates the assurance catalog, and judges the stage | +| `Smoke` | Merge queue, nightly, dispatch, `workflow_call` | End-to-end smoke slices driving the built binary against pinned public repositories | | `Update Smoke Goldens` | Manual dispatch on the branch to regenerate from | Regenerates smoke golden files per slice and opens a PR with the drift | -| `Fuzz` | Nightly schedule, manual dispatch | Native Go fuzzing over the `scripts/run-fuzz.sh` target list; uploads minimized failures as artifacts | +| `Portable stability assurance` | Manual dispatch, `workflow_call` | Repeated unit tests on Linux, macOS, and Windows plus cross-builds of every release binary | +| `Fuzz` | Nightly schedule, dispatch, `workflow_call` | Native Go fuzzing over the `scripts/run-fuzz.sh` target list; uploads minimized failures as artifacts | | `CodeQL` | Pull requests, pushes, schedule | Static analysis for Go and JavaScript | -| `SBOM Interoperability` | Schedule, manual dispatch | Binary-driven SBOM export/ingest checks against third-party tools | -| `Auto Version` | Pushes to `main` | Computes the next semver from the squash-commit prefix and creates a draft release tag | -| `Release` | Release tags | GoReleaser build/publish with signed checksums and SLSA provenance | +| `SBOM interoperability assurance` | Weekly schedule, dispatch, `workflow_call` | Validates generated SPDX and CycloneDX documents with checksum-pinned official tools | +| `Auto Version` | Manual dispatch | Runs `Release prerequisites`, then bumps the version, tags, and starts `Release` | +| `Release` | Release tags | GoReleaser build/publish with signed checksums and SLSA provenance, plus stage 2 of release assurance | +| `Release assessment` | `release: published`, dispatch | Stage 3 of release assurance against the published binaries; writes the per-release report | | `Scorecard`, `Dependency Review`, `Bomly Guard` | Various | Supply-chain posture checks and dogfooding | +Workflows that other workflows call (`Smoke`, `Portable stability assurance`, +`Fuzz`, `SBOM interoperability assurance`) keep their own triggers as well, so +each can still be run on its own. + Workflows use `actions/setup-go` with the Go version read from `go.mod`, so `gofmt`, compilation, and test behavior stay aligned between local development and GitHub Actions. Every workflow declares a top-level `permissions:` block @@ -44,12 +51,25 @@ target there. When Go minimizes a failure into `go test`, then commit the reproducer only after confirming it is a useful regression seed. +## Release assurance + +Every quality check belongs to a release stage and reports a +`bomly.assurance-check/v1` result, which the framework merges into a per-release +report. `dev-docs/RELEASE_ASSURANCE.md` describes the contracts and how to add a +check; `docs/assurance/catalog.json` is the list of checks; `docs/ASSURANCE.md` +is the public explanation. + +Workflow steps never hand-write result JSON. They call +`go run ./internal/assurance/cmd` (`emit`, `gotest`, `convert`, or +`verify-release`), which also renders the step summary from the same data. + ## Local parity - `make fmt` rewrites tracked Go files with `gofmt` - `make fmt-check` fails when tracked Go files are not formatted - `make lint` runs the repository-pinned `golangci-lint` - `make install-hooks` points Git at the `.githooks/` pre-commit hook +- `make assurance-catalog` validates the assurance catalog the way CI does ## Releases diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md new file mode 100644 index 00000000..a6bc0f84 --- /dev/null +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -0,0 +1,228 @@ +# Release assurance framework + +This document is for maintainers. The user-facing explanation is +[`docs/ASSURANCE.md`](../docs/ASSURANCE.md), and the published reports live at +[bomly.dev/assurance](https://bomly.dev/assurance). + +The framework has three parts: + +1. a **catalog** that declares every check and every public evidence claim; +2. a **check-result contract** every check writes when it finishes; +3. a **report** built by merging results into the catalog, published per release. + +Everything lives in one place. Go code is under `internal/assurance/`, data and +the public document are under `docs/assurance/`, and the workflows that run the +checks are named in each catalog entry's `source`. + +## Stages + +| Stage | Runs | Workflow | +| --- | --- | --- | +| `prerequisites` | Before a tag exists, on the source tree | `assurance-prerequisites.yml`, which calls `smoke.yml`, `portable-assurance.yml`, and `fuzz.yml` | +| `pre-release` | Inside the release pipeline, against the still-draft release | `release.yml` | +| `post-release` | After publication, against the shipped binaries | `assurance-assessment.yml` and `sbom-interoperability.yml` | + +A stage passes when every `gate` check in it passes and no declared check is +missing. `advisory` checks are always reported and never block. + +## The check-result contract + +Every check writes one JSON file per instance, named `[.].json`, +in the schema `bomly.assurance-check/v1`. Workflows upload those files as +`assurance-*` artifacts; later jobs download them all with `merge-multiple` and +hand the directory to the tool. + +Nothing hand-writes that JSON. Four commands produce it: + +```sh +# an ordinary shell step +go run ./internal/assurance/cmd emit --id cross-build --exit-code "$rc" \ + --summary "12 of 12 release targets built." \ + --metric builds_planned=12 --metric builds_completed=12 \ + --detail "linux/amd64 full=pass" --out assurance-results --step-summary +``` + +```sh +# a Go test slice +go test -tags smoke ./test/smoke/ -json ... | tee smoke.jsonl +go run ./internal/assurance/cmd gotest --id smoke --instance go \ + --input smoke.jsonl --exit-code "$rc" --echo --out assurance-results +``` + +```sh +# a tool that already writes a manifest +go run ./internal/assurance/cmd convert benchmark-run --id perf-samples \ + --input .benchmark-runs/performance/run-manifest.json --out assurance-results +``` + +```sh +# downloaded release assets +go run ./internal/assurance/cmd verify-release --dir assets --version 0.24.0 \ + --scope full --out assurance-results +``` + +`emit`, `gotest`, and `convert` read the stage and level from the catalog, so a +step only passes `--stage` when it emits something the catalog does not declare +(which the report will then flag as unknown). `--details-jsonl` reads +sub-results from a file, which keeps Windows command lines short. + +Every command fills the release tag, commit, job URL, job, and runner from the +GitHub Actions environment. Set `BOMLY_ASSURANCE_TAG` when a stage runs for a +tag that is not the checked-out ref. + +The job URL is resolved by reading the run's job list and matching the job +running on this runner, because every instance of a matrix check shares one +run and a run-level link cannot tell a reader which platform or slice a number +came from. It costs one read of public workflow metadata, falls back to the run +URL whenever that does not work, and can be set directly with +`ASSURANCE_JOB_URL`. + +## Judging a stage and building the report + +```sh +go run ./internal/assurance/cmd verdict --results assurance-results \ + --stage prerequisites --step-summary +``` + +`verdict` exits non-zero when a gate check failed or a declared check reported +nothing. That is the step that stops a release. + +```sh +go run ./internal/assurance/cmd report --results assurance-results \ + --tag v0.24.0 --commit "$SHA" --url "$RELEASE_URL" --published-at "$PUBLISHED" +``` + +`report` writes `docs/assurance/reports/.json` and updates +`docs/assurance/index.json`, prints a markdown summary, and compares the release +against the previous one listed in the index. It exits with code 3 when a result +arrives for a check the catalog does not declare, so a renamed check cannot +silently disappear from the report (`--allow-unknown` downgrades that to a note, +which the assessment uses because the matching declared check already shows as +missing). + +The assessment judges a release against **the catalog as it was at that tag**, +not the one on `main`, so a check added later is not counted as missing from an +older release. The report tooling itself comes from `main`, which is what makes +`gh workflow run assurance-assessment.yml -f tag=` able to re-render an +old release's report after a generator fix. + +The report is committed to `main` under `docs/assurance/`, because published +GitHub releases are immutable and the assessment runs after publication. The +commit uses the release app token, is marked `[skip ci]`, and retries onto +`origin/main` if the branch moved. A `bomly-assurance-report` repository +dispatch then tells the landing page a new report is available. + +## Adding a check + +1. Add the entry to `docs/assurance/catalog.json`: `id`, `title`, `area`, + `stage`, `level`, `description`, `source`, optional `expected_instances`, + `reproduce`, and — required — `proves` and `limitations` in plain language. + Checks are sorted by `id`. The `area` decides which section of the published + page the check appears in, and the order of `areas` in the catalog is the + order those sections are read in, so put a new area where it belongs in the + narrative rather than at the end. +2. Emit a result from the workflow named in `source`, and upload it as an + `assurance-*` artifact. The name only has to start with `assurance-`, since + the stage jobs download them all with `merge-multiple`; group results by job + when that is simpler, as the release workflow does with + `assurance-release-`. +3. If the check backs a public claim, add an `evidence` entry pointing at it + with `check_id` (and `instance`, when one specific leg proves the claim). + Both kinds of entry carry the same fields — title, description, proves, + limitations — because the published page renders one shape for every claim, + whichever way it is asserted. +4. Run `make assurance-catalog`, then `go test ./internal/assurance/`. + Refresh goldens with `go test ./internal/assurance/ -update` when the report + shape changes. + +Until the workflow actually emits the new result, every report will mark the +check `missing` — that is the intended behavior: gaps are loud. + +## Evidence claims + +Evidence claims are the public "we prove X, we do not prove Y" statements that +used to live in `test/evidence/cases.json`. They keep the same rigor: pinned +Git revisions, checksummed fixtures and expected-result files, explicit +reproduction commands, and mandatory limitations. `make assurance-catalog` +verifies every hash a claim names, so a golden file cannot drift away from the +claim it supports, and `--refresh` rewrites them when the goldens are +deliberately regenerated. + +A claim must carry a pinned input **and** a committed artifact. That is the line +between the two layers: if a statement would only restate what its check already +reports — "the workflow ran and passed" — it belongs in the check's `proves`, +not in a second entry that says the same thing again. + +## Re-running things + +```sh +gh workflow run assurance-prerequisites.yml -f ref=main +``` + +```sh +gh workflow run assurance-assessment.yml -f tag=v0.24.0 +``` + +A failed prerequisites run means no tag was created, so the fix is an ordinary +pull request. A failed pre-release gate leaves a draft release and no published +version: fix the cause, delete the tag and the draft, then tag again. A failed +post-release check cannot be undone in the release, so the assessment opens a +tracking issue and the published report records the failure honestly. + +`vars.RELEASE_ASSURANCE_ENFORCE` set to `false` runs everything and publishes +the report without blocking a release. Use it for the first release after a +framework change, then turn it back on. + +## Ecosystem coverage + +`expected_instances[].ecosystems` is what puts a language or package format on +the report's coverage list. Coverage is a single stamp per ecosystem, taking +the worst status of every check that exercised it — not a grid of which check +covered what. The reader's question is "was my ecosystem covered", and +answering it per check invites the false conclusion that a blank cell is a gap +when another check covered it. Adding an ecosystem to any check's instances is +enough to have it appear. + +## Changing a schema + +Four documents carry a schema version: the check result, the catalog, the +report, and the index. The report and index are read by bomly.dev, so their +versions are a contract with another repository. + +- Adding an optional field keeps the version. Every consumer ignores what it + does not know. +- Removing a field, renaming one, or changing what one means raises the version + — and the site has to learn the new shape *first*, or reports stop appearing. + The order is: teach `bomly-landing-page` (`SUPPORTED_REPORT_SCHEMAS` in + `lib/assurance.ts`, `REPORT_SCHEMAS` in `scripts/sync-assurance.mjs`) to + render both versions, ship that, then raise the version here. +- The site keeps rendering older reports it already mirrored, and skips reports + whose version it does not know with a warning, so a mismatch shows up as one + release missing from the page rather than a broken page. + +`TestSchemaVersionsArePinned` fails on any change to these strings, so raising +one is always deliberate. + +## What the automation needs + +- The Bomly Release app needs **Issues: Read and write** on `bomly-cli` for the + per-release tracking issue. Without it the assessment still publishes the + report and simply skips the issue. +- The prerequisites stage is found by looking for a successful job whose name + ends in `Prerequisites verdict` among the runs for a commit. A workflow called + with `uses:` produces no run of its own, so searching for runs of + `assurance-prerequisites.yml` would miss every stage that Auto Version + triggered. Keep that job name stable, or update the lookup in `release.yml` + and `assurance-assessment.yml` with it. +- Regenerating smoke goldens changes files the evidence claims are pinned to. + `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the + catalog alongside them; do the same when refreshing goldens by hand. + That workflow regenerates from the ref it was dispatched on, so the refresh + runs the same commit's tooling against that commit's goldens. + +## Related documents + +- [`docs/ASSURANCE.md`](../docs/ASSURANCE.md) — the public explanation +- [`dev-docs/SECURITY_ASSURANCE.md`](SECURITY_ASSURANCE.md) — trust boundaries and their regression tests +- [`dev-docs/RELEASE_CHECKLIST.md`](RELEASE_CHECKLIST.md) — the release procedure +- [`test/assurance/`](../test/assurance/) — the narrative notes behind individual checks diff --git a/dev-docs/RELEASE_CHECKLIST.md b/dev-docs/RELEASE_CHECKLIST.md index 3586c8f2..8901cc8d 100644 --- a/dev-docs/RELEASE_CHECKLIST.md +++ b/dev-docs/RELEASE_CHECKLIST.md @@ -5,14 +5,16 @@ Use this checklist when publishing a tagged Bomly CLI release. ## Before tagging - Confirm `main` is green for required checks. -- Run the smoke workflow (or confirm its latest scheduled smoke result is healthy). -- Confirm `cmd/bomly/main.go` contains the intended version after the `Auto Version` workflow. - Confirm release publishing credentials are configured in GitHub Actions. +- `Auto Version` runs the `Release prerequisites` stage (smoke, platform stability, cross-builds, fuzz, catalog) on the commit it is about to tag and refuses to tag when it fails, so there is no separate smoke run to start by hand. To pre-flight without tagging: `gh workflow run assurance-prerequisites.yml -f ref=main`. +- If the stage fails, fix the cause on `main` — for stale golden files, run `Update Smoke Goldens` and merge its PR — then start `Auto Version` again. No tag and no release exist yet. ## Release workflow - Run `Auto Version` from `main`, choosing `patch`, `minor`, or `major`. -- Wait for `Release` to finish. +- Wait for `Release` to finish. It builds the draft release, then runs the final pre-release checks against the draft (asset completeness, checksums on three platforms, cosign signature, SLSA provenance, and the released binaries) and only publishes when they pass. +- If the pre-release gate fails, nothing is published. Fix the cause, delete the tag and the draft release, then tag again. Deleting a draft does not trigger the yanking workflow. +- Confirm `cmd/bomly/main.go` contains the intended version. - Review the published GitHub release: - `bomly` archives exist for Linux, macOS, and Windows on `amd64` and `arm64`. - `bomly-lite` archives exist for the same platforms. @@ -21,9 +23,16 @@ Use this checklist when publishing a tagged Bomly CLI release. - Homebrew, Scoop, and WinGet manifest PRs were opened or updated. - The landing-page sync PR updates `/install.sh` and `/install.ps1` from this tag when those scripts changed. +## After publishing + +- `Release assessment` starts automatically once the release is published: it runs the install scripts on all three operating systems, re-downloads the public files, scans real projects with the released binary, validates its SBOM output with the official tools, and records repeated-scan timings. +- Read the report it publishes at [bomly.dev/assurance](https://bomly.dev/assurance), or the JSON it commits to `docs/assurance/reports/.json`. +- If it opens a `Release assurance: ` issue, triage it: the release is already live, so the fix is a follow-up release, not an edit to this one. + ## Verification -Run the checks against the published release tag. Replace `VERSION` in the examples below with the actual release tag, such as `v0.2.0`. +The assessment runs these automatically. Run them by hand when investigating a +report, replacing `VERSION` with the release tag, such as `v0.2.0`. ```bash gh release download VERSION --pattern SHA256SUMS --pattern 'bomly_VERSION_linux_amd64.tar.gz' diff --git a/dev-docs/adr/0036-release-assurance-is-a-catalog-and-a-result-contract.md b/dev-docs/adr/0036-release-assurance-is-a-catalog-and-a-result-contract.md new file mode 100644 index 00000000..bf10c896 --- /dev/null +++ b/dev-docs/adr/0036-release-assurance-is-a-catalog-and-a-result-contract.md @@ -0,0 +1,52 @@ +# ADR-0036: Release assurance is a declarative catalog plus a per-check result contract + +- **Date:** 2026-08-25 +- **Status:** Accepted + +## Context + +Quality checks were independent workflows whose evidence was a job status and, +in two cases, hand-written markdown in a step summary. Public claims lived in a +separate `test/evidence/cases.json` catalog with its own checker. Nothing tied a +claim to whether its check actually ran for a given release, nothing verified +the published release artifacts at all, and no release produced a single +readable answer to "did this one pass?". + +## Decision + +Release assurance is built from three pieces: + +- one catalog (`docs/assurance/catalog.json`) declaring every check, its stage, + whether it gates a release, what it proves, and what it does not — plus the + public claims, each naming the check that backs it; +- one check-result contract (`bomly.assurance-check/v1`) that every check emits + through `internal/assurance/cmd`, so shell steps never hand-write JSON; +- one report (`bomly.assurance-report/v1`), written per release into + `docs/assurance/reports/.json`, which is the only data source for the + public assurance page. + +Checks are grouped by *when they can still change the outcome*: prerequisites +run before a tag exists, pre-release checks run while the release is still a +draft, and the exhaustive assessment runs afterwards against the binaries users +actually download. + +## Consequences + +- A declared check with no result is reported as `missing` and blocks its stage + when it is a gate. Silence is never treated as success. +- A result whose id the catalog does not declare fails report generation, so a + renamed check cannot quietly vanish. +- A stale golden file is fixed by an ordinary pull request, because the checks + most prone to drift run before a tag exists. +- Workflow files are no longer pinned by checksum in the catalog. Pinning the + file proved only that the file had not changed; the per-release check result + proves the workflow ran and what it found. Fixture and expected-result files + are still checksummed, because a claim about a golden file is only as good as + that file — and regenerating goldens therefore has to refresh the catalog + (`catalog-validate --refresh`, wired into `Update Smoke Goldens`). +- Reports are committed to the default branch rather than attached to the + release, because the exhaustive stage runs after publication and GitHub + releases are immutable once published. +- The report and index schema versions are a contract with bomly.dev: adding an + optional field keeps the version, and anything else has to teach the site the + new shape first. diff --git a/dev-docs/adr/README.md b/dev-docs/adr/README.md index b92c6b5d..8ac1f9d2 100644 --- a/dev-docs/adr/README.md +++ b/dev-docs/adr/README.md @@ -56,3 +56,4 @@ status to `Superseded by [ADR-NNNN](NNNN-slug.md)`; do not rewrite history | ADR-0033 | 2026-08-24 | [Package origin is detector-asserted; SBOM export only projects it](0033-package-origin-is-detector-asserted.md) | Accepted | | ADR-0034 | 2026-08-24 | [Decisions are recorded as individual ADRs](0034-decisions-are-recorded-as-individual-adrs.md) | Accepted | | ADR-0035 | 2026-08-25 | [License emission is validated, not assumed](0035-license-emission-is-validated-not-assumed.md) | Accepted | +| ADR-0036 | 2026-08-25 | [Release assurance is a declarative catalog plus a per-check result contract](0036-release-assurance-is-a-catalog-and-a-result-contract.md) | Accepted | diff --git a/docs/ASSURANCE.md b/docs/ASSURANCE.md new file mode 100644 index 00000000..7c5dcbf7 --- /dev/null +++ b/docs/ASSURANCE.md @@ -0,0 +1,119 @@ +# Release assurance + +Every Bomly release goes through the same set of quality checks, in the same +order, and the results are published for that exact version. You can read them +at **[bomly.dev/assurance](https://bomly.dev/assurance)**, pick any release from +the selector, and print the page if you need a copy for a review file. + +This page explains what the checks are, when they run, and what they do and do +not prove. + +## Three stages + +| Stage | When it runs | What it decides | +| --- | --- | --- | +| Release prerequisites | On the source tree, before a version is tagged | Whether the code is fit to be released at all | +| Final pre-release checks | After the release files are built, while the release is still a draft | Whether the files about to be published are complete, unmodified, and signed | +| Post-release assessment | After the release is published | How the binaries people actually download behave | + +Splitting the work this way is deliberate. Checks that can be flaky, or that +need a fix in the source tree, run **before** a version number exists, so a +problem is fixed by a normal pull request instead of by a broken release. +Checks that describe the published files can only run once those files exist. + +## How the report is organised + +The report is grouped by subject, not by release stage: scanning real projects, +dependency graphs, policy decisions, reachability, upgrade guidance, comparing +two scans, other scan targets, SBOM compatibility, running everywhere, handling +broken input, speed and repeatability, what you download, and installing it. +Each section holds the checks that cover it and the evidence claims made about +it. Every check still states which stage it ran in, because that is what +decides how a failure gets fixed. + +The full list, with what every check proves and what it does not, lives in the +machine-readable catalog at +[`docs/assurance/catalog.json`](assurance/catalog.json). The assurance page +renders the same catalog, so the page and the repository can never disagree — +including the section order, which is the order the catalog declares its areas +in. + +Highlights: + +- **End-to-end scans.** Every supported ecosystem is scanned from a pinned + public example project and compared against a checked-in expected result. +- **Platform stability.** The full unit test suite runs twice on Linux, macOS, + and Windows, the Java detector suites run ten times because that is where + intermittent failures have appeared, and every release binary is + cross-compiled. +- **Parser safety.** Fuzz targets feed malformed project, configuration, + baseline, and SBOM files to the parsers that read them. +- **Release integrity.** Checksums, the Sigstore signature, and SLSA build + provenance are verified against the release files before publication. +- **Installation.** The published install scripts are run on all three + operating systems against the new release. +- **SBOM interoperability.** The SBOM documents Bomly writes are validated with + the official SPDX and CycloneDX tools, pinned by checksum. +- **Speed and stability.** The same scan is repeated with a cold and a warm + cache to record timing and confirm the output does not change. + +Some checks stop a release when they fail; others are recorded without +blocking. A claim that was not confirmed is always shown on the page. When the +problem is found after the release is already published, it also raises a +tracking issue for maintainers, so a known problem in a released version is +never quietly dropped; problems found earlier stop the release instead, and are +fixed before anything is published. Every count links to the job that produced +it, so any number on the page is one click from its log. + +## Claims, checks, and evidence + +The report uses one vocabulary, and it is worth being precise about it: + +- A **claim** is one specific statement about what a release does — that it + reads an npm lockfile correctly, or that the files you download are the ones + we signed. +- A **check** is what asserts a claim: something that ran against that exact + release and produced a result you can open. +- The **report as a whole** is the evidence: every claim, whether it was + confirmed, and a link to what confirmed it. + +Some claims are asserted directly by a check. Others are asserted by comparing +a pinned input — a public repository at a recorded commit, or a checked-in +fixture — against a checksummed result file, and name the check that performed +that comparison. Both appear as claims on the page, in the section they belong +to. A claim earns a separate entry only when it adds something the check cannot +say on its own. + +Check any claim yourself from a repository checkout: + +```sh +make assurance-catalog +``` + +```sh +go run ./internal/assurance/cmd catalog-validate --evidence graph-npm +``` + +The first command validates the whole catalog, including the checksums of every +expected-result file it names. The second prints one claim with its reproduction +command. + +## What a verified claim does not mean + +- A passing report describes the checks listed in the catalog. Software can + still fail in ways nobody has written a check for. +- Checks that reach live advisory services record what those services said on + that day. That answer can change afterwards. +- Timing numbers are observations from one continuous-integration machine, not + guarantees or limits. +- "Unreachable" in a reachability result is a confidence signal, not proof that + a package is safe. +- Release integrity checks prove the published files are the ones this + repository's release workflow built. They are not a review of what the code + does. + +## Related documents + +- [Security and trust boundaries](SECURITY.md) +- [Network and privacy](NETWORK.md) +- [Installation](INSTALLATION.md), including how to verify checksums yourself diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md deleted file mode 100644 index 07d28707..00000000 --- a/docs/EVIDENCE.md +++ /dev/null @@ -1,92 +0,0 @@ -# Reproducible evidence - -Bomly publishes the inputs, commands, expected artifacts, and limitations -behind important behavior claims. The goal is to make a claim reviewable -without private infrastructure or a one-off demonstration. - -The machine-readable catalog is -[`test/evidence/cases.json`](../test/evidence/cases.json). Check it from a -repository checkout: - -```sh -make evidence -``` - -To inspect one case: - -```sh -make evidence CASE=graph-npm -``` - -This verifies the recorded checksums and prints the exact command to reproduce -the case. - -## Evidence levels - -| Level | Meaning | -| --- | --- | -| Deterministic | Uses checked-in inputs or local services and compares a stable normalized result | -| Pinned input | Uses a public repository at a recorded commit; local tools or artifact registries can still affect build-tool-backed resolution | -| Snapshot | Records the normalized result of an input that can move, such as a container tag | -| Live service | Uses a pinned project with current advisory data; the result is a dated observation | -| Manual assurance | Runs a separately started GitHub Actions workflow and saves its detailed report as an artifact | - -Each catalog case must state both what it proves and what it does not prove. -Remote Git inputs include a full commit revision. Fixtures, workflows, and -expected results include SHA-256 checksums. - -## Case studies - -- [Dependency graph evidence](evidence/DEPENDENCY_GRAPHS.md) covers npm, pnpm, - Yarn, Bun, Go, Python, and Maven graphs. A separate deterministic - `degraded-detector-fallback` case covers visible fallback orchestration. -- [Policy and vulnerability-guidance evidence](evidence/POLICY_AND_GUIDANCE.md) - covers vulnerability and SPDX policy, baselines, source changes, persisted - findings, reachability tiers, and read-only remediation suggestions. -- [Targets and operational assurance](evidence/TARGETS_AND_OPERATIONS.md) - covers local and Git projects, containers, SBOM ingestion and validation, - repeated unit tests across supported systems, release builds, and repeatable - performance measurements. - -## Dated workflow evidence - -The -[SBOM interoperability run from July 24, 2026](https://github.com/bomly-dev/bomly-cli/actions/runs/30057587653) -completed successfully at commit -`9530b9f3bfcb3fe1d2748fa2bcfadb5e53e3346c`. The workflow generated canonical -SPDX 2.3 and CycloneDX 1.6 documents and checked them with its recorded -checksum-pinned validators. - -The -[portable stability run from July 24, 2026](https://github.com/bomly-dev/bomly-cli/actions/runs/30065452505) -completed successfully at commit -`e6bc5235f85dc909b6bc73f6ba9eb82c22c44ac4`. It repeated Go unit tests on -Linux, macOS, and Windows, repeated the Java-related and full Linux unit-test -suites, and built every release target. It did not run remote smoke tests. - -Workflow summaries explain what ran, the result, and where to inspect failures. -Their downloadable artifacts retain detailed commands, versions, diagnostics, -and hashes where the workflow produces a run manifest. - -## How to read the results - -- A checked golden proves the normalized result for its recorded input. It is - not a promise that every project in that ecosystem has the same fidelity. -- A live enrichment golden can change when advisory services add, correct, or - withdraw records. Bomly does not claim an immutable offline advisory view. -- A package count alone is not graph proof. Review identities, versions, - relationships, scopes, sources, and occurrence paths. -- A fallback warning means useful evidence may still exist, but coverage can - be lower than the preferred detector. -- `unreachable` does not mean safe. The analyzer and tier define what was - checked. -- Remediation output is read-only guidance. It does not apply or validate a - package change. -- SBOM validation proves acceptance by the named validator versions for the - canonical fixtures, not lossless conversion for every producer or consumer. - -## Provenance - -The tracked cases use Bomly-owned fixtures, Bomly-owned example repositories, -or complete public input repositories. The descriptions and test structure -are written for Bomly's own graph, pipeline, and output contracts. diff --git a/docs/README.md b/docs/README.md index 36ec6aac..fc100987 100644 --- a/docs/README.md +++ b/docs/README.md @@ -54,7 +54,7 @@ Specifications, matrices, and design deep dives. The generated pages are regener - [Architecture](ARCHITECTURE.md) — the scan pipeline, domain model, and network behavior - [Network and Privacy](NETWORK.md) — every network trigger, what it transmits, and how to stay offline - [Security and Trust Boundaries](SECURITY.md) — permissions, network behavior, plugins, input limits, and residual risks -- [Reproducible Evidence](EVIDENCE.md) — public inputs, commands, results, and limitations behind important behavior claims +- [Release assurance](ASSURANCE.md) — the checks every release goes through, what they prove, and where to read the per-release report - [Glossary](GLOSSARY.md) — every term, one sentence each ## Experimental diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json new file mode 100644 index 00000000..2abf8850 --- /dev/null +++ b/docs/assurance/catalog.json @@ -0,0 +1,1688 @@ +{ + "schema_version": "bomly.assurance-catalog/v1", + "areas": [ + { + "id": "end-to-end", + "title": "Scanning real projects", + "description": "Running the real command line against pinned example projects and comparing the entire result with a recorded one." + }, + { + "id": "dependency-graph", + "title": "Dependency graphs", + "description": "Working out which packages a project depends on, and how each one got there." + }, + { + "id": "policy", + "title": "Policy decisions", + "description": "Deciding what counts as a problem: licences, vulnerabilities, baselines, and packages that change where they come from." + }, + { + "id": "reachability", + "title": "Reachability", + "description": "Judging whether a vulnerable package is actually used by the code that depends on it." + }, + { + "id": "remediation", + "title": "Upgrade guidance", + "description": "The fixes Bomly recommends, worked out without changing anything in your project." + }, + { + "id": "diff", + "title": "Comparing two scans", + "description": "Telling what genuinely changed between two scans, and what only looks new." + }, + { + "id": "targets", + "title": "Other scan targets", + "description": "Scanning things that are not a local source tree, such as container images." + }, + { + "id": "sbom", + "title": "SBOM compatibility", + "description": "Reading SBOM documents other tools produce, and writing ones they accept." + }, + { + "id": "stability", + "title": "Running everywhere", + "description": "Behaving the same on Linux, macOS, and Windows, and building for every processor a release supports." + }, + { + "id": "parsers", + "title": "Handling broken input", + "description": "Reading damaged or hostile project, configuration, and SBOM files without crashing." + }, + { + "id": "performance", + "title": "Speed and repeatability", + "description": "How long a repeated scan takes, and whether it returns the same answer every time." + }, + { + "id": "release-integrity", + "title": "What you download", + "description": "Whether the published files are complete, unaltered, and signed by this project's release pipeline." + }, + { + "id": "installation", + "title": "Installing it", + "description": "Whether the documented ways of installing this release actually work." + }, + { + "id": "framework", + "title": "This report itself", + "description": "The catalog and contracts the report is built from, and the check that keeps them honest." + } + ], + "checks": [ + { + "id": "catalog-valid", + "title": "Assurance catalog is valid", + "area": "framework", + "stage": "prerequisites", + "level": "gate", + "description": "Validates this catalog and confirms every fixture and golden file it points at still has the recorded checksum.", + "source": { + "workflow": "assurance-prerequisites.yml", + "job": "catalog" + }, + "reproduce": [ + [ + "make", + "assurance-catalog" + ] + ], + "proves": [ + "Every claim in this report points at a real check and at repository files that have not changed since the claim was written." + ], + "limitations": [ + "A valid catalog says the paperwork is consistent; the checks themselves decide whether the software works." + ] + }, + { + "id": "cross-build", + "title": "Release binaries build for every target", + "area": "stability", + "stage": "prerequisites", + "level": "gate", + "description": "Cross-compiles the full and lite binaries for every supported operating system and processor from the same source tree.", + "source": { + "workflow": "portable-assurance.yml", + "job": "linux-stability" + }, + "reproduce": [ + [ + "gh", + "workflow", + "run", + "assurance-prerequisites.yml", + "-f", + "ref=main" + ] + ], + "proves": [ + "The source tree compiles into both binaries for every Linux, macOS, and Windows target a release ships." + ], + "limitations": [ + "A successful build is not a test: it does not run the binary or check what it produces." + ] + }, + { + "id": "fuzz", + "title": "Parsers survive malformed input", + "area": "parsers", + "stage": "prerequisites", + "level": "advisory", + "description": "Runs every registered Go fuzz target against the parsers that read untrusted project, configuration, baseline, SBOM, and plugin data.", + "source": { + "workflow": "fuzz.yml", + "job": "fuzz" + }, + "reproduce": [ + [ + "make", + "fuzz", + "FUZZTIME=60s" + ] + ], + "proves": [ + "Each registered parser handled its seed corpus and newly generated inputs without crashing during the time budget of this run." + ], + "limitations": [ + "Fuzzing explores a limited time budget. A clean run is evidence of robustness, not proof that no malformed input can cause a failure.", + "Parsers that are command-backed or handled entirely by the standard library are excluded and listed in test/assurance/PARSER_FUZZING.md." + ] + }, + { + "id": "install-script", + "title": "Install scripts install the release", + "area": "installation", + "stage": "post-release", + "level": "gate", + "description": "Runs the published install script on each operating system against this release and asks the installed binary for its version.", + "source": { + "workflow": "assurance-assessment.yml", + "job": "install-scripts" + }, + "expected_instances": [ + { + "name": "ubuntu", + "platform": "linux" + }, + { + "name": "macos", + "platform": "darwin" + }, + { + "name": "windows", + "platform": "windows" + } + ], + "reproduce": [ + [ + "sh", + "-c", + "BOMLY_VERSION= BOMLY_INSTALL_DIR=$PWD/bin sh scripts/install.sh" + ] + ], + "proves": [ + "The documented one-line install works on Linux, macOS, and Windows for this exact release, and the installed binary reports this version." + ], + "limitations": [ + "This installs into a temporary directory on a clean runner. It does not cover every shell, proxy, or locked-down corporate environment." + ] + }, + { + "id": "perf-samples", + "title": "Repeated scan speed and stability", + "area": "performance", + "stage": "post-release", + "level": "advisory", + "description": "Runs the same offline scan with the released binary several times with a cold cache and several times with a warm cache, and records timing, memory, and output stability.", + "source": { + "workflow": "assurance-assessment.yml", + "job": "perf-samples" + }, + "reproduce": [ + [ + "make", + "benchmark-samples" + ] + ], + "proves": [ + "Repeated runs of the same scan produced identical normalized output, and the recorded timings show how long that scan took on this machine." + ], + "limitations": [ + "Timing and memory are observations from one CI machine, not limits. Comparing them across releases is only meaningful because the case and the machine class stay the same." + ] + }, + { + "id": "public-download", + "title": "Published files download and match", + "area": "release-integrity", + "stage": "post-release", + "level": "gate", + "description": "Downloads every published file from its public URL, without credentials, verifies the signature on the published checksum list, and checks each file against it.", + "source": { + "workflow": "assurance-assessment.yml", + "job": "public-download" + }, + "reproduce": [ + [ + "gh", + "release", + "download", + "", + "--repo", + "bomly-dev/bomly-cli" + ] + ], + "proves": [ + "Every file a user downloads from the public release page is present, reachable without credentials, and matches a checksum list signed by this repository's release workflow." + ], + "limitations": [ + "This checks availability and integrity at one moment from one network location.", + "It confirms the published files match the signed list; it does not re-check the provenance attestation, which is verified before publication.", + "Whether each archive contains a working binary is checked separately, against the draft release and by the install scripts." + ] + }, + { + "id": "release-assets", + "title": "Every release file is attached", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Compares the files attached to the draft release against the full list a release is required to ship.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "reproduce": [ + [ + "go", + "run", + "./internal/assurance/cmd", + "verify-release", + "--dir", + "assets", + "--version", + "" + ] + ], + "proves": [ + "The release carries an archive for every supported platform, both binaries, every Linux package format, the checksum list, its signature, and the provenance file." + ], + "limitations": [ + "This counts files. Whether each archive contains a working binary is checked separately." + ] + }, + { + "id": "release-binaries", + "title": "Released binaries report this version", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Extracts the archives built for each operating system and runs the binaries inside them.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "expected_instances": [ + { + "name": "linux-amd64", + "platform": "linux" + }, + { + "name": "darwin-arm64", + "platform": "darwin" + }, + { + "name": "windows-amd64", + "platform": "windows" + } + ], + "reproduce": [ + [ + "go", + "run", + "./internal/assurance/cmd", + "verify-release", + "--dir", + "assets", + "--version", + "", + "--scope", + "native" + ] + ], + "proves": [ + "Both the full and lite binaries start on Linux, macOS, and Windows and report the version that is being released." + ], + "limitations": [ + "Only the architecture of the CI runner is executed for each operating system; other architectures are verified by checksum and build only." + ] + }, + { + "id": "release-checksums", + "title": "Release files match their checksums", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Hashes every release file each platform downloads and compares it with the published SHA256SUMS list.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "expected_instances": [ + { + "name": "linux-amd64", + "platform": "linux" + }, + { + "name": "darwin-arm64", + "platform": "darwin" + }, + { + "name": "windows-amd64", + "platform": "windows" + } + ], + "reproduce": [ + [ + "sha256sum", + "--check", + "SHA256SUMS" + ] + ], + "proves": [ + "Every file the release ships hashes to exactly the value recorded in the checksum list, and the list covers every file." + ], + "limitations": [ + "Checksums prove the files were not altered after they were built; the signature and provenance checks prove who built them." + ] + }, + { + "id": "release-provenance", + "title": "Build provenance verifies", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Verifies the SLSA build provenance attached to the release with slsa-verifier.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "reproduce": [ + [ + "slsa-verifier", + "verify-artifact", + "", + "--provenance-path", + "multiple.intoto.jsonl", + "--source-uri", + "github.com/bomly-dev/bomly-cli" + ] + ], + "proves": [ + "The release archives were produced by this repository's tagged release workflow, as recorded in signed build provenance." + ], + "limitations": [ + "Provenance describes how and where the files were built. It does not review what the source code does." + ] + }, + { + "id": "release-signature", + "title": "Checksum list is signed", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Verifies the Sigstore signature over the release checksum list with cosign.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "reproduce": [ + [ + "cosign", + "verify-blob", + "SHA256SUMS", + "--bundle", + "SHA256SUMS.sigstore.json", + "--certificate-oidc-issuer", + "https://token.actions.githubusercontent.com", + "--certificate-identity-regexp", + "^https://github.com/bomly-dev/bomly-cli/" + ] + ], + "proves": [ + "The checksum list carries a valid Sigstore signature issued to this repository's release workflow, so the checksums themselves can be trusted." + ], + "limitations": [ + "The signature covers the checksum list. Individual files are tied to it through their checksums." + ] + }, + { + "id": "released-scan", + "title": "The released binary scans real projects", + "area": "end-to-end", + "stage": "post-release", + "level": "gate", + "description": "Runs scan, diff, and explain with the binary from the published release against pinned example repositories and checks the results against the same golden files the source-tree smoke tests use.", + "source": { + "workflow": "assurance-assessment.yml", + "job": "released-scan" + }, + "expected_instances": [ + { + "name": "go", + "ecosystems": [ + "Go" + ] + }, + { + "name": "node", + "ecosystems": [ + "JavaScript" + ] + }, + { + "name": "sbom", + "ecosystems": [ + "SBOM documents" + ] + } + ], + "reproduce": [ + [ + "sh", + "-c", + "BOMLY_SMOKE_BINARY=$PWD/bin/bomly go test -tags smoke ./test/smoke/ -run 'TestScan$/scan-go$'" + ] + ], + "proves": [ + "The binary users actually download produces the same dependency graphs as the source tree it was built from." + ], + "limitations": [ + "A representative subset of ecosystems runs against the released binary; the full ecosystem matrix runs on the source tree before the release is tagged." + ] + }, + { + "id": "sbom-interoperability", + "title": "Official validators accept our SBOMs", + "area": "sbom", + "stage": "post-release", + "level": "gate", + "description": "Generates SPDX 2.3 and CycloneDX 1.7 documents with the released binary and validates them with checksum-pinned official SPDX and CycloneDX tools.", + "source": { + "workflow": "sbom-interoperability.yml", + "job": "validate" + }, + "reproduce": [ + [ + "go", + "run", + "./internal/assurance/sbominterop", + "-bomly", + "./bin/bomly" + ] + ], + "proves": [ + "The SBOM documents this release writes are accepted by the official SPDX and CycloneDX validators, at the exact validator versions recorded in the result." + ], + "limitations": [ + "Validators check that a document is well formed against its specification. They do not check that every receiving tool interprets it the same way." + ] + }, + { + "id": "smoke", + "title": "End-to-end scans of real projects", + "area": "end-to-end", + "stage": "prerequisites", + "level": "gate", + "description": "Runs scan, diff, and explain against pinned public repositories, one slice per ecosystem, and compares the whole result with checked-in golden files.", + "source": { + "workflow": "smoke.yml", + "job": "smoke" + }, + "expected_instances": [ + { + "name": "go", + "ecosystems": [ + "Go" + ] + }, + { + "name": "go-reachability", + "ecosystems": [ + "Go" + ] + }, + { + "name": "node", + "ecosystems": [ + "JavaScript" + ] + }, + { + "name": "node-detail-policy", + "ecosystems": [ + "JavaScript" + ] + }, + { + "name": "node-reachability", + "ecosystems": [ + "JavaScript" + ] + }, + { + "name": "java", + "ecosystems": [ + "Java" + ] + }, + { + "name": "java-reachability", + "ecosystems": [ + "Java" + ] + }, + { + "name": "python", + "ecosystems": [ + "Python" + ] + }, + { + "name": "python-reachability", + "ecosystems": [ + "Python" + ] + }, + { + "name": "uv", + "ecosystems": [ + "Python" + ] + }, + { + "name": "php", + "ecosystems": [ + "PHP" + ] + }, + { + "name": "ruby", + "ecosystems": [ + "Ruby" + ] + }, + { + "name": "sbom", + "ecosystems": [ + "SBOM documents" + ] + }, + { + "name": "dotnet", + "ecosystems": [ + ".NET" + ] + }, + { + "name": "rust", + "ecosystems": [ + "Rust" + ] + }, + { + "name": "dart", + "ecosystems": [ + "Dart" + ] + }, + { + "name": "swift", + "ecosystems": [ + "Swift" + ] + }, + { + "name": "elixir", + "ecosystems": [ + "Elixir" + ] + }, + { + "name": "scala", + "ecosystems": [ + "Scala" + ] + }, + { + "name": "cpp", + "ecosystems": [ + "C and C++" + ] + }, + { + "name": "container", + "ecosystems": [ + "Container images" + ] + }, + { + "name": "plugin", + "ecosystems": [ + "Plugins" + ] + } + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "proves": [ + "For every supported ecosystem, the command line produces exactly the dependency graph, policy result, and output shape recorded in the golden files." + ], + "limitations": [ + "Golden files pin one revision of one example project per ecosystem. They do not cover every project layout or every version of a package manager.", + "Cases that reach live advisory services can change when upstream data changes." + ] + }, + { + "id": "unit-portable", + "title": "Unit tests pass on every platform", + "area": "stability", + "stage": "prerequisites", + "level": "gate", + "description": "Runs the full Go unit test suite twice on Linux, macOS, and Windows, so a pass is never a single lucky run.", + "source": { + "workflow": "portable-assurance.yml", + "job": "portable" + }, + "expected_instances": [ + { + "name": "ubuntu-latest", + "platform": "linux" + }, + { + "name": "macos-latest", + "platform": "darwin" + }, + { + "name": "windows-latest", + "platform": "windows" + } + ], + "reproduce": [ + [ + "go", + "test", + "./...", + "-count=1" + ] + ], + "proves": [ + "Every unit test passes on all three supported operating systems, twice in a row, so a pass is not a one-off." + ], + "limitations": [ + "Unit tests run in process against fixtures. They do not download repositories or call external services." + ] + }, + { + "id": "unit-repeat-java", + "title": "Java detector tests are stable", + "area": "stability", + "stage": "prerequisites", + "level": "gate", + "description": "Runs the Gradle, Maven, and sbt detector test suites ten times, because build-tool-backed detectors are where intermittent failures have actually appeared.", + "source": { + "workflow": "portable-assurance.yml", + "job": "linux-stability" + }, + "reproduce": [ + [ + "go", + "test", + "./internal/detectors/gradle", + "./internal/detectors/maven", + "./internal/detectors/sbt", + "-count=10" + ] + ], + "proves": [ + "The build-tool-backed Java detector suites passed ten consecutive times." + ], + "limitations": [ + "This exercises the detectors against local fixtures with the toolchain installed on the runner." + ] + } + ], + "evidence": [ + { + "id": "baseline-policy", + "title": "Finding baseline lifecycle", + "area": "policy", + "description": "Records a package finding in a project baseline, then scans again and compares the whole result with a recorded one.", + "evidence_level": "deterministic", + "check_id": "smoke", + "instance": "go", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestFindingBaselineWorkflow$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/finding-baseline-workflow.golden.json", + "sha256": "8fa2b220af563a89d1fbaaab03a75ccc0952ae43251438bc27a2314301315aa0" + } + ], + "proves": [ + "A project baseline can keep a denied package finding visible with suppressed policy status." + ], + "limitations": [ + "The local matcher fixture is deterministic but does not measure advisory freshness." + ] + }, + { + "id": "container-inventory", + "title": "Container package inventory", + "area": "targets", + "description": "Scans a public container image and compares the operating-system packages found with a recorded inventory.", + "evidence_level": "snapshot", + "check_id": "smoke", + "instance": "container", + "inputs": [ + { + "kind": "container", + "location": "Docker Hub", + "ref": "alpine:3.20" + } + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestContainerScan/container-scan-alpine$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/container-scan-alpine.golden.json", + "sha256": "50f7d4426ead3af9ea5a07df7eeb77743e7aa1412260e1a665a7301f3c22ecc5" + } + ], + "proves": [ + "Bomly can inventory the operating-system packages in the checked Alpine image." + ], + "limitations": [ + "The upstream image tag can move; the checked-in golden is a snapshot, not an immutable image claim." + ] + }, + { + "id": "degraded-detector-fallback", + "title": "Visible detector fallback", + "area": "dependency-graph", + "description": "Forces a detector to fall back to a second technique and checks that the result says so instead of hiding it.", + "evidence_level": "deterministic", + "check_id": "unit-portable", + "inputs": [ + { + "kind": "fixture", + "location": "internal/engine/pipeline_fallback_test.go", + "sha256": "7751d6137dd39c8616857f2e0891ed4928bf2b37e72475d567c1e38262755a6d" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/engine", + "-run", + "TestPipeline_RunRecordsFallbackWarning|TestPipeline_Run_TypesFallbackWarningsAsDegradedCoverage", + "-count=1" + ] + ], + "artifacts": [ + { + "path": "internal/engine/pipeline_fallback_test.go", + "sha256": "7751d6137dd39c8616857f2e0891ed4928bf2b37e72475d567c1e38262755a6d" + } + ], + "proves": [ + "A detector fallback preserves its origin and produces a typed degraded-coverage warning." + ], + "limitations": [ + "The synthetic detector case proves orchestration behavior, not the fidelity of every concrete fallback." + ] + }, + { + "id": "graph-bun", + "title": "Bun lockfile graph", + "area": "dependency-graph", + "description": "Scans a pinned Bun example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-bun", + "ref": "v1.0.0", + "revision": "358a2a920fe9c2f7e596c514f36df1c30c1ab185" + } + ], + "required_tools": [ + "git" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-bun$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-bun.golden.json", + "sha256": "c61ccae15f3d36028ec1e741436abbf036236b9e2f018393b863e98d14360fb4" + } + ], + "proves": [ + "The native Bun detector preserves the package inventory and dependency placement represented by the pinned lockfile." + ], + "limitations": [ + "This is one Bun lockfile shape and does not prove behavior for every historical Bun format." + ] + }, + { + "id": "graph-go", + "title": "Go module graph", + "area": "dependency-graph", + "description": "Scans a pinned Go example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "go", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-go$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-go.golden.json", + "sha256": "728f949898446657c4987bc1202a8d00c78b118868b4945141e4db8c4c54cdea" + } + ], + "proves": [ + "The Go detector resolves a build-tool-backed module graph for the pinned repository." + ], + "limitations": [ + "The result depends on a compatible Go toolchain and the evidence available from that toolchain." + ] + }, + { + "id": "graph-maven", + "title": "Maven dependency graph", + "area": "dependency-graph", + "description": "Scans a pinned Maven example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "java", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-java-maven", + "ref": "v1.0.0", + "revision": "93bb3aae614e2f2c6cb65f5ea2315846f5234150" + } + ], + "required_tools": [ + "git", + "java", + "mvn" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-maven$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-maven.golden.json", + "sha256": "a1077544af4bd637311e1767abe2a140e3ba8f0f65a64e662378e5ccca07de9b" + } + ], + "proves": [ + "The Maven detector resolves a build-tool-backed dependency graph for the pinned repository." + ], + "limitations": [ + "Artifact resolution depends on Maven repositories and a compatible Java and Maven installation." + ] + }, + { + "id": "graph-npm", + "title": "npm lockfile graph", + "area": "dependency-graph", + "description": "Scans a pinned npm example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-npm", + "ref": "v1.0.0", + "revision": "559a762aeef68b0e5c818f62dfba67abc369912f" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-npm$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-npm.golden.json", + "sha256": "82f741b8f7291cb1f885cba788a2a7e3505b143a9f2ded646d01334b8abfaa8a" + } + ], + "proves": [ + "The npm detector preserves lockfile package inventory, versions, scopes, and dependency placement." + ], + "limitations": [ + "This case covers the lockfile versions present in one pinned repository." + ] + }, + { + "id": "graph-pnpm", + "title": "pnpm lockfile graph", + "area": "dependency-graph", + "description": "Scans a pinned pnpm example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-pnpm", + "ref": "v1.0.0", + "revision": "f1b0959f916dfb91db70c54a75b15e5b7f3d16af" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-pnpm$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-pnpm.golden.json", + "sha256": "92b867725f0617f273d8f52d1d9174ace6a6712f4d9fb11da988e455e764a221" + } + ], + "proves": [ + "The pnpm detector preserves lockfile package inventory and dependency placement." + ], + "limitations": [ + "This case does not cover every pnpm lockfile generation." + ] + }, + { + "id": "graph-python", + "title": "Python requirements graph", + "area": "dependency-graph", + "description": "Scans a pinned Python example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "python", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-python-pip", + "revision": "fe04c758134b95dab102e1fce10275f7d18c0cf2" + } + ], + "required_tools": [ + "git", + "pip" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-python-pip$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-python-pip.golden.json", + "sha256": "025d14ad1a3c93f3af22a4a83a358ee85b42d782f50798b285048e35e8632bf2" + } + ], + "proves": [ + "The pip detector reads the pinned requirements lock and preserves the resolved Python package graph." + ], + "limitations": [ + "Unpinned requirements and environment inspection have different fidelity and are not proven by this case." + ] + }, + { + "id": "graph-yarn", + "title": "Yarn lockfile graph", + "area": "dependency-graph", + "description": "Scans a pinned Yarn example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-yarn", + "ref": "v1.0.0", + "revision": "c84017b43bf0f6ea74281f4174a0c89b88b8cddf" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-yarn$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-yarn.golden.json", + "sha256": "1e1a5f1579089169b1cf3f55bb399ea83591bfc660dcac82d3355ab83c98c2af" + } + ], + "proves": [ + "The Yarn detector preserves package inventory and dependency placement represented by the pinned lockfile." + ], + "limitations": [ + "This case covers one Yarn lockfile family; fallback behavior is documented separately." + ] + }, + { + "id": "license-policy", + "title": "Complex and invalid SPDX policy", + "area": "policy", + "description": "Evaluates licence policy against a matrix of SPDX expressions, including combined, nested, exception, custom, and invalid ones.", + "evidence_level": "deterministic", + "check_id": "unit-portable", + "inputs": [ + { + "kind": "fixture", + "location": "internal/auditors/license/spdx_policy_matrix_test.go", + "sha256": "04d8e3528b33687a09d317b867f7c24068081c1ac60bb53cee1e0a03c0b143f2" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/auditors/license", + "-run", + "TestLicenseAuditorComplexSPDX|TestLicenseAuditorInvalidSPDXExpressionMatrix", + "-count=1" + ] + ], + "artifacts": [ + { + "path": "internal/auditors/license/spdx_policy_matrix_test.go", + "sha256": "04d8e3528b33687a09d317b867f7c24068081c1ac60bb53cee1e0a03c0b143f2" + } + ], + "proves": [ + "The license auditor evaluates AND, OR, nested, exception, custom-reference, and invalid SPDX expressions under allow and deny policy." + ], + "limitations": [ + "The tests prove policy evaluation after license data is present; they do not prove the accuracy of every upstream license source." + ] + }, + { + "id": "persisted-risk", + "title": "Risk that persists across a version change", + "area": "diff", + "description": "Compares two scans across a package version change and checks how a finding that survives the change is classified.", + "evidence_level": "deterministic", + "check_id": "unit-portable", + "inputs": [ + { + "kind": "fixture", + "location": "internal/engine/diff/diff_test.go", + "sha256": "abe87f7b95e55e5cf70bd0684c36027757308f7f405a9db370a894238f16f195" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/engine/diff", + "-run", + "TestRun_SameVulnerabilityAcrossVersionBumpPersists|TestRun_SameLicenseIssueAcrossVersionBumpPersists", + "-count=1" + ] + ], + "artifacts": [ + { + "path": "internal/engine/diff/diff_test.go", + "sha256": "abe87f7b95e55e5cf70bd0684c36027757308f7f405a9db370a894238f16f195" + } + ], + "proves": [ + "A vulnerability or license finding that remains across a package version change is classified as persisted instead of one resolved and one introduced finding." + ], + "limitations": [ + "Persistence uses the canonical package-finding identity; a genuinely different advisory or rule remains a distinct finding." + ] + }, + { + "id": "reachability-go", + "title": "Go vulnerability reachability", + "area": "reachability", + "description": "Analyses a pinned Go project and checks the reachability evidence attached to each matched vulnerability.", + "evidence_level": "live-service", + "check_id": "smoke", + "instance": "go-reachability", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-go-reachability$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-go-reachability.golden.json", + "sha256": "2d1db6318d75c491b8866ff9643ae80635702591046e4be89c362ca1b828a4cd" + } + ], + "proves": [ + "The Go analyzer attaches named analyzer, tier, status, and reason evidence to matched vulnerabilities." + ], + "limitations": [ + "Advisories come from live services and may change; an unreachable result is not proof that a package is safe." + ] + }, + { + "id": "reachability-java", + "title": "Java package reachability", + "area": "reachability", + "description": "Analyses a pinned Java project and checks the reachability evidence attached to each matched vulnerability.", + "evidence_level": "live-service", + "check_id": "smoke", + "instance": "java-reachability", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-java-maven", + "ref": "v1.0.0", + "revision": "93bb3aae614e2f2c6cb65f5ea2315846f5234150" + } + ], + "required_tools": [ + "git", + "java", + "mvn" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-java-maven-reachability$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-java-maven-reachability.golden.json", + "sha256": "cae9114ef8280849afad9135b92b7a6a7e1cb3c0c25f9e049a60448b912f7874" + } + ], + "proves": [ + "The Java analyzer separates package-reachable and package-unreachable vulnerability evidence." + ], + "limitations": [ + "This is package-tier evidence, not symbol-level proof, and live advisory results may change." + ] + }, + { + "id": "reachability-node", + "title": "JavaScript package reachability", + "area": "reachability", + "description": "Analyses a pinned JavaScript project and checks the reachability evidence attached to each matched vulnerability.", + "evidence_level": "live-service", + "check_id": "smoke", + "instance": "node-reachability", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-npm", + "ref": "v1.0.0", + "revision": "559a762aeef68b0e5c818f62dfba67abc369912f" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-npm-reachability$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-npm-reachability.golden.json", + "sha256": "3fb1b3b9a11e65fa4b76daa56c90098ce442831d79bb1fdaeba675f201fb78ec" + } + ], + "proves": [ + "The JavaScript analyzer separates package-reachable and package-unreachable vulnerability evidence." + ], + "limitations": [ + "This is package-tier evidence, dynamic loading can reduce confidence, and live advisory results may change." + ] + }, + { + "id": "reachability-python", + "title": "Python package reachability", + "area": "reachability", + "description": "Analyses a pinned Python project and checks the reachability evidence attached to each matched vulnerability.", + "evidence_level": "live-service", + "check_id": "smoke", + "instance": "python-reachability", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-python-pip", + "revision": "fe04c758134b95dab102e1fce10275f7d18c0cf2" + } + ], + "required_tools": [ + "git", + "pip" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-python-pip-reachability$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-python-pip-reachability.golden.json", + "sha256": "61d9b6d3e91e33755c19c7d2b35c0b2fb1c6a98692cdf57dc4630323ad315ca4" + } + ], + "proves": [ + "The Python analyzer separates package-reachable and package-unreachable vulnerability evidence." + ], + "limitations": [ + "This is package-tier evidence, reflective imports can reduce confidence, and live advisory results may change." + ] + }, + { + "id": "remediation-read-only", + "title": "Canonical read-only remediation guidance", + "area": "remediation", + "description": "Derives upgrade guidance for a set of findings and checks the fix status, versions, and per-occurrence advice it produces.", + "evidence_level": "deterministic", + "check_id": "unit-portable", + "inputs": [ + { + "kind": "fixture", + "location": "internal/remediation/derive_test.go", + "sha256": "8afb2ca42404363f3eaaf29eaa20e0c57ddb92aed6209b5720af2c8ba8681e97" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/remediation", + "-run", + "TestDerivePackageRemediation|TestDeriveBuildsCanonicalOccurrenceSuggestions", + "-count=1" + ] + ], + "artifacts": [ + { + "path": "internal/remediation/derive_test.go", + "sha256": "8afb2ca42404363f3eaaf29eaa20e0c57ddb92aed6209b5720af2c8ba8681e97" + } + ], + "proves": [ + "One central component derives fix status, recommended versions, occurrence actions, and detector advice without applying changes." + ], + "limitations": [ + "Suggestions are read-only evidence and are not guaranteed to work in every checkout." + ] + }, + { + "id": "sbom-cyclonedx-ingest", + "title": "CycloneDX 1.6 ingestion", + "area": "sbom", + "description": "Reads a checked-in CycloneDX document and compares the dependency graph it produces with a recorded one.", + "evidence_level": "deterministic", + "check_id": "smoke", + "instance": "sbom", + "inputs": [ + { + "kind": "fixture", + "location": "test/smoke/testdata/sboms/go.cdx.json", + "sha256": "78454a7207f98ba0c8d35df8a257f9185c313d22691cc90a06f11809021bed93" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-sbom-cyclonedx$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json", + "sha256": "54f5e33a47d884e1705d22102dd4745b4474295d0cc56da827bc3f172792dabb" + } + ], + "proves": [ + "Bomly ingests the checked CycloneDX 1.6 dependency graph." + ], + "limitations": [ + "The fixture covers one document shape and does not imply lossless conversion from every producer." + ] + }, + { + "id": "sbom-spdx-ingest", + "title": "SPDX 2.3 ingestion", + "area": "sbom", + "description": "Reads a checked-in SPDX document and compares the dependency graph it produces with a recorded one.", + "evidence_level": "deterministic", + "check_id": "smoke", + "instance": "sbom", + "inputs": [ + { + "kind": "fixture", + "location": "test/smoke/testdata/sboms/go.spdx.json", + "sha256": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-sbom-spdx$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/testdata/golden/scan-sbom-spdx.golden.json", + "sha256": "d49e4fc3f9c61fdb4c533116d9a5988e1841c7ccacc3675b93fb069d5ec14a00" + } + ], + "proves": [ + "Bomly ingests the checked SPDX 2.3 dependency graph." + ], + "limitations": [ + "The fixture covers one document shape and does not imply lossless conversion from every producer." + ] + }, + { + "id": "source-change-policy", + "title": "Registry-to-Git source-change review", + "area": "policy", + "description": "Compares two scans of the same project where one package moved from a registry to a Git source, and checks how that is flagged.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node-detail-policy", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-npm", + "ref": "assurance/dependency-source-registry-v1", + "revision": "f6127099dad7f8b6fbaa7ed1ceb8e7b0d1e8c864" + }, + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-npm", + "ref": "assurance/dependency-source-git-v1", + "revision": "96ffda21548628ce36201741b56164ca5c4405b6" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestDependencyDetailRiskPolicy$" + ] + ], + "artifacts": [ + { + "path": "test/smoke/audit_test.go", + "sha256": "96fcd3b6a411a9651edce806fdc0e60f2e71208c33f4d91a4640177ab2b828fe" + } + ], + "proves": [ + "Diff marks a same-version registry-to-Git move for review and the package auditor can fail it when source-change policy is enabled." + ], + "limitations": [ + "A source change is a review signal, not proof of malicious intent, and unknown source evidence cannot be classified." + ] + }, + { + "id": "vulnerability-policy", + "title": "Vulnerability policy constraints", + "area": "policy", + "description": "Evaluates vulnerability policy against a matrix of severities, reachability tiers, known exploitation, and advisory allowlists.", + "evidence_level": "deterministic", + "check_id": "unit-portable", + "inputs": [ + { + "kind": "fixture", + "location": "internal/auditors/vulnerability/policy_matrix_test.go", + "sha256": "8cd341ca5eea7df5ce50a1bdbe076637154e8f4ba760b64ba4ea366435a9a460" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/auditors/vulnerability", + "-run", + "TestAuditorSeverityReachabilityExploitabilityAndAllowlistMatrix", + "-count=1" + ] + ], + "artifacts": [ + { + "path": "internal/auditors/vulnerability/policy_matrix_test.go", + "sha256": "8cd341ca5eea7df5ce50a1bdbe076637154e8f4ba760b64ba4ea366435a9a460" + } + ], + "proves": [ + "The vulnerability auditor composes severity, reachability, known exploitation, and advisory allowlists with the documented repeated-constraint behavior." + ], + "limitations": [ + "Policy can only evaluate vulnerability, reachability, and exploitation evidence that is present in the registry." + ] + } + ] +} diff --git a/docs/evidence/DEPENDENCY_GRAPHS.md b/docs/evidence/DEPENDENCY_GRAPHS.md deleted file mode 100644 index 7c28eab5..00000000 --- a/docs/evidence/DEPENDENCY_GRAPHS.md +++ /dev/null @@ -1,86 +0,0 @@ -# Dependency graph evidence - -Bomly's findings depend on the graph produced before enrichment or audit. -These public cases check the graph first: package identity, version, -relationship, scope, source, and the manifest that owns each occurrence. - -The cases use complete public example repositories at recorded Git commits. -Their normalized JSON results are checked into `test/smoke/testdata/golden`. -The evidence catalog records a SHA-256 checksum for every result, so an -expected change must also update the public evidence record. - -## What is covered - -| Case | Input evidence | Detector path | What the case checks | -| --- | --- | --- | --- | -| `graph-npm` | `package-lock.json` | npm lockfile detector | Package inventory, versions, scopes, and placement | -| `graph-pnpm` | `pnpm-lock.yaml` | pnpm lockfile detector | Package inventory and placement | -| `graph-yarn` | `yarn.lock` | Yarn lockfile detector | Package inventory and placement | -| `graph-bun` | `bun.lock` | Native Bun detector | Package inventory and placement | -| `graph-go` | `go.mod` and Go tool output | Go detector | Build-tool-backed module graph | -| `graph-python` | Pinned requirements lock | pip detector | Resolved Python package graph | -| `graph-maven` | `pom.xml` and Maven output | Maven detector | Build-tool-backed JVM graph | - -This is placement evidence, not just a list of package names. The Node cases -retain duplicate versions as separate package identities and keep occurrences -attached to the dependency paths represented by each lockfile. - -## Reproduce a case - -From a Bomly CLI checkout: - -```sh -make evidence CASE=graph-npm -``` - -The command verifies the catalog and prints the exact focused smoke command: - -```sh -go test -tags smoke ./test/smoke/ -v -count=1 -timeout 15m \ - -run 'TestScan$/scan-npm$' -``` - -The smoke test builds Bomly, clones the recorded public input, selects the -named detector, normalizes only documented volatile fields, and compares the -result with the checked-in JSON. - -## Example review workflow - -A maintainer updates a lockfile and wants to confirm that Bomly still sees the -same dependency structure: - -1. Run the matching graph case before changing the detector. -2. Make the detector change. -3. Run the case again. -4. Review every golden change as a package, relationship, scope, source, or - manifest-placement change. -5. Update the catalog checksum only when the new graph is intentional. - -This makes a changed count a starting point for review, not the conclusion. -The actual package and occurrence records explain what changed. - -## Degraded and unknown evidence - -Bomly can fall back when a preferred detector cannot run. A fallback is not -silently treated as equivalent: the result records which detector failed, -which detector ran, and whether coverage was reduced. Unresolved package -placement remains `unknown`; a synthetic link to a manifest helps navigation -but does not become a direct or transitive parent claim. - -The graph cases above pin their detector selector. If that detector is not -ready or fails, the smoke test fails instead of accepting a differently shaped -fallback result. - -## Limits - -- A case proves the checked repository and lockfile shape, not every format - version ever produced by that package manager. -- Build-tool-backed cases depend on compatible local tools and may need - registry access to resolve artifacts. -- A fallback inventory can contain useful packages while carrying less - relationship or source evidence than a native graph. -- Project roots, workspaces, local paths, Git dependencies, and arbitrary URL - dependencies stay in the graph but are not queried as published registry - packages. -- Graph evidence does not prove advisory accuracy. Matching is a later step - with separate evidence and limits. diff --git a/docs/evidence/POLICY_AND_GUIDANCE.md b/docs/evidence/POLICY_AND_GUIDANCE.md deleted file mode 100644 index c9859666..00000000 --- a/docs/evidence/POLICY_AND_GUIDANCE.md +++ /dev/null @@ -1,117 +0,0 @@ -# Policy and vulnerability-guidance evidence - -Bomly separates observed evidence from policy decisions: - -- enrichment attaches vulnerability and license information to packages; -- analysis can add reachability evidence; -- audit evaluates the available evidence against the selected policy; -- read-only remediation guidance is derived from vulnerability and dependency - evidence during enrichment. - -That separation matters when reviewing results. A vulnerability can be -present but warning-only, suppressed by a baseline, or failing policy. -Reachability is another property of the same vulnerability; it does not -replace severity or policy status. - -## Policy cases - -| Case | What it checks | Expected outcome | -| --- | --- | --- | -| `vulnerability-policy` | Severity, reachability, known exploitation, advisory allowlists, and repeated constraints | Only findings that satisfy the selected constraints fail | -| `license-policy` | SPDX `AND`, `OR`, nesting, exceptions, custom references, and invalid expressions | Valid expressions keep their Boolean meaning; invalid expressions remain visible and fail | -| `baseline-policy` | Create and automatically use a project finding baseline | The finding remains present with suppressed policy status | -| `source-change-policy` | Same-version move from a registry package to a Git source | Diff requests review; `--fail-on source-change` can fail the audited change | -| `persisted-risk` | Same finding remains across a package version change | The finding is reported as persisted, not hidden as one resolved and one new finding | - -Run any case through the public catalog: - -```sh -make evidence CASE=license-policy -make evidence CASE=source-change-policy -make evidence CASE=persisted-risk -``` - -Each command prints the focused test command, the recorded inputs, and the -case limitation. - -## Example policy workflow - -A team wants high-severity vulnerabilities and risky source changes to block -a pull request while keeping accepted findings visible: - -```sh -bomly diff \ - --base main \ - --head HEAD \ - --enrich \ - --audit \ - --fail-on high \ - --fail-on source-change -``` - -If a package changed from a known registry source to Git or an arbitrary URL, -Bomly explains that registry-based vulnerability matching may no longer cover -it. That is a review signal, not a claim that the change is malicious. Source -formats that cannot prove origin remain unknown. - -A finding baseline does not delete matching findings. It changes their policy -status to `suppressed`, so JSON, SARIF, Guard, MCP, and human output can still -show the accepted risk. - -## Reachability cases - -The catalog includes `reachability-go`, `reachability-node`, -`reachability-python`, and `reachability-java`. - -- The Go case records the analyzer and the tier actually returned for each - vulnerability. -- The JavaScript, Python, and Java cases prove package-tier reachable and - package-tier unreachable branches. -- Every case keeps the analyzer name, tier, status, and plain-language reason - with the vulnerability. - -These cases use current advisory services, so the checked golden is a dated -observation. Re-running later may legitimately discover added, changed, or -withdrawn advisories. - -`unreachable` never means `safe`. Static analysis can miss dynamic loading, -reflection, generated code, tests, build tags, or code paths outside the -analyzer's model. Use reachability to prioritize review, not to erase the -underlying vulnerability. - -## Read-only remediation guidance - -The `remediation-read-only` case exercises the central remediation component. -For each vulnerable package, it derives: - -- whether the available evidence describes a complete fix, partial fix, no - upstream fix, or an unknown fix state; -- a recommended version only when all vulnerability evidence supports one; -- an occurrence-specific suggested action; -- package-manager advice supplied by the detector when available. - -The guidance is derived during enrichment and applies only to vulnerabilities. -License and package-policy findings stay audit results because their meaning -depends on project policy. - -Suggestions do not edit manifests, run package managers, or claim that a -change is guaranteed to work. Unknown parents and non-registry occurrences -receive manual-review guidance instead of a guessed parent upgrade. - -Reproduce the deterministic derivation matrix: - -```sh -make evidence CASE=remediation-read-only -``` - -## Limits - -- `--audit` still requires `--enrich` by default. This avoids an accidental - audit with fewer findings because matching was forgotten. -- Policy cannot make missing evidence complete. Unknown source, license, - reachability, or fix data remains unknown. -- Live advisory services can change independently. Bomly does not ship an - immutable offline advisory snapshot. -- Source changes, reachability, and remediation are different signals. None - of them alone proves that a dependency is safe or malicious. -- Bomly provides read-only remediation suggestions; it does not apply fixes. diff --git a/docs/evidence/TARGETS_AND_OPERATIONS.md b/docs/evidence/TARGETS_AND_OPERATIONS.md deleted file mode 100644 index d71a3d8f..00000000 --- a/docs/evidence/TARGETS_AND_OPERATIONS.md +++ /dev/null @@ -1,126 +0,0 @@ -# Targets and operational assurance - -Bomly accepts local projects, Git repositories, container images, and existing -SBOMs. The public evidence uses the same engine paths as the CLI and states -where the input itself is not immutable. - -## Target cases - -| Target | Public case | What is checked | -| --- | --- | --- | -| Local project | `baseline-policy` | A public Git fixture is materialized locally, then scanned and audited through `--path` | -| Git repository | `graph-npm`, `graph-go`, and the other graph cases | The CLI clones a recorded commit and runs the selected detector | -| Container image | `container-inventory` | Built-in inventory reads packages from the checked Alpine image | -| SPDX SBOM | `sbom-spdx-ingest` | The checked SPDX 2.3 graph is ingested through the SBOM detector | -| CycloneDX SBOM | `sbom-cyclonedx-ingest` | The checked CycloneDX 1.6 graph is ingested through the SBOM detector | - -Inspect or reproduce one: - -```sh -make evidence CASE=container-inventory -make evidence CASE=sbom-spdx-ingest -``` - -The container smoke case currently uses `alpine:3.20`. The tag can move, so -the checked-in golden is explicitly a snapshot rather than an immutable image -claim. The Git cases separately record the full commit behind their readable -tag or ref. - -## Example SBOM workflow - -A release engineer receives a supplier SBOM and wants to apply the same policy -used for source scans: - -```sh -bomly scan \ - --sbom \ - --path supplier.spdx.json \ - --enrich \ - --audit \ - --fail-on high -``` - -The deterministic ingestion cases check Bomly's dependency graph. The -`SBOM interoperability assurance` workflow -(`.github/workflows/sbom-interoperability.yml`) in this repository adds an -external check. It runs weekly and can be started on demand: - -```sh -gh workflow run sbom-interoperability.yml -``` - -It builds the released CLI surface, uses the binary itself to generate SPDX -2.3 and CycloneDX 1.7 files from a pinned fixture SBOM, verifies the -downloaded validator checksums, runs the official validators -(`spdx/tools-java` and `cyclonedx-cli`), and uploads the generated files, -their checksums, and the validation logs as a public run artifact. - -Validator versions and download checksums stay in the workflow file, so -changing either requires an intentional, reviewable evidence update. See -[`test/assurance/SBOM_INTEROPERABILITY.md`](../../test/assurance/SBOM_INTEROPERABILITY.md) -for the workflow summary and failure-investigation steps. - -## Supported-system checks - -Unit-test and build assurance — the unit suite and release-target builds — -runs in this repository's public CI on every change. - -The publicly verifiable evidence for this repository is: - -- the `SBOM interoperability assurance` workflow described above, which - drives the built binary and official validators in public CI; -- the smoke suite, which runs the built binary end to end against pinned - public repositories and checked-in golden outputs; -- signed releases with SLSA build provenance, which tie each published - binary to the public revision it was built from. - -## Repeatable performance measurements - -Run: - -```sh -make benchmark-samples -``` - -The `performance-stability` case uses the checked SPDX fixture and the -lightweight Bomly binary. It records five isolated cold-cache scans and five -shared-cache warm scans under `.benchmark-runs/performance`. See -[`test/assurance/BENCHMARK_RUNS.md`](../../test/assurance/BENCHMARK_RUNS.md) -for the full description. - -The resulting `bomly.benchmark-run/v1` report includes: - -- repository and executable revisions and hashes; -- host and Go runtime details; -- exact command, working directory, cache mode, and network state; -- exit status, output size and hashes, timing, and peak memory for every run; -- median, variation, and an approximate 95% confidence interval. - -The stable gates are successful exit status, normalized output consistency, -and an optional explicit output-size cap. Wall time and memory remain -machine-specific measurements for review rather than universal limits. - -## Example release-confidence workflow - -Before a broad release: - -1. Run `make test` and the relevant pinned smoke slices. -2. Run `make benchmark-samples` and compare the report with the previous run - from a comparable host. -3. If SBOM output changed, start the interoperability workflow and inspect its - generated artifact hashes and validator results. -4. Record the repository commit with every retained workflow or benchmark - report. - -## Limits - -- Remote services and package registries can be temporarily unavailable. -- Build-tool-backed graph resolution can vary with tool versions; the catalog - states required tools and checked source revisions. -- Local repository scans can contain any number of individually bounded - files. Per-file parser limits do not create a total project-size limit. -- Portable Git options bound time and checkout validation but cannot reliably - cap transfer bytes or `.git` object storage before checkout completes. -- Workflow artifacts have retention periods. The workflow file, pinned tools, - reproduction command, and result link remain public after an artifact - expires, but the raw artifact may need to be regenerated. diff --git a/docs/manifest.json b/docs/manifest.json index 26a019d9..1f956b57 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -30,7 +30,7 @@ { "slug": "getting-started", "title": "Getting started", - "description": "First scan, enrich, audit, diff — all in five minutes.", + "description": "First scan, enrich, audit, diff \u2014 all in five minutes.", "group": "start" }, { @@ -60,7 +60,7 @@ { "slug": "output-formats", "title": "Output formats", - "description": "Text, JSON, SARIF, and SBOM artifacts — when to use each and how to combine them.", + "description": "Text, JSON, SARIF, and SBOM artifacts \u2014 when to use each and how to combine them.", "group": "start" }, { @@ -196,11 +196,10 @@ "group": "reference" }, { - "slug": "evidence", - "title": "Reproducible evidence", - "description": "Public inputs, commands, results, and limitations behind important behavior claims.", - "group": "reference", - "hasChildren": true + "slug": "assurance", + "title": "Release assurance", + "description": "The checks every release goes through, what they prove, and where to read the results.", + "group": "reference" }, { "slug": "glossary", diff --git a/internal/assurance/aggregate.go b/internal/assurance/aggregate.go new file mode 100644 index 00000000..5b743daf --- /dev/null +++ b/internal/assurance/aggregate.go @@ -0,0 +1,430 @@ +package assurance + +import ( + "sort" + "strconv" + "strings" + "time" +) + +// BuildOptions configures how a report is assembled from check results. +type BuildOptions struct { + // Release identifies the release being reported on. + Release Release + // Stages limits the report to these stages; empty means every stage. + Stages []Stage + // StageRuns maps a stage to the workflow run that produced its results. + StageRuns map[Stage]string + // Previous is the previous release's report, used for trends. + Previous *Report + // IncludeEvidence attaches the public evidence claims to the report. + IncludeEvidence bool + // GeneratedBy names the tool version that produced the report. + GeneratedBy string + // Now is the report timestamp; the zero value means time.Now(). + Now time.Time +} + +// BuildReport merges check results into the catalog and produces the report +// that the public assurance page renders. Declared checks without a result are +// reported as missing; reported checks the catalog does not declare are listed +// separately so nothing is silently dropped. +func BuildReport(catalog Catalog, results []CheckResult, opts BuildOptions) Report { + now := opts.Now + if now.IsZero() { + now = time.Now() + } + stages := opts.Stages + if len(stages) == 0 { + stages = Stages() + } + selected := make(map[Stage]struct{}, len(stages)) + for _, stage := range stages { + selected[stage] = struct{}{} + } + + byCheck := make(map[string][]CheckResult, len(results)) + for _, result := range results { + byCheck[result.ID] = append(byCheck[result.ID], result) + } + + report := Report{ + SchemaVersion: ReportSchema, + GeneratedAt: now.UTC().Format(time.RFC3339), + Release: opts.Release, + Environment: Environment{GeneratedBy: opts.GeneratedBy, Runners: collectRunners(results)}, + } + + statusByCheck := make(map[string]Status, len(catalog.Checks)) + for _, check := range catalog.Checks { + if _, wanted := selected[check.Stage]; !wanted { + continue + } + reported := buildCheck(check, byCheck[check.ID]) + statusByCheck[check.ID] = reported.Status + report.Checks = append(report.Checks, reported) + } + sort.SliceStable(report.Checks, func(i, j int) bool { return report.Checks[i].ID < report.Checks[j].ID }) + + declared := make(map[string]struct{}, len(catalog.Checks)) + for _, check := range catalog.Checks { + declared[check.ID] = struct{}{} + } + for _, result := range results { + if _, known := declared[result.ID]; known { + continue + } + report.Unknown = append(report.Unknown, UnknownResult{ + ID: result.ID, Instance: result.Instance, Stage: result.Stage, Status: result.Status, + }) + } + sort.Slice(report.Unknown, func(i, j int) bool { + if report.Unknown[i].ID != report.Unknown[j].ID { + return report.Unknown[i].ID < report.Unknown[j].ID + } + return report.Unknown[i].Instance < report.Unknown[j].Instance + }) + + for _, stage := range Stages() { + if _, wanted := selected[stage]; !wanted { + continue + } + stageReport := StageReport{ID: stage, Title: stage.Title(), RunURL: opts.StageRuns[stage]} + var stageChecks []ReportCheck + for _, check := range report.Checks { + if check.Stage != stage { + continue + } + stageChecks = append(stageChecks, check) + stageReport.CheckIDs = append(stageReport.CheckIDs, check.ID) + } + stageReport.Verdict = summarize(stageChecks) + stageReport.Status = stageReport.Verdict.Overall + report.Stages = append(report.Stages, stageReport) + } + report.Verdict = summarize(report.Checks) + + if opts.IncludeEvidence { + report.Evidence = buildEvidence(catalog, report, selected) + } + + // Areas are listed in catalog order, which is the order the published page + // reads in. An area earns a section when it has checks or evidence. + for _, area := range catalog.Areas { + reported := AreaReport{ID: area.ID, Title: area.Title, Description: area.Description} + var areaChecks []ReportCheck + for _, check := range report.Checks { + if check.Area == area.ID { + areaChecks = append(areaChecks, check) + reported.CheckIDs = append(reported.CheckIDs, check.ID) + } + } + for _, evidence := range report.Evidence { + if evidence.Area == area.ID { + reported.EvidenceIDs = append(reported.EvidenceIDs, evidence.ID) + } + } + if len(reported.CheckIDs) == 0 && len(reported.EvidenceIDs) == 0 { + continue + } + reported.Verdict = summarize(areaChecks) + reported.Status = reported.Verdict.Overall + // An area proven only by evidence takes the status of the checks + // backing those claims, so it can never look better than they are. + if len(areaChecks) == 0 { + reported.Status = evidenceAreaStatus(report, reported.EvidenceIDs) + } + report.Areas = append(report.Areas, reported) + } + report.Coverage = buildCoverage(catalog, report, selected) + if opts.Previous != nil { + report.Trends = buildTrends(*opts.Previous, report) + } + return report +} + +func buildCheck(check Check, results []CheckResult) ReportCheck { + reported := ReportCheck{ + ID: check.ID, Title: check.Title, Area: check.Area, Stage: check.Stage, + Level: check.Level, Description: check.Description, Source: check.Source, + Reproduce: check.Reproduce, Proves: check.Proves, Limitations: check.Limitations, + } + sort.Slice(results, func(i, j int) bool { return results[i].Key() < results[j].Key() }) + + seen := make(map[string]struct{}, len(results)) + status := Status("") + for _, result := range results { + name := result.Instance + if name == "" { + name = "default" + } + seen[name] = struct{}{} + reported.Instances = append(reported.Instances, InstanceReport{ + Name: name, Status: result.Status, Summary: result.Summary, + DurationMS: result.DurationMS, RunURL: result.RunURL, Runner: result.Runner, + Metrics: result.Metrics, Details: result.Details, + Artifacts: result.Artifacts, Links: result.Links, + }) + reported.DurationMS += result.DurationMS + if status == "" { + status = result.Status + } else { + status = Worse(status, result.Status) + } + } + for _, expected := range check.ExpectedInstances { + if _, present := seen[expected.Name]; !present { + reported.MissingInstances = append(reported.MissingInstances, expected.Name) + } + } + switch { + case len(results) == 0: + reported.Status = StatusMissing + reported.Summary = "No result was reported for this check." + default: + reported.Status = status + if len(reported.MissingInstances) > 0 { + reported.Status = Worse(reported.Status, StatusMissing) + } + reported.Summary = checkSummary(reported) + } + reported.Metrics = mergeMetrics(reported.Instances) + return reported +} + +func checkSummary(check ReportCheck) string { + if len(check.Instances) == 1 && len(check.MissingInstances) == 0 { + return check.Instances[0].Summary + } + counts := map[Status]int{} + for _, instance := range check.Instances { + counts[instance.Status]++ + } + parts := []string{plural(counts[StatusPass], "instance", "instances") + " passed"} + for _, status := range []Status{StatusFail, StatusDegraded, StatusSkip} { + if counts[status] > 0 { + parts = append(parts, plural(counts[status], "instance", "instances")+" "+string(status)) + } + } + if len(check.MissingInstances) > 0 { + parts = append(parts, plural(len(check.MissingInstances), "instance", "instances")+" reported nothing") + } + return strings.Join(parts, ", ") + "." +} + +func plural(count int, singular, pluralWord string) string { + word := pluralWord + if count == 1 { + word = singular + } + return strconv.Itoa(count) + " " + word +} + +// additiveMetrics are the metric names that stay meaningful when summed across +// the instances of one check. Everything else (medians, percentiles, sizes) +// stays visible per instance only, because summing it would be misleading. +var additiveMetrics = map[string]bool{ + "assets": true, + "builds_completed": true, + "builds_planned": true, + "cases": true, + "checks": true, + "completed_runs": true, + "packages": true, + "planned_runs": true, + "targets": true, + "tests_failed": true, + "tests_passed": true, + "tests_skipped": true, + "tests_total": true, +} + +func mergeMetrics(instances []InstanceReport) map[string]float64 { + if len(instances) == 0 { + return nil + } + if len(instances) == 1 { + if len(instances[0].Metrics) == 0 { + return nil + } + merged := make(map[string]float64, len(instances[0].Metrics)) + for key, value := range instances[0].Metrics { + merged[key] = value + } + return merged + } + merged := map[string]float64{} + for _, instance := range instances { + for key, value := range instance.Metrics { + if !additiveMetrics[key] { + continue + } + merged[key] += value + } + } + if len(merged) == 0 { + return nil + } + return merged +} + +func summarize(checks []ReportCheck) Verdict { + verdict := Verdict{Overall: StatusPass, Checks: len(checks)} + gateStatus := StatusPass + advisoryStatus := StatusPass + for _, check := range checks { + switch check.Status { + case StatusPass: + verdict.Passed++ + case StatusFail: + verdict.Failed++ + case StatusDegraded: + verdict.Degraded++ + case StatusSkip: + verdict.Skipped++ + case StatusMissing: + verdict.Missing++ + } + if check.Status == StatusMissing { + verdict.MissingChecks = append(verdict.MissingChecks, check.ID) + } + if check.Level == LevelGate { + gateStatus = Worse(gateStatus, check.Status) + // A gate that was skipped did not hold; it simply did not run, and + // treating that as a pass is exactly the silence this framework is + // meant to prevent. + if check.Status == StatusFail || check.Status == StatusDegraded || check.Status == StatusSkip { + verdict.GatesFailed = append(verdict.GatesFailed, check.ID) + } + continue + } + advisoryStatus = Worse(advisoryStatus, check.Status) + if check.Status == StatusFail || check.Status == StatusDegraded { + verdict.AdvisoriesFailed = append(verdict.AdvisoriesFailed, check.ID) + } + } + verdict.Overall = gateStatus + if verdict.Overall == StatusPass && advisoryStatus != StatusPass { + verdict.Overall = StatusDegraded + } + return verdict +} + +// evidenceAreaStatus is the worst status among the claims in an area. +func evidenceAreaStatus(report Report, ids []string) Status { + status := StatusPass + for _, id := range ids { + for _, evidence := range report.Evidence { + if evidence.ID == id { + status = Worse(status, evidence.Status) + } + } + } + return status +} + +func buildEvidence(catalog Catalog, report Report, selected map[Stage]struct{}) []ReportEvidence { + var entries []ReportEvidence + for _, evidence := range catalog.Evidence { + check, exists := catalog.Check(evidence.CheckID) + if !exists { + continue + } + if _, wanted := selected[check.Stage]; !wanted { + continue + } + status := StatusMissing + if reported, found := report.Check(evidence.CheckID); found { + status = reported.Status + if evidence.Instance != "" { + status = StatusMissing + for _, instance := range reported.Instances { + if instance.Name == evidence.Instance { + status = instance.Status + break + } + } + } + } + entries = append(entries, ReportEvidence{ + ID: evidence.ID, Title: evidence.Title, Area: evidence.Area, + Description: evidence.Description, + EvidenceLevel: evidence.EvidenceLevel, CheckID: evidence.CheckID, + Instance: evidence.Instance, Status: status, Inputs: evidence.Inputs, + RequiredTools: evidence.RequiredTools, Reproduce: evidence.Reproduce, + Artifacts: evidence.Artifacts, Proves: evidence.Proves, Limitations: evidence.Limitations, + }) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].ID < entries[j].ID }) + return entries +} + +// buildCoverage answers one question per ecosystem: was it exercised for this +// release, and did that hold? An ecosystem covered by several checks takes the +// worst of them, so a passing stamp cannot hide a failure elsewhere. +func buildCoverage(catalog Catalog, report Report, selected map[Stage]struct{}) Coverage { + statuses := map[string]Status{} + for _, check := range catalog.Checks { + if _, wanted := selected[check.Stage]; !wanted { + continue + } + reported, found := report.Check(check.ID) + if !found { + continue + } + for _, expected := range check.ExpectedInstances { + if len(expected.Ecosystems) == 0 { + continue + } + status := StatusMissing + for _, instance := range reported.Instances { + if instance.Name == expected.Name { + status = instance.Status + break + } + } + for _, ecosystem := range expected.Ecosystems { + if existing, present := statuses[ecosystem]; present { + statuses[ecosystem] = Worse(existing, status) + continue + } + statuses[ecosystem] = status + } + } + } + names := make([]string, 0, len(statuses)) + for ecosystem := range statuses { + names = append(names, ecosystem) + } + sort.Strings(names) + coverage := Coverage{} + for _, name := range names { + coverage.Ecosystems = append(coverage.Ecosystems, EcosystemCoverage{Name: name, Status: statuses[name]}) + } + return coverage +} + +func collectRunners(results []CheckResult) []Runner { + seen := map[Runner]struct{}{} + var runners []Runner + for _, result := range results { + if result.Runner == (Runner{}) { + continue + } + if _, exists := seen[result.Runner]; exists { + continue + } + seen[result.Runner] = struct{}{} + runners = append(runners, result.Runner) + } + sort.Slice(runners, func(i, j int) bool { + if runners[i].OS != runners[j].OS { + return runners[i].OS < runners[j].OS + } + if runners[i].Arch != runners[j].Arch { + return runners[i].Arch < runners[j].Arch + } + return runners[i].GoVersion < runners[j].GoVersion + }) + return runners +} diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go new file mode 100644 index 00000000..b213235d --- /dev/null +++ b/internal/assurance/assurance_test.go @@ -0,0 +1,499 @@ +package assurance + +import ( + "encoding/json" + "flag" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +var updateGoldens = flag.Bool("update", false, "rewrite golden files") + +func loadFixture(t *testing.T, name string) []CheckResult { + t.Helper() + results, err := LoadResults(filepath.Join("testdata", "fixtures", name, "results")) + if err != nil { + t.Fatalf("load %s fixture: %v", name, err) + } + return results +} + +func fixtureCatalog(t *testing.T) Catalog { + t.Helper() + catalog, err := LoadCatalog(filepath.Join("testdata", "catalog.json")) + if err != nil { + t.Fatalf("load fixture catalog: %v", err) + } + return catalog +} + +func buildFixtureReport(t *testing.T, fixture string, previous *Report) Report { + t.Helper() + return BuildReport(fixtureCatalog(t), loadFixture(t, fixture), BuildOptions{ + Release: Release{Tag: "v9.9.9", Version: "9.9.9", Commit: "0f2103c7e671653e519cf5edb0d3e86020202ecf"}, + Previous: previous, + IncludeEvidence: true, + GeneratedBy: "assurance-test", + Now: time.Date(2026, 8, 19, 12, 0, 0, 0, time.UTC), + }) +} + +func TestStatusSeverityOrdering(t *testing.T) { + ordered := []Status{StatusPass, StatusSkip, StatusMissing, StatusDegraded, StatusFail} + for index := 1; index < len(ordered); index++ { + if ordered[index].Severity() <= ordered[index-1].Severity() { + t.Fatalf("%s must be more severe than %s", ordered[index], ordered[index-1]) + } + if got := Worse(ordered[index-1], ordered[index]); got != ordered[index] { + t.Fatalf("Worse(%s, %s) = %s", ordered[index-1], ordered[index], got) + } + } +} + +func TestParseCheckResultRejectsInvalidDocuments(t *testing.T) { + valid := CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Instance: "go", + Stage: StagePrerequisites, Level: LevelGate, Status: StatusPass, + Summary: "18 of 18 tests passed.", + } + encoded, err := valid.Encode() + if err != nil { + t.Fatalf("encode: %v", err) + } + if _, err := ParseCheckResult(encoded); err != nil { + t.Fatalf("valid result rejected: %v", err) + } + + cases := map[string]func(*CheckResult){ + "bad schema": func(r *CheckResult) { r.SchemaVersion = "other/v1" }, + "bad id": func(r *CheckResult) { r.ID = "Smoke Tests" }, + "bad instance": func(r *CheckResult) { r.Instance = "go/../etc" }, + "bad stage": func(r *CheckResult) { r.Stage = "whenever" }, + "bad level": func(r *CheckResult) { r.Level = "critical" }, + "bad status": func(r *CheckResult) { r.Status = "green" }, + "missing is not reportable": func(r *CheckResult) { r.Status = StatusMissing }, + "no summary": func(r *CheckResult) { r.Summary = " " }, + "bad detail status": func(r *CheckResult) { r.Details = []Detail{{Name: "x", Status: "green"}} }, + "detail without name": func(r *CheckResult) { r.Details = []Detail{{Name: "", Status: StatusPass}} }, + "link without url": func(r *CheckResult) { r.Links = []Link{{Label: "run"}} }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + broken := valid + mutate(&broken) + data, err := json.Marshal(broken) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if _, err := ParseCheckResult(data); err == nil { + t.Fatal("expected the document to be rejected") + } + }) + } +} + +func TestParseCheckResultRejectsUnknownFields(t *testing.T) { + data := []byte(`{"schema_version":"bomly.assurance-check/v1","id":"smoke","stage":"prerequisites",` + + `"status":"pass","summary":"ok","surprise":true}`) + if _, err := ParseCheckResult(data); err == nil { + t.Fatal("expected unknown fields to be rejected") + } +} + +func TestLoadResultsIgnoresForeignJSON(t *testing.T) { + dir := t.TempDir() + nested := filepath.Join(dir, "artifact") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "run-manifest.json"), + []byte(`{"schema_version":"bomly.benchmark-run/v1"}`), 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } + result := CheckResult{ + SchemaVersion: CheckSchema, ID: "fuzz", Stage: StagePrerequisites, + Level: LevelAdvisory, Status: StatusPass, Summary: "28 targets ran.", + } + encoded, err := result.Encode() + if err != nil { + t.Fatalf("encode: %v", err) + } + if err := os.WriteFile(filepath.Join(nested, result.FileName()), encoded, 0o644); err != nil { + t.Fatalf("write result: %v", err) + } + loaded, err := LoadResults(dir) + if err != nil { + t.Fatalf("load results: %v", err) + } + if len(loaded) != 1 || loaded[0].ID != "fuzz" { + t.Fatalf("expected only the check result, got %+v", loaded) + } +} + +func TestBuildReportMergesInstancesAndFlagsGaps(t *testing.T) { + report := buildFixtureReport(t, "mixed-failure", nil) + + smoke, found := report.Check("smoke") + if !found { + t.Fatal("smoke check missing from the report") + } + if smoke.Status != StatusMissing { + t.Fatalf("smoke status = %s, want missing because one slice reported nothing", smoke.Status) + } + if len(smoke.MissingInstances) != 1 || smoke.MissingInstances[0] != "node" { + t.Fatalf("missing instances = %v, want [node]", smoke.MissingInstances) + } + if report.Verdict.Overall != StatusMissing && report.Verdict.Overall != StatusFail { + t.Fatalf("overall verdict = %s, want a blocking verdict", report.Verdict.Overall) + } + if !report.Verdict.Blocking() { + t.Fatal("a failed gate check must block") + } + if len(report.Unknown) != 1 || report.Unknown[0].ID != "mystery-check" { + t.Fatalf("unknown results = %+v, want the undeclared check", report.Unknown) + } + if len(report.Stages) != 3 { + t.Fatalf("stages = %d, want 3", len(report.Stages)) + } + for _, stage := range report.Stages { + if stage.Title != stage.ID.Title() { + t.Fatalf("stage %s title = %q", stage.ID, stage.Title) + } + } +} + +func TestBuildReportPassesWhenEveryCheckReports(t *testing.T) { + report := buildFixtureReport(t, "all-pass", nil) + if report.Verdict.Overall != StatusPass { + t.Fatalf("overall verdict = %s, want pass", report.Verdict.Overall) + } + if report.Verdict.Blocking() { + t.Fatal("a passing report must not block") + } + smoke, _ := report.Check("smoke") + if len(smoke.Instances) != 2 { + t.Fatalf("smoke instances = %d, want 2", len(smoke.Instances)) + } + if smoke.Metrics["tests_total"] != 42 { + t.Fatalf("merged tests_total = %v, want 42", smoke.Metrics["tests_total"]) + } + if len(report.Coverage.Ecosystems) != 2 { + t.Fatalf("coverage ecosystems = %v", report.Coverage.Ecosystems) + } + if len(report.Evidence) != 1 || report.Evidence[0].Status != StatusPass { + t.Fatalf("evidence = %+v, want the go graph claim passing", report.Evidence) + } +} + +func TestBuildReportAdvisoryFailureDegradesButDoesNotBlock(t *testing.T) { + results := loadFixture(t, "all-pass") + for index := range results { + if results[index].ID == "perf-samples" { + results[index].Status = StatusFail + } + } + report := BuildReport(fixtureCatalog(t), results, BuildOptions{ + Release: Release{Tag: "v9.9.9"}, Now: time.Unix(0, 0).UTC(), + }) + if report.Verdict.Overall != StatusDegraded { + t.Fatalf("overall verdict = %s, want degraded", report.Verdict.Overall) + } + if report.Verdict.Blocking() { + t.Fatal("an advisory failure must not block a release") + } +} + +func TestTrendsCompareMetricsAndStatuses(t *testing.T) { + previous := buildFixtureReport(t, "all-pass", nil) + current := buildFixtureReport(t, "mixed-failure", &previous) + if current.Trends == nil { + t.Fatal("expected trends against the previous release") + } + if current.Trends.PreviousTag != "v9.9.9" { + t.Fatalf("previous tag = %q", current.Trends.PreviousTag) + } + var found bool + for _, metric := range current.Trends.Metrics { + if metric.CheckID == "perf-samples" && metric.Metric == "cold_median_ms" { + found = true + if metric.Delta != 293 { + t.Fatalf("cold_median_ms delta = %v, want 293", metric.Delta) + } + if metric.Better != betterLower { + t.Fatalf("cold_median_ms better = %q, want lower", metric.Better) + } + } + } + if !found { + t.Fatal("expected a cold_median_ms trend") + } + if len(current.Trends.Changed) == 0 { + t.Fatal("expected changed checks between the two fixtures") + } +} + +func TestReportGoldens(t *testing.T) { + previous := buildFixtureReport(t, "all-pass", nil) + for _, testCase := range []struct { + name string + previous *Report + }{ + {name: "all-pass"}, + {name: "mixed-failure", previous: &previous}, + } { + t.Run(testCase.name, func(t *testing.T) { + report := buildFixtureReport(t, testCase.name, testCase.previous) + encoded, err := report.Encode() + if err != nil { + t.Fatalf("encode: %v", err) + } + compareGolden(t, testCase.name+".report.json", encoded) + markdown := RenderMarkdown(report, MarkdownOptions{IncludeChecks: true, IncludeTrends: true}) + compareGolden(t, testCase.name+".summary.md", []byte(markdown)) + + // The report must survive a round trip through its own parser. + if _, err := ParseReport(encoded); err != nil { + t.Fatalf("parse encoded report: %v", err) + } + }) + } +} + +func compareGolden(t *testing.T, name string, actual []byte) { + t.Helper() + path := filepath.Join("testdata", "golden", name) + if *updateGoldens { + if err := os.WriteFile(path, actual, 0o644); err != nil { + t.Fatalf("update golden: %v", err) + } + return + } + expected, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read golden %s: %v (run go test ./internal/assurance -update)", name, err) + } + if string(expected) != string(actual) { + t.Fatalf("golden %s differs; run go test ./internal/assurance -update to refresh", name) + } +} + +func TestParseGoTestEventsSummarisesRuns(t *testing.T) { + stream := strings.Join([]string{ + `{"Action":"run","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan"}`, + `{"Action":"output","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan","Output":"=== RUN\n"}`, + `{"Action":"pass","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan/scan-go","Elapsed":41.5}`, + `{"Action":"fail","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan/scan-npm","Elapsed":2}`, + `{"Action":"skip","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan/scan-bun","Elapsed":0}`, + `{"Action":"fail","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan","Elapsed":44}`, + `{"Action":"fail","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Elapsed":44}`, + "go: downloading something", + }, "\n") + summary, err := ParseGoTestEvents(strings.NewReader(stream), nil) + if err != nil { + t.Fatalf("parse: %v", err) + } + if summary.Total != 4 || summary.Passed != 1 || summary.Failed != 2 || summary.Skipped != 1 { + t.Fatalf("summary = %+v", summary) + } + if len(summary.Anomalies) != 1 { + t.Fatalf("anomalies = %v, want the non-JSON line", summary.Anomalies) + } + result := summary.ToCheckResult(CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Instance: "go", + Stage: StagePrerequisites, Level: LevelGate, + }, 1) + if err := result.Validate(); err != nil { + t.Fatalf("converted result invalid: %v", err) + } + if result.Status != StatusFail { + t.Fatalf("status = %s, want fail", result.Status) + } + if result.Metrics["tests_failed"] != 2 { + t.Fatalf("tests_failed = %v", result.Metrics["tests_failed"]) + } +} + +func TestParseGoTestEventsBuildFailure(t *testing.T) { + summary, err := ParseGoTestEvents(strings.NewReader("# github.com/example\nsyntax error\n"), nil) + if err != nil { + t.Fatalf("parse: %v", err) + } + result := summary.ToCheckResult(CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Stage: StagePrerequisites, Level: LevelGate, + }, 2) + if result.Status != StatusFail { + t.Fatalf("status = %s, want fail when no test ran and the command failed", result.Status) + } + if !strings.Contains(result.Summary, "exited with code 2") { + t.Fatalf("summary = %q", result.Summary) + } +} + +// TestGoTestZeroTestsFailsInsteadOfSkipping guards the case where a -run +// pattern stops matching: the command succeeds, no test runs, and the check +// must not be able to slide past a gate. +func TestGoTestZeroTestsFailsInsteadOfSkipping(t *testing.T) { + summary, err := ParseGoTestEvents(strings.NewReader(""), nil) + if err != nil { + t.Fatalf("parse: %v", err) + } + result := summary.ToCheckResult(CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Instance: "go", + Stage: StagePrerequisites, Level: LevelGate, + }, 0) + if result.Status != StatusFail { + t.Fatalf("status = %s, want fail when no test ran", result.Status) + } + if !strings.Contains(result.Summary, "proved nothing") { + t.Fatalf("summary = %q", result.Summary) + } +} + +// TestSkippedGateBlocks keeps a gate check that did not run from being counted +// as a pass. +func TestSkippedGateBlocks(t *testing.T) { + results := loadFixture(t, "all-pass") + for index := range results { + if results[index].ID == "release-checksums" { + results[index].Status = StatusSkip + } + } + report := BuildReport(fixtureCatalog(t), results, BuildOptions{ + Release: Release{Tag: "v9.9.9"}, Now: time.Unix(0, 0).UTC(), + }) + if !report.Verdict.Blocking() { + t.Fatal("a skipped gate check must block a release") + } +} + +func TestMatchJobURLPicksTheRunningJobOnThisRunner(t *testing.T) { + payload := []byte(`{"jobs":[ + {"name":"Portable suite (ubuntu-latest)","status":"completed","runner_name":"GitHub Actions 3","html_url":"https://example.test/job/1"}, + {"name":"Portable suite (macos-latest)","status":"in_progress","runner_name":"GitHub Actions 7","html_url":"https://example.test/job/2"}, + {"name":"Portable suite (windows-latest)","status":"in_progress","runner_name":"GitHub Actions 9","html_url":"https://example.test/job/3"} + ]}`) + url, err := MatchJobURL(payload, "GitHub Actions 7") + if err != nil { + t.Fatalf("match: %v", err) + } + if url != "https://example.test/job/2" { + t.Fatalf("url = %q, want the running job on this runner", url) + } + + for name, args := range map[string][2]string{ + "unknown runner": {string(payload), "GitHub Actions 42"}, + "no runner name": {string(payload), ""}, + "empty payload": {`{"jobs":[]}`, "GitHub Actions 7"}, + } { + t.Run(name, func(t *testing.T) { + url, err := MatchJobURL([]byte(args[0]), args[1]) + if err != nil { + t.Fatalf("match: %v", err) + } + if url != "" { + t.Fatalf("url = %q, want no match so the caller links the run", url) + } + }) + } + + ambiguous := []byte(`{"jobs":[ + {"name":"a","status":"in_progress","runner_name":"shared","html_url":"https://example.test/job/1"}, + {"name":"b","status":"in_progress","runner_name":"shared","html_url":"https://example.test/job/2"} + ]}`) + if url, err := MatchJobURL(ambiguous, "shared"); err != nil || url != "" { + t.Fatalf("ambiguous match returned %q (err=%v), want no link", url, err) + } + + if _, err := MatchJobURL([]byte("not json"), "runner"); err == nil { + t.Fatal("expected malformed payload to be reported") + } +} + +// TestSchemaVersionsArePinned makes a schema change a deliberate act. The +// published reports are read by bomly.dev, which renders a known list of +// versions: adding an optional field keeps the version, while removing or +// repurposing one has to raise it and be taught to the site first. +func TestSchemaVersionsArePinned(t *testing.T) { + for name, actual := range map[string]string{ + "check": CheckSchema, + "catalog": CatalogSchema, + "report": ReportSchema, + "index": IndexSchema, + } { + expected := map[string]string{ + "check": "bomly.assurance-check/v1", + "catalog": "bomly.assurance-catalog/v1", + "report": "bomly.assurance-report/v1", + "index": "bomly.assurance-index/v1", + }[name] + if actual != expected { + t.Errorf("%s schema is %q, want %q; raising it requires teaching bomly.dev the new shape first "+ + "(SUPPORTED_REPORT_SCHEMAS in lib/assurance.ts and REPORT_SCHEMAS in scripts/sync-assurance.mjs)", + name, actual, expected) + } + } +} + +// TestUntrustedNativeArchivesGuardsTheProbe covers the rule that a release +// verifier must never execute a file its own checksum check did not vouch for. +func TestUntrustedNativeArchivesGuardsTheProbe(t *testing.T) { + full := ArchiveName("bomly", "9.9.9", runtime.GOOS, runtime.GOARCH) + lite := ArchiveName("bomly-lite", "9.9.9", runtime.GOOS, runtime.GOARCH) + + for name, testCase := range map[string]struct { + outcome ChecksumOutcome + want []string + }{ + "both verified": {ChecksumOutcome{Verified: []string{full, lite}}, nil}, + "one mismatched": {ChecksumOutcome{Verified: []string{lite}, Mismatched: []string{full}}, []string{full}}, + // The result is sorted, and "bomly-lite_" sorts before "bomly_". + "unlisted entirely": {ChecksumOutcome{Verified: []string{"unrelated.tar.gz"}}, + []string{lite, full}}, + "nothing verified": {ChecksumOutcome{}, []string{lite, full}}, + } { + t.Run(name, func(t *testing.T) { + got := UntrustedNativeArchives(testCase.outcome, "9.9.9") + if len(got) != len(testCase.want) { + t.Fatalf("untrusted = %v, want %v", got, testCase.want) + } + for index, archive := range testCase.want { + if got[index] != archive { + t.Fatalf("untrusted = %v, want %v", got, testCase.want) + } + } + }) + } +} + +// TestParseReportRequiresClaimDescriptions keeps a published report from +// carrying a claim the page would render as an empty card. +func TestParseReportRequiresClaimDescriptions(t *testing.T) { + golden, err := os.ReadFile(filepath.Join("testdata", "golden", "all-pass.report.json")) + if err != nil { + t.Fatalf("read golden report: %v", err) + } + if _, err := ParseReport(golden); err != nil { + t.Fatalf("golden report rejected: %v", err) + } + + var report map[string]any + if err := json.Unmarshal(golden, &report); err != nil { + t.Fatalf("decode golden report: %v", err) + } + evidence, ok := report["evidence"].([]any) + if !ok || len(evidence) == 0 { + t.Fatal("golden report has no evidence to blank out") + } + evidence[0].(map[string]any)["description"] = " " + blanked, err := json.Marshal(report) + if err != nil { + t.Fatalf("encode report: %v", err) + } + if _, err := ParseReport(blanked); err == nil { + t.Fatal("expected a claim with a blank description to be rejected") + } +} diff --git a/internal/assurance/catalog.go b/internal/assurance/catalog.go new file mode 100644 index 00000000..f6e346fe --- /dev/null +++ b/internal/assurance/catalog.go @@ -0,0 +1,534 @@ +package assurance + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// CatalogSchema is the schema identifier of the assurance catalog. +const CatalogSchema = "bomly.assurance-catalog/v1" + +// MaxCatalogBytes bounds the catalog document. +const MaxCatalogBytes = 4 << 20 + +// DefaultCatalogPath is the repository-relative catalog location. +const DefaultCatalogPath = "docs/assurance/catalog.json" + +var ( + revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + containerDigestPattern = regexp.MustCompile(`@sha256:[0-9a-f]{64}$`) +) + +// EvidenceLevel describes how strong a public evidence claim is. +type EvidenceLevel string + +// Evidence strength levels, from strongest to weakest guarantee. +const ( + // EvidenceDeterministic is reproducible in process with no external input. + EvidenceDeterministic EvidenceLevel = "deterministic" + // EvidencePinnedInput depends on a pinned repository revision or fixture. + EvidencePinnedInput EvidenceLevel = "pinned-input" + // EvidenceSnapshot depends on an upstream tag that can move. + EvidenceSnapshot EvidenceLevel = "snapshot" + // EvidenceLiveService depends on data from a live service. + EvidenceLiveService EvidenceLevel = "live-service" +) + +// Valid reports whether the evidence level is known. +func (e EvidenceLevel) Valid() bool { + switch e { + case EvidenceDeterministic, EvidencePinnedInput, EvidenceSnapshot, EvidenceLiveService: + return true + default: + return false + } +} + +// Catalog declares every check the framework expects and every public evidence +// claim those checks back. It is the single source the report is built against. +type Catalog struct { + SchemaVersion string `json:"schema_version"` + Areas []Area `json:"areas"` + Checks []Check `json:"checks"` + Evidence []Evidence `json:"evidence"` +} + +// Area groups related checks and evidence for presentation. +type Area struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` +} + +// Source records which workflow and job produce a check's results. +type Source struct { + Workflow string `json:"workflow"` + Job string `json:"job"` +} + +// ExpectedInstance declares one matrix leg a check must report. +type ExpectedInstance struct { + Name string `json:"name"` + Ecosystems []string `json:"ecosystems,omitempty"` + Platform string `json:"platform,omitempty"` +} + +// Check declares one quality check: which stage runs it, whether it gates the +// release, which instances must report, and what it does and does not prove. +type Check struct { + ID string `json:"id"` + Title string `json:"title"` + Area string `json:"area"` + Stage Stage `json:"stage"` + Level Level `json:"level"` + Description string `json:"description"` + Source Source `json:"source"` + ExpectedInstances []ExpectedInstance `json:"expected_instances,omitempty"` + Reproduce [][]string `json:"reproduce,omitempty"` + Proves []string `json:"proves"` + Limitations []string `json:"limitations"` +} + +// Input names a pinned repository, fixture, container image, or service an +// evidence claim depends on. +type Input struct { + Kind string `json:"kind"` + Location string `json:"location"` + Ref string `json:"ref,omitempty"` + Revision string `json:"revision,omitempty"` + SHA256 string `json:"sha256,omitempty"` +} + +// EvidenceArtifact is a repository file that records an evidence claim's result. +type EvidenceArtifact struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` +} + +// Evidence is one public claim about Bomly's behavior, proven by a pinned input +// and a committed result file, and backed by a check whose per-release status +// shows whether the claim still holds. A claim that would only restate what its +// check already reports does not belong here — the check card says it once. +type Evidence struct { + ID string `json:"id"` + Title string `json:"title"` + Area string `json:"area"` + Description string `json:"description"` + EvidenceLevel EvidenceLevel `json:"evidence_level"` + CheckID string `json:"check_id"` + Instance string `json:"instance,omitempty"` + Inputs []Input `json:"inputs"` + RequiredTools []string `json:"required_tools,omitempty"` + Reproduce [][]string `json:"reproduce"` + Artifacts []EvidenceArtifact `json:"artifacts"` + Proves []string `json:"proves"` + Limitations []string `json:"limitations"` +} + +// ParseCatalog decodes and structurally validates a catalog document. It does +// not touch the filesystem; use VerifyArtifacts for the repository-side checks. +func ParseCatalog(data []byte) (Catalog, error) { + if len(data) > MaxCatalogBytes { + return Catalog{}, fmt.Errorf("catalog is %d bytes, limit is %d", len(data), MaxCatalogBytes) + } + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + var catalog Catalog + if err := decoder.Decode(&catalog); err != nil { + return Catalog{}, fmt.Errorf("decode assurance catalog: %w", err) + } + if err := ensureEOF(decoder, "assurance catalog"); err != nil { + return Catalog{}, err + } + if err := catalog.Validate(); err != nil { + return Catalog{}, err + } + return catalog, nil +} + +// LoadCatalog reads and validates the catalog at path. +func LoadCatalog(path string) (Catalog, error) { + data, err := readBounded(path, MaxCatalogBytes) + if err != nil { + return Catalog{}, err + } + return ParseCatalog(data) +} + +// Validate reports whether the catalog is internally consistent. +func (c Catalog) Validate() error { + if c.SchemaVersion != CatalogSchema { + return fmt.Errorf("unsupported assurance catalog schema %q", c.SchemaVersion) + } + if len(c.Areas) == 0 { + return errCatalog("catalog declares no areas") + } + areas := make(map[string]struct{}, len(c.Areas)) + for index, area := range c.Areas { + if !idPattern.MatchString(area.ID) { + return fmt.Errorf("area %d has invalid id %q", index+1, area.ID) + } + if _, exists := areas[area.ID]; exists { + return fmt.Errorf("duplicate area %q", area.ID) + } + if strings.TrimSpace(area.Title) == "" || strings.TrimSpace(area.Description) == "" { + return fmt.Errorf("area %q requires a title and description", area.ID) + } + areas[area.ID] = struct{}{} + } + if len(c.Checks) == 0 { + return errCatalog("catalog declares no checks") + } + checks := make(map[string]Check, len(c.Checks)) + previous := "" + for index, check := range c.Checks { + if !idPattern.MatchString(check.ID) { + return fmt.Errorf("check %d has invalid id %q", index+1, check.ID) + } + if _, exists := checks[check.ID]; exists { + return fmt.Errorf("duplicate check %q", check.ID) + } + if previous != "" && check.ID < previous { + return fmt.Errorf("checks are not sorted by id: %q follows %q", check.ID, previous) + } + previous = check.ID + if err := validateCheck(check, areas); err != nil { + return fmt.Errorf("check %q: %w", check.ID, err) + } + checks[check.ID] = check + } + if len(c.Evidence) == 0 { + return errCatalog("catalog declares no evidence") + } + seenEvidence := make(map[string]struct{}, len(c.Evidence)) + previous = "" + for index, evidence := range c.Evidence { + if !idPattern.MatchString(evidence.ID) { + return fmt.Errorf("evidence %d has invalid id %q", index+1, evidence.ID) + } + if _, exists := seenEvidence[evidence.ID]; exists { + return fmt.Errorf("duplicate evidence %q", evidence.ID) + } + if previous != "" && evidence.ID < previous { + return fmt.Errorf("evidence is not sorted by id: %q follows %q", evidence.ID, previous) + } + previous = evidence.ID + seenEvidence[evidence.ID] = struct{}{} + if err := validateEvidence(evidence, areas, checks); err != nil { + return fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + } + return nil +} + +func validateCheck(check Check, areas map[string]struct{}) error { + if strings.TrimSpace(check.Title) == "" || strings.TrimSpace(check.Description) == "" { + return errCatalog("title and description are required") + } + if _, exists := areas[check.Area]; !exists { + return fmt.Errorf("unknown area %q", check.Area) + } + if !check.Stage.Valid() { + return fmt.Errorf("unsupported stage %q", check.Stage) + } + if !check.Level.Valid() { + return fmt.Errorf("unsupported level %q", check.Level) + } + if strings.TrimSpace(check.Source.Workflow) == "" || strings.TrimSpace(check.Source.Job) == "" { + return errCatalog("source workflow and job are required") + } + seen := make(map[string]struct{}, len(check.ExpectedInstances)) + for _, instance := range check.ExpectedInstances { + if !instancePattern.MatchString(instance.Name) { + return fmt.Errorf("invalid expected instance %q", instance.Name) + } + if _, exists := seen[instance.Name]; exists { + return fmt.Errorf("duplicate expected instance %q", instance.Name) + } + seen[instance.Name] = struct{}{} + } + if err := validateCommands(check.Reproduce); err != nil { + return err + } + return validateClaims(check.Proves, check.Limitations) +} + +func validateEvidence(evidence Evidence, areas map[string]struct{}, checks map[string]Check) error { + // Every claim carries the same fields whichever way it is asserted, so the + // published report can render one shape for all of them. + if strings.TrimSpace(evidence.Title) == "" || strings.TrimSpace(evidence.Description) == "" { + return errCatalog("title and description are required") + } + if _, exists := areas[evidence.Area]; !exists { + return fmt.Errorf("unknown area %q", evidence.Area) + } + if !evidence.EvidenceLevel.Valid() { + return fmt.Errorf("unsupported evidence level %q", evidence.EvidenceLevel) + } + check, exists := checks[evidence.CheckID] + if !exists { + return fmt.Errorf("unknown backing check %q", evidence.CheckID) + } + if evidence.Instance != "" { + if !instancePattern.MatchString(evidence.Instance) { + return fmt.Errorf("invalid instance %q", evidence.Instance) + } + if !hasInstance(check, evidence.Instance) { + return fmt.Errorf("check %q does not declare instance %q", evidence.CheckID, evidence.Instance) + } + } + if len(evidence.Inputs) == 0 { + return errCatalog("at least one input is required") + } + for _, input := range evidence.Inputs { + if err := validateInput(evidence.EvidenceLevel, input); err != nil { + return err + } + } + if len(evidence.Reproduce) == 0 { + return errCatalog("at least one reproduction command is required") + } + if err := validateCommands(evidence.Reproduce); err != nil { + return err + } + if len(evidence.Artifacts) == 0 { + return errCatalog("evidence requires at least one hashed artifact") + } + for _, artifact := range evidence.Artifacts { + if !hashPattern.MatchString(artifact.SHA256) { + return fmt.Errorf("artifact %q has an invalid SHA-256 hash", artifact.Path) + } + if err := validateRepoPath(artifact.Path); err != nil { + return err + } + } + return validateClaims(evidence.Proves, evidence.Limitations) +} + +func hasInstance(check Check, name string) bool { + for _, instance := range check.ExpectedInstances { + if instance.Name == name { + return true + } + } + return false +} + +func validateInput(level EvidenceLevel, input Input) error { + if strings.TrimSpace(input.Location) == "" { + return errCatalog("input location is required") + } + switch input.Kind { + case "git": + if !revisionPattern.MatchString(input.Revision) { + return errCatalog("git input requires a full lowercase commit revision") + } + case "fixture": + if !hashPattern.MatchString(input.SHA256) { + return errCatalog("fixture input requires a SHA-256 hash") + } + return validateRepoPath(input.Location) + case "container": + if input.Ref == "" { + return errCatalog("container input requires an image reference") + } + if level == EvidencePinnedInput && !containerDigestPattern.MatchString(input.Ref) { + return errCatalog("pinned container input requires an immutable sha256 digest") + } + default: + return fmt.Errorf("unsupported input kind %q", input.Kind) + } + return nil +} + +func validateCommands(commands [][]string) error { + for index, command := range commands { + if len(command) == 0 { + return fmt.Errorf("command %d is empty", index+1) + } + for _, argument := range command { + if argument == "" { + return fmt.Errorf("command %d contains an empty argument", index+1) + } + } + } + return nil +} + +func validateClaims(proves, limitations []string) error { + if len(proves) == 0 || len(limitations) == 0 { + return errCatalog("proves and limitations must both be explicit") + } + for _, entry := range append(append([]string{}, proves...), limitations...) { + if strings.TrimSpace(entry) == "" { + return errCatalog("proves and limitations cannot contain blank entries") + } + } + return nil +} + +func validateRepoPath(path string) error { + clean := filepath.Clean(filepath.FromSlash(path)) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return fmt.Errorf("path %q must stay inside the repository", path) + } + return nil +} + +// RefreshArtifacts rewrites every recorded hash from the file it names and +// reports how many changed. Golden files are regenerated by their own tooling, +// so the catalog has to be able to catch up without anyone editing hashes by +// hand. +func (c *Catalog) RefreshArtifacts(root string) (int, error) { + changed := 0 + for evidenceIndex := range c.Evidence { + evidence := &c.Evidence[evidenceIndex] + for artifactIndex := range evidence.Artifacts { + artifact := &evidence.Artifacts[artifactIndex] + actual, err := hashRepositoryFile(root, artifact.Path) + if err != nil { + return changed, fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + if actual != artifact.SHA256 { + artifact.SHA256 = actual + changed++ + } + } + for inputIndex := range evidence.Inputs { + input := &evidence.Inputs[inputIndex] + if input.Kind != "fixture" { + continue + } + actual, err := hashRepositoryFile(root, input.Location) + if err != nil { + return changed, fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + if actual != input.SHA256 { + input.SHA256 = actual + changed++ + } + } + } + return changed, nil +} + +// Encode renders the catalog as indented JSON with a trailing newline. +func (c Catalog) Encode() ([]byte, error) { + data, err := json.MarshalIndent(c, "", " ") + if err != nil { + return nil, fmt.Errorf("encode assurance catalog: %w", err) + } + return append(data, '\n'), nil +} + +// VerifyArtifacts confirms every catalog artifact and fixture input still +// matches its recorded hash inside root. +func (c Catalog) VerifyArtifacts(root string) error { + for _, evidence := range c.Evidence { + for _, artifact := range evidence.Artifacts { + if err := verifyHashedFile(root, artifact.Path, artifact.SHA256); err != nil { + return fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + } + for _, input := range evidence.Inputs { + if input.Kind != "fixture" { + continue + } + if err := verifyHashedFile(root, input.Location, input.SHA256); err != nil { + return fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + } + } + return nil +} + +func verifyHashedFile(root, path, want string) error { + actual, err := hashRepositoryFile(root, path) + if err != nil { + return err + } + if actual != want { + return fmt.Errorf("%q hash is %s, want %s", path, actual, want) + } + return nil +} + +// hashRepositoryFile reads one repository file and returns its SHA-256, after +// confirming the path stays inside root. +func hashRepositoryFile(root, path string) (string, error) { + if err := validateRepoPath(path); err != nil { + return "", err + } + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return "", fmt.Errorf("resolve repository root: %w", err) + } + resolved, err := filepath.EvalSymlinks(filepath.Join(root, filepath.FromSlash(path))) + if err != nil { + return "", fmt.Errorf("resolve %q: %w", path, err) + } + relative, err := filepath.Rel(resolvedRoot, resolved) + if err != nil { + return "", fmt.Errorf("resolve %q relative to repository: %w", path, err) + } + if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("path %q resolves outside the repository", path) + } + info, err := os.Stat(resolved) + if err != nil { + return "", fmt.Errorf("inspect %q: %w", path, err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("%q is not a regular file", path) + } + data, err := os.ReadFile(resolved) + if err != nil { + return "", fmt.Errorf("read %q: %w", path, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// ChecksForStage returns the catalog checks that belong to one stage. +func (c Catalog) ChecksForStage(stage Stage) []Check { + var selected []Check + for _, check := range c.Checks { + if check.Stage == stage { + selected = append(selected, check) + } + } + return selected +} + +// Check looks up one catalog check by ID. +func (c Catalog) Check(id string) (Check, bool) { + for _, check := range c.Checks { + if check.ID == id { + return check, true + } + } + return Check{}, false +} + +// AreaTitle returns the display title of an area, falling back to its ID. +func (c Catalog) AreaTitle(id string) string { + for _, area := range c.Areas { + if area.ID == id { + return area.Title + } + } + return id +} + +type catalogError string + +func (e catalogError) Error() string { return string(e) } + +func errCatalog(message string) error { return catalogError(message) } diff --git a/internal/assurance/catalog_test.go b/internal/assurance/catalog_test.go new file mode 100644 index 00000000..d2210ce5 --- /dev/null +++ b/internal/assurance/catalog_test.go @@ -0,0 +1,292 @@ +package assurance + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "gopkg.in/yaml.v3" +) + +// repositoryRoot is the module root relative to this package. +const repositoryRoot = "../.." + +func repositoryCatalog(t *testing.T) Catalog { + t.Helper() + catalog, err := LoadCatalog(filepath.Join(repositoryRoot, filepath.FromSlash(DefaultCatalogPath))) + if err != nil { + t.Fatalf("load repository catalog: %v", err) + } + return catalog +} + +func TestRepositoryCatalogIsValid(t *testing.T) { + catalog := repositoryCatalog(t) + if err := catalog.VerifyArtifacts(repositoryRoot); err != nil { + t.Fatalf("catalog artifacts drifted: %v", err) + } + for _, stage := range Stages() { + if len(catalog.ChecksForStage(stage)) == 0 { + t.Fatalf("no checks declared for the %s stage", stage) + } + } + gates := 0 + for _, check := range catalog.Checks { + if check.Level == LevelGate { + gates++ + } + } + if gates == 0 { + t.Fatal("the catalog declares no gate checks, so nothing could block a release") + } +} + +// TestCatalogSmokeInstancesMatchWorkflowMatrix keeps the declared smoke slices +// and the workflow matrix in step: a slice added to one and not the other would +// otherwise silently drop out of the report. +func TestCatalogSmokeInstancesMatchWorkflowMatrix(t *testing.T) { + catalog := repositoryCatalog(t) + check, found := catalog.Check("smoke") + if !found { + t.Fatal("the catalog does not declare the smoke check") + } + declared := map[string]struct{}{} + for _, instance := range check.ExpectedInstances { + declared[instance.Name] = struct{}{} + } + + data, err := os.ReadFile(filepath.Join(repositoryRoot, ".github", "workflows", "smoke.yml")) + if err != nil { + t.Fatalf("read smoke workflow: %v", err) + } + var workflow struct { + Jobs map[string]struct { + Strategy struct { + Matrix struct { + Slice []struct { + Name string `yaml:"name"` + } `yaml:"slice"` + } `yaml:"matrix"` + } `yaml:"strategy"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(data, &workflow); err != nil { + t.Fatalf("decode smoke workflow: %v", err) + } + slices := workflow.Jobs["smoke"].Strategy.Matrix.Slice + if len(slices) == 0 { + t.Fatal("the smoke workflow declares no slices") + } + seen := map[string]struct{}{} + for _, slice := range slices { + seen[slice.Name] = struct{}{} + if _, ok := declared[slice.Name]; !ok { + t.Errorf("smoke slice %q runs in CI but is not declared in %s", slice.Name, DefaultCatalogPath) + } + } + for name := range declared { + if _, ok := seen[name]; !ok { + t.Errorf("smoke slice %q is declared in the catalog but not in the workflow matrix", name) + } + } +} + +func TestCatalogRejectsInvalidDocuments(t *testing.T) { + base := func() Catalog { + return Catalog{ + SchemaVersion: CatalogSchema, + Areas: []Area{{ID: "end-to-end", Title: "End to end", Description: "Real runs."}}, + Checks: []Check{{ + ID: "smoke", Title: "Smoke", Area: "end-to-end", Stage: StagePrerequisites, + Level: LevelGate, Description: "Runs real scans.", + Source: Source{Workflow: "smoke.yml", Job: "smoke"}, + Proves: []string{"It scans."}, Limitations: []string{"One project per ecosystem."}, + }}, + Evidence: []Evidence{{ + ID: "graph-go", Title: "Go graph", Area: "end-to-end", + Description: "Scans a pinned example project and compares the result with a recorded one.", + EvidenceLevel: EvidencePinnedInput, CheckID: "smoke", + Inputs: []Input{{ + Kind: "git", Location: "https://example.test/repo", + Revision: "0f2103c7e671653e519cf5edb0d3e86020202ecf", + }}, + Reproduce: [][]string{{"make", "smoke"}}, + Artifacts: []EvidenceArtifact{{ + Path: "go.mod", SHA256: "0000000000000000000000000000000000000000000000000000000000000000", + }}, + Proves: []string{"It resolves."}, + Limitations: []string{"One toolchain."}, + }}, + } + } + if _, err := ParseCatalog(mustJSON(t, base())); err != nil { + t.Fatalf("valid catalog rejected: %v", err) + } + + cases := map[string]func(*Catalog){ + "bad schema": func(c *Catalog) { c.SchemaVersion = "other/v1" }, + "no areas": func(c *Catalog) { c.Areas = nil }, + "unknown check area": func(c *Catalog) { c.Checks[0].Area = "nowhere" }, + "bad stage": func(c *Catalog) { c.Checks[0].Stage = "later" }, + "bad level": func(c *Catalog) { c.Checks[0].Level = "blocking" }, + "no source": func(c *Catalog) { c.Checks[0].Source = Source{} }, + "blank claim": func(c *Catalog) { c.Checks[0].Proves = []string{" "} }, + "no limitations": func(c *Catalog) { c.Checks[0].Limitations = nil }, + "unknown backing": func(c *Catalog) { c.Evidence[0].CheckID = "nothing" }, + "evidence description": func(c *Catalog) { c.Evidence[0].Description = " " }, + "undeclared instance": func(c *Catalog) { c.Evidence[0].Instance = "go" }, + "bad evidence level": func(c *Catalog) { c.Evidence[0].EvidenceLevel = "vibes" }, + "no inputs": func(c *Catalog) { c.Evidence[0].Inputs = nil }, + "git without revision": func(c *Catalog) { + c.Evidence[0].Inputs = []Input{{Kind: "git", Location: "https://example.test/repo"}} + }, + "unsupported input kind": func(c *Catalog) { + c.Evidence[0].Inputs = []Input{{Kind: "workflow", Location: ".github/workflows/smoke.yml"}} + }, + "evidence without an artifact": func(c *Catalog) { c.Evidence[0].Artifacts = nil }, + "fixture without hash": func(c *Catalog) { + c.Evidence[0].Inputs = []Input{{Kind: "fixture", Location: "go.mod"}} + }, + "escaping fixture path": func(c *Catalog) { + c.Evidence[0].Inputs = []Input{{ + Kind: "fixture", Location: "../../../etc/passwd", + SHA256: "0000000000000000000000000000000000000000000000000000000000000000", + }} + }, + "unsorted checks": func(c *Catalog) { + c.Checks = append(c.Checks, c.Checks[0]) + c.Checks[1].ID = "aaa" + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + catalog := base() + mutate(&catalog) + if _, err := ParseCatalog(mustJSON(t, catalog)); err == nil { + t.Fatal("expected the catalog to be rejected") + } + }) + } +} + +func TestVerifyArtifactsDetectsDrift(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } + catalog := Catalog{Evidence: []Evidence{{ + ID: "example", + Artifacts: []EvidenceArtifact{{ + Path: "golden.json", SHA256: "1111111111111111111111111111111111111111111111111111111111111111", + }}, + }}} + if err := catalog.VerifyArtifacts(root); err == nil { + t.Fatal("expected a hash mismatch to be reported") + } +} + +func mustJSON(t *testing.T, value any) []byte { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return data +} + +// TestRepositoryCatalogProducesACompleteReport feeds the catalog a synthetic +// passing result for every check and instance it declares. It guards the wiring +// between the two: an evidence claim pointing at an instance no check can ever +// report, or a check whose instances cannot all be satisfied, shows up here +// instead of as a permanently "missing" entry in a published release report. +func TestRepositoryCatalogProducesACompleteReport(t *testing.T) { + catalog := repositoryCatalog(t) + var results []CheckResult + for _, check := range catalog.Checks { + instances := []string{""} + if len(check.ExpectedInstances) > 0 { + instances = nil + for _, instance := range check.ExpectedInstances { + instances = append(instances, instance.Name) + } + } + for _, instance := range instances { + results = append(results, CheckResult{ + SchemaVersion: CheckSchema, ID: check.ID, Instance: instance, + Stage: check.Stage, Level: check.Level, Status: StatusPass, + Summary: "Synthetic passing result.", + }) + } + } + report := BuildReport(catalog, results, BuildOptions{ + Release: Release{Tag: "v0.0.0", Version: "0.0.0"}, + IncludeEvidence: true, + }) + if report.Verdict.Overall != StatusPass { + t.Fatalf("verdict = %s, want pass; missing=%v gates=%v", + report.Verdict.Overall, report.Verdict.MissingChecks, report.Verdict.GatesFailed) + } + if len(report.Unknown) != 0 { + t.Fatalf("unknown results = %+v", report.Unknown) + } + if len(report.Checks) != len(catalog.Checks) { + t.Fatalf("reported %d checks, catalog declares %d", len(report.Checks), len(catalog.Checks)) + } + if len(report.Evidence) != len(catalog.Evidence) { + t.Fatalf("reported %d evidence claims, catalog declares %d", len(report.Evidence), len(catalog.Evidence)) + } + for _, evidence := range report.Evidence { + if evidence.Status != StatusPass { + t.Errorf("evidence %q resolved to %s even though every check passed", evidence.ID, evidence.Status) + } + } + for _, stage := range Stages() { + var found bool + for _, reported := range report.Stages { + if reported.ID == stage { + found = true + if reported.Verdict.Checks == 0 { + t.Errorf("stage %s reported no checks", stage) + } + } + } + if !found { + t.Errorf("stage %s is missing from the report", stage) + } + } + if len(report.Coverage.Ecosystems) == 0 { + t.Fatal("the coverage matrix is empty") + } +} + +func TestRefreshArtifactsRewritesDriftedHashes(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } + catalog := Catalog{Evidence: []Evidence{{ + ID: "example", + Artifacts: []EvidenceArtifact{{ + Path: "golden.json", SHA256: "1111111111111111111111111111111111111111111111111111111111111111", + }}, + Inputs: []Input{{ + Kind: "fixture", Location: "golden.json", + SHA256: "2222222222222222222222222222222222222222222222222222222222222222", + }}, + }}} + changed, err := catalog.RefreshArtifacts(root) + if err != nil { + t.Fatalf("refresh: %v", err) + } + if changed != 2 { + t.Fatalf("refreshed %d hashes, want 2", changed) + } + if err := catalog.VerifyArtifacts(root); err != nil { + t.Fatalf("refreshed catalog still fails verification: %v", err) + } + again, err := catalog.RefreshArtifacts(root) + if err != nil || again != 0 { + t.Fatalf("second refresh changed %d hashes (err=%v), want 0", again, err) + } +} diff --git a/internal/assurance/cmd/commands.go b/internal/assurance/cmd/commands.go new file mode 100644 index 00000000..c636eac0 --- /dev/null +++ b/internal/assurance/cmd/commands.go @@ -0,0 +1,592 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "time" + + "github.com/bomly-dev/bomly-cli/internal/assurance" +) + +func goos() string { return runtime.GOOS } +func goarch() string { return runtime.GOARCH } +func goVersion() string { return runtime.Version() } + +// ------------------------------------------------------------------ emit --- + +func runEmit(args []string) error { + flags := flag.NewFlagSet("emit", flag.ExitOnError) + id := flags.String("id", "", "catalog check id") + instance := flags.String("instance", "", "matrix instance name, such as an ecosystem or platform") + stage := flags.String("stage", "", "override the stage declared in the catalog") + level := flags.String("level", "", "override the level declared in the catalog") + status := flags.String("status", "", "explicit status: pass, fail, degraded, or skip") + exitCode := flags.Int("exit-code", 0, "command exit code that decides the status") + summary := flags.String("summary", "", "one-line plain-language summary") + durationMS := flags.Float64("duration-ms", 0, "check duration in milliseconds") + startedAt := flags.String("started-at", "", "RFC 3339 start time") + out := flags.String("out", "assurance-results", "directory to write the check result into") + catalogPath := flags.String("catalog", "", "assurance catalog path") + detailsFile := flags.String("details-jsonl", "", "file of JSON detail objects, one per line") + stepSummary := flags.Bool("step-summary", false, "append a markdown block to the workflow step summary") + var metrics, details, artifacts, links stringList + flags.Var(&metrics, "metric", "numeric metric as name=value (repeatable)") + flags.Var(&details, "detail", "sub-result as name=status[:note] (repeatable)") + flags.Var(&artifacts, "artifact", "produced file as name=path (repeatable)") + flags.Var(&links, "link", "related link as label=url (repeatable)") + if err := flags.Parse(args); err != nil { + return err + } + if *id == "" || *summary == "" { + return fmt.Errorf("emit requires --id and --summary") + } + + result := baseResult(*id, *instance) + if *startedAt != "" { + result.StartedAt = *startedAt + } + result.FinishedAt = time.Now().UTC().Format(time.RFC3339) + result.DurationMS = *durationMS + if result.DurationMS == 0 { + result.DurationMS = elapsedMS(result.StartedAt, result.FinishedAt) + } + result.Summary = *summary + result.Status = statusFromExit(*exitCode) + if *status != "" { + result.Status = assurance.Status(*status) + } + catalogCtx, err := loadContext(*catalogPath) + if err != nil { + return err + } + if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { + return err + } + for _, entry := range metrics { + name, raw, err := splitPair(entry, "metric") + if err != nil { + return err + } + value, convErr := strconv.ParseFloat(raw, 64) + if convErr != nil { + return fmt.Errorf("metric %q must be numeric: %w", entry, convErr) + } + if result.Metrics == nil { + result.Metrics = map[string]float64{} + } + result.Metrics[name] = value + } + for _, entry := range details { + detail, err := parseDetail(entry) + if err != nil { + return err + } + result.Details = append(result.Details, detail) + } + if *detailsFile != "" { + loaded, err := readDetailLines(*detailsFile) + if err != nil { + return err + } + result.Details = append(result.Details, loaded...) + } + for _, entry := range artifacts { + name, path, err := splitPair(entry, "artifact") + if err != nil { + return err + } + artifact := assurance.Artifact{Name: name} + if info, statErr := os.Stat(path); statErr == nil && info.Mode().IsRegular() { + artifact.Bytes = info.Size() + } + result.Artifacts = append(result.Artifacts, artifact) + } + for _, entry := range links { + label, url, err := splitPair(entry, "link") + if err != nil { + return err + } + result.Links = append(result.Links, assurance.Link{Label: label, URL: url}) + } + return writeResult(*out, result, *stepSummary, reproduceFor(catalogCtx.catalog, result.ID)) +} + +func parseDetail(entry string) (assurance.Detail, error) { + name, rest, err := splitPair(entry, "detail") + if err != nil { + return assurance.Detail{}, err + } + status, note, _ := strings.Cut(rest, ":") + detail := assurance.Detail{ + Name: name, + Status: assurance.Status(strings.TrimSpace(status)), + Note: strings.TrimSpace(note), + } + if !detail.Status.Valid() { + return assurance.Detail{}, fmt.Errorf("detail %q has unsupported status %q", entry, status) + } + return detail, nil +} + +// readDetailLines reads sub-results from a JSONL file so shell steps can record +// many items without building long command lines. +func readDetailLines(path string) ([]assurance.Detail, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open details file: %w", err) + } + defer file.Close() + var details []assurance.Detail + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + var raw struct { + Name string `json:"name"` + Status string `json:"status"` + ExitCode *int `json:"exit_code"` + Note string `json:"note"` + DurationMS float64 `json:"duration_ms"` + DurationS float64 `json:"duration_s"` + } + if err := json.Unmarshal([]byte(line), &raw); err != nil { + return nil, fmt.Errorf("decode details file line: %w", err) + } + detail := assurance.Detail{Name: raw.Name, Note: raw.Note, DurationMS: raw.DurationMS} + if detail.DurationMS == 0 && raw.DurationS > 0 { + detail.DurationMS = raw.DurationS * 1000 + } + switch { + case raw.Status != "": + detail.Status = assurance.Status(raw.Status) + case raw.ExitCode != nil: + detail.Status = statusFromExit(*raw.ExitCode) + default: + detail.Status = assurance.StatusPass + } + if detail.Name == "" || !detail.Status.Valid() { + return nil, fmt.Errorf("details file line %q needs a name and a valid status", line) + } + details = append(details, detail) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read details file: %w", err) + } + return details, nil +} + +func elapsedMS(started, finished string) float64 { + start, err := time.Parse(time.RFC3339, started) + if err != nil { + return 0 + } + end, err := time.Parse(time.RFC3339, finished) + if err != nil { + return 0 + } + return float64(end.Sub(start).Milliseconds()) +} + +// ---------------------------------------------------------------- gotest --- + +func runGoTest(args []string) error { + flags := flag.NewFlagSet("gotest", flag.ExitOnError) + id := flags.String("id", "", "catalog check id") + instance := flags.String("instance", "", "matrix instance name") + stage := flags.String("stage", "", "override the stage declared in the catalog") + level := flags.String("level", "", "override the level declared in the catalog") + input := flags.String("input", "", `file holding "go test -json" output`) + exitCode := flags.Int("exit-code", 0, "exit code of the test command") + echo := flags.Bool("echo", false, "replay test output to stdout so CI logs stay readable") + out := flags.String("out", "assurance-results", "directory to write the check result into") + catalogPath := flags.String("catalog", "", "assurance catalog path") + stepSummary := flags.Bool("step-summary", false, "append a markdown block to the workflow step summary") + if err := flags.Parse(args); err != nil { + return err + } + if *id == "" || *input == "" { + return fmt.Errorf("gotest requires --id and --input") + } + file, err := os.Open(*input) + if err != nil { + return fmt.Errorf("open test output: %w", err) + } + defer file.Close() + var echoWriter *os.File + if *echo { + echoWriter = os.Stdout + } + summary, err := assurance.ParseGoTestEvents(file, writerOrNil(echoWriter)) + if err != nil { + return err + } + result := baseResult(*id, *instance) + result.FinishedAt = time.Now().UTC().Format(time.RFC3339) + catalogCtx, err := loadContext(*catalogPath) + if err != nil { + return err + } + if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { + return err + } + return writeResult(*out, summary.ToCheckResult(result, *exitCode), *stepSummary, reproduceFor(catalogCtx.catalog, result.ID)) +} + +// reproduceFor returns the catalog's local reproduction command for a check. +func reproduceFor(catalog *assurance.Catalog, id string) [][]string { + if catalog == nil { + return nil + } + if check, found := catalog.Check(id); found { + return check.Reproduce + } + return nil +} + +func writerOrNil(file *os.File) *os.File { + if file == nil { + return nil + } + return file +} + +// --------------------------------------------------------------- convert --- + +func runConvert(args []string) error { + if len(args) == 0 { + return fmt.Errorf("convert requires a manifest kind: benchmark-run or sbom-assurance") + } + kind := args[0] + flags := flag.NewFlagSet("convert "+kind, flag.ExitOnError) + id := flags.String("id", "", "catalog check id") + instance := flags.String("instance", "", "matrix instance name") + stage := flags.String("stage", "", "override the stage declared in the catalog") + level := flags.String("level", "", "override the level declared in the catalog") + input := flags.String("input", "", "manifest path") + out := flags.String("out", "assurance-results", "directory to write the check result into") + catalogPath := flags.String("catalog", "", "assurance catalog path") + stepSummary := flags.Bool("step-summary", false, "append a markdown block to the workflow step summary") + if err := flags.Parse(args[1:]); err != nil { + return err + } + if *id == "" || *input == "" { + return fmt.Errorf("convert requires --id and --input") + } + data, err := os.ReadFile(*input) + if err != nil { + return fmt.Errorf("read manifest: %w", err) + } + result := baseResult(*id, *instance) + result.FinishedAt = time.Now().UTC().Format(time.RFC3339) + catalogCtx, err := loadContext(*catalogPath) + if err != nil { + return err + } + if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { + return err + } + var converted assurance.CheckResult + switch kind { + case "benchmark-run": + converted, err = assurance.ConvertBenchmarkRun(data, result) + case "sbom-assurance": + converted, err = assurance.ConvertSBOMAssurance(data, result) + default: + return fmt.Errorf("unsupported manifest kind %q", kind) + } + if err != nil { + return err + } + return writeResult(*out, converted, *stepSummary, reproduceFor(catalogCtx.catalog, converted.ID)) +} + +// -------------------------------------------------------- verify-release --- + +func runVerifyRelease(args []string) error { + flags := flag.NewFlagSet("verify-release", flag.ExitOnError) + dir := flags.String("dir", "", "directory holding downloaded release assets") + version := flags.String("version", "", "release version without the leading v") + scope := flags.String("scope", "full", "full checks every expected asset; native checks only this platform's") + out := flags.String("out", "assurance-results", "directory to write check results into") + workDir := flags.String("work", "", "extraction directory (a temporary directory by default)") + catalogPath := flags.String("catalog", "", "assurance catalog path") + stepSummary := flags.Bool("step-summary", false, "append markdown blocks to the workflow step summary") + if err := flags.Parse(args); err != nil { + return err + } + if *dir == "" || *version == "" { + return fmt.Errorf("verify-release requires --dir and --version") + } + trimmed := strings.TrimPrefix(*version, "v") + catalogCtx, err := loadContext(*catalogPath) + if err != nil { + return err + } + work := *workDir + if work == "" { + created, err := os.MkdirTemp("", "assurance-release-") + if err != nil { + return fmt.Errorf("create extraction directory: %w", err) + } + defer os.RemoveAll(created) + work = created + } + + failures := 0 + emit := func(result assurance.CheckResult) error { + if result.Status == assurance.StatusFail { + failures++ + } + return writeResult(*out, result, *stepSummary, reproduceFor(catalogCtx.catalog, result.ID)) + } + + if *scope == "full" { + presence, err := assurance.InspectAssets(*dir, trimmed) + if err != nil { + return err + } + result := baseResult("release-assets", "") + result.FinishedAt = time.Now().UTC().Format(time.RFC3339) + if err := applyCatalog(&result, catalogCtx.catalog, "", ""); err != nil { + return err + } + result.Status = assurance.StatusPass + result.Metrics = map[string]float64{ + "assets": float64(len(presence.Present)), + "assets_missing": float64(len(presence.Missing)), + } + for _, name := range presence.Present { + result.Details = append(result.Details, assurance.Detail{Name: name, Status: assurance.StatusPass}) + } + for _, name := range presence.Missing { + result.Status = assurance.StatusFail + result.Details = append(result.Details, assurance.Detail{ + Name: name, Status: assurance.StatusFail, Note: "not attached to the release", + }) + } + result.Summary = fmt.Sprintf("%d of %d expected release assets are attached.", + len(presence.Present), len(presence.Present)+len(presence.Missing)) + if len(presence.Extra) > 0 { + result.Summary += fmt.Sprintf(" %d unexpected file(s): %s.", + len(presence.Extra), strings.Join(presence.Extra, ", ")) + } + if err := emit(result); err != nil { + return err + } + } + + sumsData, err := os.ReadFile(filepath.Join(*dir, "SHA256SUMS")) + if err != nil { + return fmt.Errorf("read SHA256SUMS: %w", err) + } + entries, err := assurance.ParseSHA256SUMS(sumsData) + if err != nil { + return err + } + outcome, err := assurance.VerifyChecksums(*dir, entries) + if err != nil { + return err + } + platform := runtime.GOOS + "-" + runtime.GOARCH + checksums := baseResult("release-checksums", platform) + checksums.FinishedAt = time.Now().UTC().Format(time.RFC3339) + if err := applyCatalog(&checksums, catalogCtx.catalog, "", ""); err != nil { + return err + } + checksums.Status = assurance.StatusPass + checksums.Metrics = map[string]float64{ + "verified": float64(len(outcome.Verified)), + "mismatched": float64(len(outcome.Mismatched)), + "listed": float64(len(entries)), + } + for _, name := range outcome.Verified { + checksums.Details = append(checksums.Details, assurance.Detail{Name: name, Status: assurance.StatusPass}) + } + for _, name := range outcome.Mismatched { + checksums.Status = assurance.StatusFail + checksums.Details = append(checksums.Details, assurance.Detail{ + Name: name, Status: assurance.StatusFail, Note: "hash does not match SHA256SUMS", + }) + } + for _, name := range outcome.Unlisted { + checksums.Status = assurance.StatusFail + checksums.Details = append(checksums.Details, assurance.Detail{ + Name: name, Status: assurance.StatusFail, Note: "downloaded but absent from SHA256SUMS", + }) + } + if *scope == "full" && len(outcome.NotDownloaded) > 0 { + checksums.Status = assurance.StatusFail + for _, name := range outcome.NotDownloaded { + checksums.Details = append(checksums.Details, assurance.Detail{ + Name: name, Status: assurance.StatusFail, Note: "listed in SHA256SUMS but not attached", + }) + } + } + checksums.Summary = fmt.Sprintf("%d downloaded assets match SHA256SUMS, which lists %d files.", + len(outcome.Verified), len(entries)) + if len(outcome.Mismatched) > 0 { + checksums.Summary = fmt.Sprintf("%d assets do not match SHA256SUMS: %s.", + len(outcome.Mismatched), strings.Join(outcome.Mismatched, ", ")) + } + if err := emit(checksums); err != nil { + return err + } + + binaries := baseResult("release-binaries", platform) + binaries.FinishedAt = time.Now().UTC().Format(time.RFC3339) + if err := applyCatalog(&binaries, catalogCtx.catalog, "", ""); err != nil { + return err + } + + // Running a binary whose checksum did not match would invert the point of + // checking it: the checksum is what decides whether the file may be + // trusted, so a mismatch stops the probe rather than preceding it. + if untrusted := assurance.UntrustedNativeArchives(outcome, trimmed); len(untrusted) > 0 { + binaries.Status = assurance.StatusFail + binaries.Summary = fmt.Sprintf( + "Not run: %s did not match SHA256SUMS, so the binaries inside were not trusted enough to execute.", + strings.Join(untrusted, ", ")) + for _, archive := range untrusted { + binaries.Details = append(binaries.Details, assurance.Detail{ + Name: archive, Status: assurance.StatusFail, Note: "checksum mismatch, binary not run", + }) + } + if err := emit(binaries); err != nil { + return err + } + return blockingError{ + message: fmt.Sprintf("%d release verification check(s) failed", failures+1), code: 1, + } + } + + probes, err := assurance.ProbeNativeBinaries(context.Background(), *dir, trimmed, work) + if err != nil { + return err + } + binaries.Status = assurance.StatusPass + passed := 0 + for _, probe := range probes { + note := probe.Note + if probe.Status == assurance.StatusPass { + note = probe.Output + passed++ + } else { + binaries.Status = assurance.StatusFail + } + binaries.Details = append(binaries.Details, assurance.Detail{ + Name: probe.Archive, Status: probe.Status, Note: firstLine(note), + }) + } + binaries.Metrics = map[string]float64{"binaries": float64(len(probes)), "binaries_passed": float64(passed)} + binaries.Summary = fmt.Sprintf("%d of %d %s/%s binaries report version %s.", + passed, len(probes), runtime.GOOS, runtime.GOARCH, trimmed) + if err := emit(binaries); err != nil { + return err + } + + if failures > 0 { + return blockingError{message: fmt.Sprintf("%d release verification check(s) failed", failures), code: 1} + } + return nil +} + +func firstLine(value string) string { + if index := strings.IndexAny(value, "\r\n"); index >= 0 { + return strings.TrimSpace(value[:index]) + } + return strings.TrimSpace(value) +} + +// --------------------------------------------------------------- verdict --- + +func runVerdict(args []string) error { + flags := flag.NewFlagSet("verdict", flag.ExitOnError) + resultsDir := flags.String("results", "assurance-results", "directory of collected check results") + stage := flags.String("stage", "", "stage to judge") + catalogPath := flags.String("catalog", "", "assurance catalog path") + tag := flags.String("tag", "", "release tag, when one exists") + stepSummary := flags.Bool("step-summary", false, "append the verdict to the workflow step summary") + jsonOut := flags.String("json", "", "also write the stage report as JSON to this path") + if err := flags.Parse(args); err != nil { + return err + } + if *stage == "" { + return fmt.Errorf("verdict requires --stage") + } + selected := assurance.Stage(*stage) + if !selected.Valid() { + return fmt.Errorf("unsupported stage %q", *stage) + } + catalog, _, err := mustLoadCatalog(*catalogPath) + if err != nil { + return err + } + results, err := assurance.LoadResults(*resultsDir) + if err != nil { + return err + } + report := assurance.BuildReport(catalog, filterStage(results, selected), assurance.BuildOptions{ + Release: assurance.Release{Tag: *tag, Version: strings.TrimPrefix(*tag, "v")}, + Stages: []assurance.Stage{selected}, + StageRuns: map[assurance.Stage]string{selected: runURL()}, + GeneratedBy: "assurance", + }) + markdown := assurance.RenderMarkdown(report, assurance.MarkdownOptions{ + Heading: selected.Title(), + IncludeChecks: true, + }) + fmt.Print(markdown) + if *stepSummary { + if err := appendStepSummary(markdown); err != nil { + return err + } + } + if *jsonOut != "" { + data, encodeErr := report.Encode() + if encodeErr != nil { + return encodeErr + } + if err := os.WriteFile(*jsonOut, data, 0o644); err != nil { + return fmt.Errorf("write stage report: %w", err) + } + } + if report.Verdict.Blocking() { + return blockingError{ + message: fmt.Sprintf("%s did not pass: %s", selected.Title(), strings.Join(blockers(report.Verdict), ", ")), + code: 1, + } + } + return nil +} + +func blockers(verdict assurance.Verdict) []string { + var names []string + names = append(names, verdict.GatesFailed...) + for _, id := range verdict.MissingChecks { + names = append(names, id+" (no result)") + } + sort.Strings(names) + return names +} + +func filterStage(results []assurance.CheckResult, stage assurance.Stage) []assurance.CheckResult { + var selected []assurance.CheckResult + for _, result := range results { + if result.Stage == stage { + selected = append(selected, result) + } + } + return selected +} diff --git a/internal/assurance/cmd/joburl.go b/internal/assurance/cmd/joburl.go new file mode 100644 index 00000000..a7276569 --- /dev/null +++ b/internal/assurance/cmd/joburl.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/bomly-dev/bomly-cli/internal/assurance" +) + +// jobURL returns the URL a check result should point at: the job it ran in when +// that can be resolved, and the workflow run otherwise. +// +// Every instance of a matrix check shares one run, so run-level links cannot +// tell a reader which platform or slice a number came from. Resolving the job +// costs one read of public workflow metadata and is skipped silently whenever +// it does not work. +func jobURL(runURL string) string { + if override := os.Getenv("ASSURANCE_JOB_URL"); override != "" { + return override + } + repository := os.Getenv("GITHUB_REPOSITORY") + runID := os.Getenv("GITHUB_RUN_ID") + runnerName := os.Getenv("RUNNER_NAME") + if repository == "" || runID == "" || runnerName == "" { + return runURL + } + endpoint := fmt.Sprintf("https://api.github.com/repos/%s/actions/runs/%s/jobs?per_page=100", repository, runID) + if attempt := os.Getenv("GITHUB_RUN_ATTEMPT"); attempt != "" { + endpoint = fmt.Sprintf("https://api.github.com/repos/%s/actions/runs/%s/attempts/%s/jobs?per_page=100", + repository, runID, attempt) + } + payload, err := fetchJSON(endpoint) + if err != nil { + fmt.Fprintf(os.Stderr, "assurance: could not resolve the job URL (%v); linking the run instead\n", err) + return runURL + } + resolved, err := assurance.MatchJobURL(payload, runnerName) + if err != nil || resolved == "" { + return runURL + } + return resolved +} + +func fetchJSON(endpoint string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + request.Header.Set("Accept", "application/vnd.github+json") + // The token lifts the rate limit; the endpoint is readable without one on a + // public repository, so a missing token is not an error. + for _, name := range []string{"GH_TOKEN", "GITHUB_TOKEN"} { + if token := os.Getenv(name); token != "" { + request.Header.Set("Authorization", "Bearer "+token) + break + } + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("workflow jobs request returned %s", response.Status) + } + return io.ReadAll(io.LimitReader(response.Body, 8<<20)) +} diff --git a/internal/assurance/cmd/main.go b/internal/assurance/cmd/main.go new file mode 100644 index 00000000..e0f0d6a6 --- /dev/null +++ b/internal/assurance/cmd/main.go @@ -0,0 +1,276 @@ +// Command assurance drives Bomly's release assurance framework. Quality checks +// call it to emit their results, and the release pipeline calls it to judge a +// stage and to build the per-release report the public assurance page renders. +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/bomly-dev/bomly-cli/internal/assurance" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + var err error + switch os.Args[1] { + case "emit": + err = runEmit(os.Args[2:]) + case "gotest": + err = runGoTest(os.Args[2:]) + case "convert": + err = runConvert(os.Args[2:]) + case "verify-release": + err = runVerifyRelease(os.Args[2:]) + case "verdict": + err = runVerdict(os.Args[2:]) + case "report": + err = runReport(os.Args[2:]) + case "catalog-validate": + err = runCatalogValidate(os.Args[2:]) + case "-h", "--help", "help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "assurance: unknown command %q\n\n", os.Args[1]) + usage() + os.Exit(2) + } + if err != nil { + var blocking blockingError + if errors.As(err, &blocking) { + fmt.Fprintln(os.Stderr, "assurance:", blocking.Error()) + os.Exit(blocking.code) + } + fmt.Fprintln(os.Stderr, "assurance:", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `assurance drives Bomly's release assurance framework. + +Commands: + emit write a check result from flags + gotest turn a "go test -json" stream into a check result + convert turn a tool manifest into a check result + verify-release verify downloaded release assets and emit check results + verdict judge one stage from collected check results + report build the per-release assurance report and index + catalog-validate validate the assurance catalog and print its contents + +Run "assurance -h" for the flags of one command. +`) +} + +type blockingError struct { + message string + code int +} + +func (e blockingError) Error() string { return e.message } + +// ---------------------------------------------------------------- shared --- + +type resultContext struct { + catalog *assurance.Catalog + catalogPath string + root string +} + +// loadContext resolves the catalog a result is described by. A catalog that +// cannot be read at all leaves the context empty, so a check running outside a +// checkout can still emit with an explicit --stage and --level; a catalog that +// exists but does not parse is reported, because silently treating every check +// as undeclared would drop the stage and level a gate depends on. +func loadContext(catalogPath string) (resultContext, error) { + ctx := resultContext{catalogPath: catalogPath} + root, err := repositoryRoot() + if err != nil { + return ctx, nil + } + ctx.root = root + path := catalogPath + if path == "" { + path = filepath.Join(root, filepath.FromSlash(assurance.DefaultCatalogPath)) + if _, statErr := os.Stat(path); statErr != nil { + return ctx, nil + } + } else if !filepath.IsAbs(path) { + path = filepath.Join(root, filepath.FromSlash(path)) + } + catalog, err := assurance.LoadCatalog(path) + if err != nil { + return ctx, err + } + ctx.catalog = &catalog + ctx.catalogPath = path + return ctx, nil +} + +func mustLoadCatalog(catalogPath string) (assurance.Catalog, string, error) { + ctx, err := loadContext(catalogPath) + if err != nil { + return assurance.Catalog{}, "", err + } + if ctx.catalog == nil { + path := catalogPath + if path == "" { + path = assurance.DefaultCatalogPath + } + resolved := path + if ctx.root != "" && !filepath.IsAbs(path) { + resolved = filepath.Join(ctx.root, filepath.FromSlash(path)) + } + catalog, err := assurance.LoadCatalog(resolved) + if err != nil { + return assurance.Catalog{}, "", err + } + return catalog, resolved, nil + } + return *ctx.catalog, ctx.catalogPath, nil +} + +func repositoryRoot() (string, error) { + current, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("resolve working directory: %w", err) + } + for { + if info, statErr := os.Stat(filepath.Join(current, "go.mod")); statErr == nil && !info.IsDir() { + return current, nil + } + parent := filepath.Dir(current) + if parent == current { + return "", errors.New("find repository root: go.mod not found") + } + current = parent + } +} + +// baseResult fills the fields every check result shares from the environment. +func baseResult(id, instance string) assurance.CheckResult { + result := assurance.CheckResult{ + SchemaVersion: assurance.CheckSchema, + ID: id, + Instance: instance, + Ref: os.Getenv("GITHUB_REF"), + Commit: os.Getenv("GITHUB_SHA"), + Tag: os.Getenv("BOMLY_ASSURANCE_TAG"), + Job: os.Getenv("GITHUB_JOB"), + Runner: assurance.Runner{OS: goos(), Arch: goarch(), GoVersion: goVersion()}, + StartedAt: time.Now().UTC().Format(time.RFC3339), + } + result.Version = strings.TrimPrefix(result.Tag, "v") + result.RunURL = jobURL(runURL()) + return result +} + +func runURL() string { + server := os.Getenv("GITHUB_SERVER_URL") + repository := os.Getenv("GITHUB_REPOSITORY") + runID := os.Getenv("GITHUB_RUN_ID") + if server == "" || repository == "" || runID == "" { + return "" + } + url := fmt.Sprintf("%s/%s/actions/runs/%s", server, repository, runID) + if attempt := os.Getenv("GITHUB_RUN_ATTEMPT"); attempt != "" { + url += "/attempts/" + attempt + } + return url +} + +// applyCatalog fills the stage and level a check is declared with. +func applyCatalog(result *assurance.CheckResult, catalog *assurance.Catalog, stage, level string) error { + if catalog != nil { + if check, found := catalog.Check(result.ID); found { + if result.Stage == "" { + result.Stage = check.Stage + } + if result.Level == "" { + result.Level = check.Level + } + } + } + if stage != "" { + result.Stage = assurance.Stage(stage) + } + if level != "" { + result.Level = assurance.Level(level) + } + if result.Stage == "" { + return fmt.Errorf("check %q is not in the catalog, so --stage is required", result.ID) + } + return nil +} + +func writeResult(outDir string, result assurance.CheckResult, stepSummary bool, reproduce [][]string) error { + if err := result.Validate(); err != nil { + return err + } + if err := os.MkdirAll(outDir, 0o755); err != nil { + return fmt.Errorf("create result directory: %w", err) + } + data, err := result.Encode() + if err != nil { + return err + } + path := filepath.Join(outDir, result.FileName()) + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + fmt.Printf("%s %s: %s\n", assurance.StatusIcon(result.Status), result.Key(), result.Summary) + if stepSummary { + if err := appendStepSummary(assurance.RenderResultMarkdown(result, reproduce)); err != nil { + return err + } + } + return nil +} + +func appendStepSummary(markdown string) error { + path := os.Getenv("GITHUB_STEP_SUMMARY") + if path == "" { + return nil + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("open step summary: %w", err) + } + defer file.Close() + if _, err := file.WriteString(markdown); err != nil { + return fmt.Errorf("write step summary: %w", err) + } + return nil +} + +type stringList []string + +func (s *stringList) String() string { return strings.Join(*s, ",") } + +func (s *stringList) Set(value string) error { + *s = append(*s, value) + return nil +} + +func splitPair(value, what string) (string, string, error) { + key, rest, found := strings.Cut(value, "=") + if !found || strings.TrimSpace(key) == "" { + return "", "", fmt.Errorf("%s %q must be written as name=value", what, value) + } + return strings.TrimSpace(key), strings.TrimSpace(rest), nil +} + +func statusFromExit(code int) assurance.Status { + if code == 0 { + return assurance.StatusPass + } + return assurance.StatusFail +} diff --git a/internal/assurance/cmd/report.go b/internal/assurance/cmd/report.go new file mode 100644 index 00000000..e5a857c6 --- /dev/null +++ b/internal/assurance/cmd/report.go @@ -0,0 +1,386 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/bomly-dev/bomly-cli/internal/assurance" +) + +// defaultReportDir is the repository directory holding published reports. +const defaultReportDir = "docs/assurance" + +func runReport(args []string) error { + flags := flag.NewFlagSet("report", flag.ExitOnError) + resultsDir := flags.String("results", "assurance-results", "directory of collected check results") + catalogPath := flags.String("catalog", "", "assurance catalog path") + tag := flags.String("tag", "", "release tag, such as v0.24.0") + commit := flags.String("commit", "", "release commit") + releaseURL := flags.String("url", "", "release page URL") + publishedAt := flags.String("published-at", "", "RFC 3339 publication time") + previous := flags.String("previous", "", "previous release report for trends (auto by default)") + outDir := flags.String("out", "", "directory to write reports and the index into") + summaryOut := flags.String("summary-out", "", "write the markdown summary to this path") + stepSummary := flags.Bool("step-summary", false, "append the markdown summary to the workflow step summary") + stages := flags.String("stages", "", "comma-separated stages to include (every stage by default)") + prerequisitesRun := flags.String("prerequisites-run", "", "workflow run URL for the prerequisites stage") + preReleaseRun := flags.String("pre-release-run", "", "workflow run URL for the pre-release stage") + assessmentRun := flags.String("assessment-run", "", "workflow run URL for the post-release stage") + failOnBlocking := flags.Bool("fail-on-blocking", false, "exit non-zero when a gate check did not pass") + allowUnknown := flags.Bool("allow-unknown", false, "report results the catalog does not declare instead of failing") + if err := flags.Parse(args); err != nil { + return err + } + if *tag == "" { + return fmt.Errorf("report requires --tag") + } + catalog, catalogFile, err := mustLoadCatalog(*catalogPath) + if err != nil { + return err + } + results, err := assurance.LoadResults(*resultsDir) + if err != nil { + return err + } + targetDir := *outDir + if targetDir == "" { + root := filepath.Dir(filepath.Dir(catalogFile)) + targetDir = filepath.Join(root, filepath.FromSlash(defaultReportDir)) + if filepath.Base(filepath.Dir(catalogFile)) == "assurance" { + targetDir = filepath.Dir(catalogFile) + } + } + indexPath := filepath.Join(targetDir, "index.json") + + index := assurance.Index{SchemaVersion: assurance.IndexSchema} + if data, readErr := os.ReadFile(indexPath); readErr == nil { + loaded, parseErr := assurance.ParseIndex(data) + if parseErr != nil { + return parseErr + } + index = loaded + } + + previousReport, previousTag, err := resolvePrevious(*previous, *tag, targetDir, index) + if err != nil { + return err + } + + options := assurance.BuildOptions{ + Release: assurance.Release{ + Tag: *tag, Version: strings.TrimPrefix(*tag, "v"), + Commit: *commit, URL: *releaseURL, PublishedAt: *publishedAt, + }, + StageRuns: map[assurance.Stage]string{ + assurance.StagePrerequisites: *prerequisitesRun, + assurance.StagePreRelease: *preReleaseRun, + assurance.StagePostRelease: *assessmentRun, + }, + Previous: previousReport, + IncludeEvidence: true, + GeneratedBy: "assurance", + } + if *stages != "" { + for _, name := range strings.Split(*stages, ",") { + stage := assurance.Stage(strings.TrimSpace(name)) + if !stage.Valid() { + return fmt.Errorf("unsupported stage %q", name) + } + options.Stages = append(options.Stages, stage) + } + } + report := assurance.BuildReport(catalog, results, options) + + reportPath := filepath.Join(targetDir, "reports", *tag+".json") + if err := os.MkdirAll(filepath.Dir(reportPath), 0o755); err != nil { + return fmt.Errorf("create report directory: %w", err) + } + data, err := report.Encode() + if err != nil { + return err + } + if err := os.WriteFile(reportPath, data, 0o644); err != nil { + return fmt.Errorf("write assurance report: %w", err) + } + + index.SchemaVersion = assurance.IndexSchema + index.GeneratedAt = report.GeneratedAt + index.Releases = upsertIndexEntry(index.Releases, assurance.IndexEntry{ + Tag: report.Release.Tag, Version: report.Release.Version, + PublishedAt: report.Release.PublishedAt, GeneratedAt: report.GeneratedAt, + Verdict: report.Verdict.Overall, Gates: len(report.Verdict.GatesFailed), + Path: "reports/" + report.Release.Tag + ".json", + }) + if len(index.Releases) > 0 { + index.Latest = index.Releases[0].Tag + } + indexData, err := index.Encode() + if err != nil { + return err + } + if err := os.WriteFile(indexPath, indexData, 0o644); err != nil { + return fmt.Errorf("write assurance index: %w", err) + } + + markdown := assurance.RenderMarkdown(report, assurance.MarkdownOptions{ + IncludeChecks: true, IncludeTrends: true, + }) + fmt.Print(markdown) + if *summaryOut != "" { + if err := os.WriteFile(*summaryOut, []byte(markdown), 0o644); err != nil { + return fmt.Errorf("write markdown summary: %w", err) + } + } + if *stepSummary { + if err := appendStepSummary(markdown); err != nil { + return err + } + } + fmt.Fprintf(os.Stderr, "wrote %s and %s\n", reportPath, indexPath) + if previousTag != "" { + fmt.Fprintf(os.Stderr, "compared against %s\n", previousTag) + } + if len(report.Unknown) > 0 && !*allowUnknown { + return blockingError{ + message: fmt.Sprintf("%d reported check(s) are not declared in the catalog", len(report.Unknown)), + code: 3, + } + } + if *failOnBlocking && report.Verdict.Blocking() { + return blockingError{ + message: "the release did not pass every gate check: " + strings.Join(blockers(report.Verdict), ", "), + code: 1, + } + } + return nil +} + +func resolvePrevious(flagValue, tag, targetDir string, index assurance.Index) (*assurance.Report, string, error) { + switch flagValue { + case "none": + return nil, "", nil + case "": + previousTag := previousRelease(tag, index) + if previousTag == "" { + return nil, "", nil + } + path := filepath.Join(targetDir, "reports", previousTag+".json") + report, err := assurance.LoadReport(path) + if err != nil { + // A missing previous report is not an error: the first release has + // none, and an older one may predate the framework. + return nil, "", nil + } + return &report, previousTag, nil + default: + report, err := assurance.LoadReport(flagValue) + if err != nil { + return nil, "", err + } + return &report, report.Release.Tag, nil + } +} + +// previousRelease returns the highest released tag below the current one. +func previousRelease(tag string, index assurance.Index) string { + current, err := semver.NewVersion(strings.TrimPrefix(tag, "v")) + if err != nil { + return "" + } + best := "" + var bestVersion *semver.Version + for _, entry := range index.Releases { + candidate, parseErr := semver.NewVersion(strings.TrimPrefix(entry.Tag, "v")) + if parseErr != nil || !candidate.LessThan(current) { + continue + } + if bestVersion == nil || candidate.GreaterThan(bestVersion) { + bestVersion = candidate + best = entry.Tag + } + } + return best +} + +func upsertIndexEntry(entries []assurance.IndexEntry, entry assurance.IndexEntry) []assurance.IndexEntry { + replaced := false + for index, existing := range entries { + if existing.Tag == entry.Tag { + entries[index] = entry + replaced = true + break + } + } + if !replaced { + entries = append(entries, entry) + } + sort.SliceStable(entries, func(i, j int) bool { + left, leftErr := semver.NewVersion(strings.TrimPrefix(entries[i].Tag, "v")) + right, rightErr := semver.NewVersion(strings.TrimPrefix(entries[j].Tag, "v")) + if leftErr != nil || rightErr != nil { + return entries[i].Tag > entries[j].Tag + } + return left.GreaterThan(right) + }) + return entries +} + +// -------------------------------------------------------- catalog-validate --- + +func runCatalogValidate(args []string) error { + flags := flag.NewFlagSet("catalog-validate", flag.ExitOnError) + catalogPath := flags.String("catalog", "", "assurance catalog path") + checkID := flags.String("check", "", "print one check") + evidenceID := flags.String("evidence", "", "print one evidence claim") + skipArtifacts := flags.Bool("skip-artifacts", false, "skip repository artifact hash verification") + refresh := flags.Bool("refresh", false, "rewrite recorded checksums from the files they name") + if err := flags.Parse(args); err != nil { + return err + } + catalog, catalogFile, err := mustLoadCatalog(*catalogPath) + if err != nil { + return err + } + if *refresh { + resolvedRoot, rootErr := repositoryRoot() + if rootErr != nil { + return rootErr + } + changed, refreshErr := catalog.RefreshArtifacts(resolvedRoot) + if refreshErr != nil { + return refreshErr + } + if changed > 0 { + data, encodeErr := catalog.Encode() + if encodeErr != nil { + return encodeErr + } + if err := os.WriteFile(catalogFile, data, 0o644); err != nil { + return fmt.Errorf("write assurance catalog: %w", err) + } + } + fmt.Printf("Refreshed %d recorded checksum(s) in %s.\n", changed, relativeToWorkingDir(catalogFile)) + return nil + } + if !*skipArtifacts { + resolvedRoot, rootErr := repositoryRoot() + if rootErr != nil { + return rootErr + } + if err := catalog.VerifyArtifacts(resolvedRoot); err != nil { + return err + } + } + fmt.Printf("Validated %s: %d areas, %d checks, %d evidence claims.\n", + relativeToWorkingDir(catalogFile), len(catalog.Areas), len(catalog.Checks), len(catalog.Evidence)) + + if *checkID != "" || *evidenceID != "" { + if *checkID != "" { + check, found := catalog.Check(*checkID) + if !found { + return fmt.Errorf("unknown check %q", *checkID) + } + printCheck(catalog, check) + } + if *evidenceID != "" { + for _, evidence := range catalog.Evidence { + if evidence.ID == *evidenceID { + printEvidence(evidence) + return nil + } + } + return fmt.Errorf("unknown evidence claim %q", *evidenceID) + } + return nil + } + for _, stage := range assurance.Stages() { + checks := catalog.ChecksForStage(stage) + if len(checks) == 0 { + continue + } + fmt.Printf("\n%s (%d checks)\n", stage.Title(), len(checks)) + for _, check := range checks { + instances := "" + if len(check.ExpectedInstances) > 0 { + instances = fmt.Sprintf(", %d instances", len(check.ExpectedInstances)) + } + fmt.Printf(" %-24s %-9s %s%s\n", check.ID, check.Level, check.Title, instances) + } + } + fmt.Printf("\nEvidence (%d claims)\n", len(catalog.Evidence)) + for _, evidence := range catalog.Evidence { + fmt.Printf(" %-26s %-16s backed by %s\n", evidence.ID, evidence.EvidenceLevel, evidence.CheckID) + } + return nil +} + +func printCheck(catalog assurance.Catalog, check assurance.Check) { + fmt.Printf("\n%s — %s\n", check.ID, check.Title) + fmt.Printf(" Area: %s (%s); stage: %s; level: %s\n", + check.Area, catalog.AreaTitle(check.Area), check.Stage, check.Level) + fmt.Printf(" Source: %s job %s\n", check.Source.Workflow, check.Source.Job) + for _, instance := range check.ExpectedInstances { + fmt.Printf(" Instance: %s\n", instance.Name) + } + for _, command := range check.Reproduce { + fmt.Printf(" Reproduce: %s\n", shellCommand(command)) + } + for _, claim := range check.Proves { + fmt.Printf(" Proves: %s\n", claim) + } + for _, limitation := range check.Limitations { + fmt.Printf(" Limitation: %s\n", limitation) + } +} + +func printEvidence(evidence assurance.Evidence) { + fmt.Printf("\n%s — %s\n", evidence.ID, evidence.Title) + fmt.Printf(" Area: %s; evidence: %s; backed by check %s\n", + evidence.Area, evidence.EvidenceLevel, evidence.CheckID) + for _, input := range evidence.Inputs { + fmt.Printf(" Input: %s %s\n", input.Kind, input.Location) + } + for _, command := range evidence.Reproduce { + fmt.Printf(" Reproduce: %s\n", shellCommand(command)) + } + for _, claim := range evidence.Proves { + fmt.Printf(" Proves: %s\n", claim) + } + for _, limitation := range evidence.Limitations { + fmt.Printf(" Limitation: %s\n", limitation) + } +} + +func relativeToWorkingDir(path string) string { + working, err := os.Getwd() + if err != nil { + return path + } + relative, err := filepath.Rel(working, path) + if err != nil { + return path + } + return relative +} + +func shellCommand(command []string) string { + quoted := make([]string, len(command)) + for index, argument := range command { + if argument != "" && strings.IndexFunc(argument, func(r rune) bool { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + return false + } + return !strings.ContainsRune("@%_+=:,./-", r) + }) == -1 { + quoted[index] = argument + continue + } + quoted[index] = "'" + strings.ReplaceAll(argument, "'", "'\"'\"'") + "'" + } + return strings.Join(quoted, " ") +} diff --git a/internal/assurance/contract.go b/internal/assurance/contract.go new file mode 100644 index 00000000..c96e813b --- /dev/null +++ b/internal/assurance/contract.go @@ -0,0 +1,348 @@ +// Package assurance implements Bomly's release assurance framework: the shared +// check-result contract every quality check emits, the declarative catalog that +// declares which checks must exist, and the report the public assurance page +// renders for each release. +package assurance + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// CheckSchema is the schema identifier every check result carries. +const CheckSchema = "bomly.assurance-check/v1" + +// MaxResultBytes bounds a single check-result document. +const MaxResultBytes = 8 << 20 + +// Stage names the release phase a check belongs to. +type Stage string + +// The three release assurance stages. +const ( + // StagePrerequisites runs on the source tree before a release tag exists. + StagePrerequisites Stage = "prerequisites" + // StagePreRelease runs inside the release pipeline against the draft release. + StagePreRelease Stage = "pre-release" + // StagePostRelease runs after publication against the shipped artifacts. + StagePostRelease Stage = "post-release" +) + +// Stages lists every release assurance stage in execution order. +func Stages() []Stage { + return []Stage{StagePrerequisites, StagePreRelease, StagePostRelease} +} + +// Valid reports whether the stage is one of the three known stages. +func (s Stage) Valid() bool { + switch s { + case StagePrerequisites, StagePreRelease, StagePostRelease: + return true + default: + return false + } +} + +// Title returns the human-readable stage name used in reports. +func (s Stage) Title() string { + switch s { + case StagePrerequisites: + return "Release prerequisites" + case StagePreRelease: + return "Final pre-release checks" + case StagePostRelease: + return "Post-release assessment" + default: + return string(s) + } +} + +// Level distinguishes checks that block a release from advisory observations. +type Level string + +// Check enforcement levels. +const ( + // LevelGate blocks the stage when it does not pass. + LevelGate Level = "gate" + // LevelAdvisory is reported but never blocks a release. + LevelAdvisory Level = "advisory" +) + +// Valid reports whether the level is known. +func (l Level) Valid() bool { return l == LevelGate || l == LevelAdvisory } + +// Status is the outcome of a check or check instance. +type Status string + +// Check outcomes, ordered from best to worst by Severity. +const ( + // StatusPass means the check completed and every expectation held. + StatusPass Status = "pass" + // StatusSkip means the check did not run for a declared reason. + StatusSkip Status = "skip" + // StatusMissing means no result was reported for a declared check. + StatusMissing Status = "missing" + // StatusDegraded means the check completed with reduced confidence. + StatusDegraded Status = "degraded" + // StatusFail means the check did not hold. + StatusFail Status = "fail" +) + +// Valid reports whether the status is one a check result may carry. +func (s Status) Valid() bool { + switch s { + case StatusPass, StatusSkip, StatusMissing, StatusDegraded, StatusFail: + return true + default: + return false + } +} + +// Severity ranks statuses so the worst instance decides a merged check. +func (s Status) Severity() int { + switch s { + case StatusPass: + return 0 + case StatusSkip: + return 1 + case StatusMissing: + return 2 + case StatusDegraded: + return 3 + case StatusFail: + return 4 + default: + return 5 + } +} + +// Worse returns whichever status is more severe. +func Worse(a, b Status) Status { + if b.Severity() > a.Severity() { + return b + } + return a +} + +// Runner records the machine a check instance executed on. +type Runner struct { + OS string `json:"os,omitempty"` + Arch string `json:"arch,omitempty"` + GoVersion string `json:"go_version,omitempty"` +} + +// Detail is one named sub-result inside a check instance, such as a single +// smoke test, fuzz target, or cross-build target. +type Detail struct { + Name string `json:"name"` + Status Status `json:"status"` + Note string `json:"note,omitempty"` + DurationMS float64 `json:"duration_ms,omitempty"` +} + +// Artifact records a file a check produced or verified. +type Artifact struct { + Name string `json:"name"` + SHA256 string `json:"sha256,omitempty"` + Bytes int64 `json:"bytes,omitempty"` +} + +// Link is a labelled URL shown alongside a check. +type Link struct { + Label string `json:"label"` + URL string `json:"url"` +} + +// CheckResult is what a single check instance writes when it finishes. One +// file per instance; instances sharing an ID merge into one reported check. +type CheckResult struct { + SchemaVersion string `json:"schema_version"` + ID string `json:"id"` + Instance string `json:"instance,omitempty"` + Stage Stage `json:"stage"` + Level Level `json:"level,omitempty"` + Status Status `json:"status"` + StartedAt string `json:"started_at,omitempty"` + FinishedAt string `json:"finished_at,omitempty"` + DurationMS float64 `json:"duration_ms,omitempty"` + Ref string `json:"ref,omitempty"` + Tag string `json:"tag,omitempty"` + Commit string `json:"commit,omitempty"` + Version string `json:"version,omitempty"` + RunURL string `json:"run_url,omitempty"` + Job string `json:"job,omitempty"` + Runner Runner `json:"runner,omitempty"` + Summary string `json:"summary"` + Metrics map[string]float64 `json:"metrics,omitempty"` + Details []Detail `json:"details,omitempty"` + Artifacts []Artifact `json:"artifacts,omitempty"` + Links []Link `json:"links,omitempty"` +} + +var idPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) + +// instancePattern allows the platform and slice names checks report, which +// include dots (ubuntu-24.04) and underscores (linux_amd64). +var instancePattern = regexp.MustCompile(`^[a-zA-Z0-9]+(?:[._-][a-zA-Z0-9]+)*$`) + +// ParseCheckResult decodes and validates one check-result document. +func ParseCheckResult(data []byte) (CheckResult, error) { + if len(data) > MaxResultBytes { + return CheckResult{}, fmt.Errorf("check result is %d bytes, limit is %d", len(data), MaxResultBytes) + } + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + var result CheckResult + if err := decoder.Decode(&result); err != nil { + return CheckResult{}, fmt.Errorf("decode check result: %w", err) + } + if err := ensureEOF(decoder, "check result"); err != nil { + return CheckResult{}, err + } + if err := result.Validate(); err != nil { + return CheckResult{}, err + } + return result, nil +} + +// Validate reports whether the check result satisfies the contract. +func (r CheckResult) Validate() error { + if r.SchemaVersion != CheckSchema { + return fmt.Errorf("unsupported check-result schema %q", r.SchemaVersion) + } + if !idPattern.MatchString(r.ID) { + return fmt.Errorf("invalid check id %q", r.ID) + } + if r.Instance != "" && !instancePattern.MatchString(r.Instance) { + return fmt.Errorf("check %q has invalid instance %q", r.ID, r.Instance) + } + if !r.Stage.Valid() { + return fmt.Errorf("check %q has unsupported stage %q", r.ID, r.Stage) + } + if r.Level != "" && !r.Level.Valid() { + return fmt.Errorf("check %q has unsupported level %q", r.ID, r.Level) + } + if !r.Status.Valid() || r.Status == StatusMissing { + return fmt.Errorf("check %q has unsupported status %q", r.ID, r.Status) + } + if strings.TrimSpace(r.Summary) == "" { + return fmt.Errorf("check %q requires a summary", r.ID) + } + for index, detail := range r.Details { + if strings.TrimSpace(detail.Name) == "" { + return fmt.Errorf("check %q detail %d requires a name", r.ID, index+1) + } + if !detail.Status.Valid() { + return fmt.Errorf("check %q detail %q has unsupported status %q", r.ID, detail.Name, detail.Status) + } + } + for index, artifact := range r.Artifacts { + if strings.TrimSpace(artifact.Name) == "" { + return fmt.Errorf("check %q artifact %d requires a name", r.ID, index+1) + } + } + for index, link := range r.Links { + if strings.TrimSpace(link.Label) == "" || strings.TrimSpace(link.URL) == "" { + return fmt.Errorf("check %q link %d requires a label and url", r.ID, index+1) + } + } + return nil +} + +// Key identifies a check instance uniquely within one run. +func (r CheckResult) Key() string { + if r.Instance == "" { + return r.ID + } + return r.ID + "." + r.Instance +} + +// FileName is the on-disk name a check result is written under. +func (r CheckResult) FileName() string { return r.Key() + ".json" } + +// Encode renders the check result as indented JSON with a trailing newline. +func (r CheckResult) Encode() ([]byte, error) { + data, err := json.MarshalIndent(r, "", " ") + if err != nil { + return nil, fmt.Errorf("encode check result: %w", err) + } + return append(data, '\n'), nil +} + +// LoadResults reads every check-result document under dir, recursively, so a +// directory of merged CI artifacts can be consumed directly. Files that are not +// check results are ignored; malformed check results are an error. +func LoadResults(dir string) ([]CheckResult, error) { + var results []CheckResult + walkErr := filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + return nil + } + data, readErr := readBounded(path, MaxResultBytes) + if readErr != nil { + return readErr + } + if !looksLikeCheckResult(data) { + return nil + } + result, parseErr := ParseCheckResult(data) + if parseErr != nil { + return fmt.Errorf("%s: %w", path, parseErr) + } + results = append(results, result) + return nil + }) + if walkErr != nil { + return nil, fmt.Errorf("load check results from %s: %w", dir, walkErr) + } + sort.Slice(results, func(i, j int) bool { return results[i].Key() < results[j].Key() }) + return results, nil +} + +func looksLikeCheckResult(data []byte) bool { + var probe struct { + SchemaVersion string `json:"schema_version"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return false + } + return probe.SchemaVersion == CheckSchema +} + +func readBounded(path string, limit int64) ([]byte, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open %s: %w", path, err) + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, limit+1)) + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("read %s: file exceeds %d bytes", path, limit) + } + return data, nil +} + +func ensureEOF(decoder *json.Decoder, what string) error { + var extra any + err := decoder.Decode(&extra) + if errors.Is(err, io.EOF) { + return nil + } + if err == nil { + return fmt.Errorf("decode %s: multiple JSON values", what) + } + return fmt.Errorf("decode %s trailing data: %w", what, err) +} diff --git a/internal/assurance/convert.go b/internal/assurance/convert.go new file mode 100644 index 00000000..b26696ab --- /dev/null +++ b/internal/assurance/convert.go @@ -0,0 +1,220 @@ +package assurance + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" +) + +// BenchmarkRunSchema is the schema of the performance sample manifest produced +// by internal/assurance/perfrun. +const BenchmarkRunSchema = "bomly.benchmark-run/v1" + +// SBOMAssuranceSchema is the schema of the SBOM interoperability run manifest +// produced by internal/assurance/sbominterop. +const SBOMAssuranceSchema = "bomly.sbom-assurance-run/v1" + +// maxManifestBytes bounds a converted manifest document. +const maxManifestBytes = 32 << 20 + +type benchmarkManifest struct { + SchemaVersion string `json:"schema_version"` + Case struct { + Name string `json:"name"` + SamplesPerMode int `json:"samples_per_mode"` + NetworkState string `json:"network_state"` + } `json:"case"` + Summaries []struct { + Mode string `json:"mode"` + Samples int `json:"samples"` + MedianMS float64 `json:"median_ms"` + MeanMS float64 `json:"mean_ms"` + ConfidenceInterval95 [2]float64 `json:"confidence_interval_95_ms"` + PeakMemoryBytes uint64 `json:"peak_memory_bytes"` + } `json:"summaries"` + Gates struct { + Passed bool `json:"passed"` + AllExitCodesZero bool `json:"all_exit_codes_zero"` + NormalizedOutputStable bool `json:"normalized_output_stable"` + OutputCapPassed bool `json:"output_cap_passed"` + FailureReason string `json:"failure_reason"` + } `json:"gates"` +} + +// ConvertBenchmarkRun turns a performance sample manifest into a check result. +func ConvertBenchmarkRun(data []byte, base CheckResult) (CheckResult, error) { + var manifest benchmarkManifest + if err := decodeManifest(data, &manifest, "performance manifest"); err != nil { + return CheckResult{}, err + } + if manifest.SchemaVersion != BenchmarkRunSchema { + return CheckResult{}, fmt.Errorf("unsupported performance manifest schema %q", manifest.SchemaVersion) + } + result := base + result.SchemaVersion = CheckSchema + result.Status = StatusPass + if !manifest.Gates.Passed { + result.Status = StatusFail + } + result.Metrics = map[string]float64{"samples_per_mode": float64(manifest.Case.SamplesPerMode)} + var peak float64 + for _, summary := range manifest.Summaries { + mode := strings.ToLower(summary.Mode) + result.Metrics[mode+"_median_ms"] = round(summary.MedianMS) + result.Metrics[mode+"_mean_ms"] = round(summary.MeanMS) + result.Metrics[mode+"_ci95_upper_ms"] = round(summary.ConfidenceInterval95[1]) + if float64(summary.PeakMemoryBytes) > peak { + peak = float64(summary.PeakMemoryBytes) + } + result.Details = append(result.Details, Detail{ + Name: mode + " cache", + Status: StatusPass, + Note: fmt.Sprintf("%d samples, median %.0f ms, mean %.0f ms", + summary.Samples, summary.MedianMS, summary.MeanMS), + DurationMS: round(summary.MedianMS), + }) + } + if peak > 0 { + result.Metrics["peak_memory_bytes"] = peak + } + switch { + case manifest.Gates.FailureReason != "": + result.Summary = fmt.Sprintf("Performance sampling for %s failed: %s.", + manifest.Case.Name, manifest.Gates.FailureReason) + case len(manifest.Summaries) == 0: + result.Status = StatusFail + result.Summary = "The performance run recorded no samples." + default: + result.Summary = fmt.Sprintf("%s completed %d samples per cache mode with identical normalized output (%s).", + manifest.Case.Name, manifest.Case.SamplesPerMode, describeModes(result.Metrics)) + } + return result, nil +} + +func describeModes(metrics map[string]float64) string { + var parts []string + for _, mode := range []string{"cold", "warm"} { + if value, ok := metrics[mode+"_median_ms"]; ok { + parts = append(parts, fmt.Sprintf("%s median %.0f ms", mode, value)) + } + } + if len(parts) == 0 { + return "no timings recorded" + } + return strings.Join(parts, ", ") +} + +type sbomAssuranceManifest struct { + SchemaVersion string `json:"schema_version"` + Validators []struct { + Name string `json:"name"` + Version string `json:"version"` + SHA256 string `json:"sha256"` + } `json:"validators"` + Artifacts []struct { + Format string `json:"format"` + Path string `json:"path"` + SHA256 string `json:"sha256"` + Bytes int64 `json:"bytes"` + } `json:"artifacts"` + Commands []struct { + Executable string `json:"executable"` + Args []string `json:"args"` + ExitCode int `json:"exit_code"` + DurationMS int64 `json:"duration_ms"` + } `json:"commands"` + Failure string `json:"failure,omitempty"` +} + +// ConvertSBOMAssurance turns an SBOM interoperability run manifest into a +// check result. +func ConvertSBOMAssurance(data []byte, base CheckResult) (CheckResult, error) { + var manifest sbomAssuranceManifest + if err := decodeManifest(data, &manifest, "SBOM assurance manifest"); err != nil { + return CheckResult{}, err + } + if manifest.SchemaVersion != SBOMAssuranceSchema { + return CheckResult{}, fmt.Errorf("unsupported SBOM assurance manifest schema %q", manifest.SchemaVersion) + } + result := base + result.SchemaVersion = CheckSchema + result.Status = StatusPass + failures := 0 + for _, command := range manifest.Commands { + status := StatusPass + if command.ExitCode != 0 { + status = StatusFail + failures++ + } + result.Details = append(result.Details, Detail{ + Name: strings.TrimSpace(filepath.Base(command.Executable) + " " + firstArgument(command.Args)), + Status: status, + Note: fmt.Sprintf("exit code %d", command.ExitCode), + DurationMS: float64(command.DurationMS), + }) + } + for _, validator := range manifest.Validators { + result.Details = append(result.Details, Detail{ + Name: validator.Name + " " + validator.Version, + Status: StatusPass, + Note: "checksum " + shortHash(validator.SHA256), + }) + } + for _, artifact := range manifest.Artifacts { + result.Artifacts = append(result.Artifacts, Artifact{ + Name: artifact.Format, SHA256: artifact.SHA256, Bytes: artifact.Bytes, + }) + } + result.Metrics = map[string]float64{ + "validators": float64(len(manifest.Validators)), + "documents": float64(len(manifest.Artifacts)), + } + switch { + case manifest.Failure != "": + result.Status = StatusFail + result.Summary = "SBOM interoperability run failed: " + manifest.Failure + "." + case failures > 0: + result.Status = StatusFail + result.Summary = fmt.Sprintf("%d of %d validator commands failed.", failures, len(manifest.Commands)) + case len(manifest.Artifacts) == 0: + result.Status = StatusFail + result.Summary = "The run produced no SBOM documents to validate." + default: + names := make([]string, 0, len(manifest.Validators)) + for _, validator := range manifest.Validators { + names = append(names, validator.Name+" "+validator.Version) + } + result.Summary = fmt.Sprintf("%d generated SBOM documents passed %s.", + len(manifest.Artifacts), strings.Join(names, " and ")) + } + return result, nil +} + +func firstArgument(args []string) string { + if len(args) == 0 { + return "" + } + return args[0] +} + +func shortHash(hash string) string { + if len(hash) <= 12 { + return hash + } + return hash[:12] +} + +func round(value float64) float64 { + return float64(int64(value*100+0.5)) / 100 +} + +func decodeManifest(data []byte, target any, what string) error { + if len(data) > maxManifestBytes { + return fmt.Errorf("%s is %d bytes, limit is %d", what, len(data), maxManifestBytes) + } + if err := json.Unmarshal(data, target); err != nil { + return fmt.Errorf("decode %s: %w", what, err) + } + return nil +} diff --git a/internal/assurance/convert_test.go b/internal/assurance/convert_test.go new file mode 100644 index 00000000..1900b8e2 --- /dev/null +++ b/internal/assurance/convert_test.go @@ -0,0 +1,191 @@ +package assurance + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func baseFor(id string, stage Stage, level Level) CheckResult { + return CheckResult{SchemaVersion: CheckSchema, ID: id, Stage: stage, Level: level} +} + +func TestConvertBenchmarkRun(t *testing.T) { + manifest := map[string]any{ + "schema_version": BenchmarkRunSchema, + "case": map[string]any{"name": "canonical-sbom-scan", "samples_per_mode": 5}, + "summaries": []map[string]any{ + {"mode": "cold", "samples": 5, "median_ms": 412.4, "mean_ms": 430.0, + "confidence_interval_95_ms": []float64{400, 460}, "peak_memory_bytes": 91234304}, + {"mode": "warm", "samples": 5, "median_ms": 288.1, "mean_ms": 291.0, + "confidence_interval_95_ms": []float64{280, 300}, "peak_memory_bytes": 80234304}, + }, + "gates": map[string]any{"passed": true}, + } + data, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal: %v", err) + } + result, err := ConvertBenchmarkRun(data, baseFor("perf-samples", StagePostRelease, LevelAdvisory)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if err := result.Validate(); err != nil { + t.Fatalf("converted result invalid: %v", err) + } + if result.Status != StatusPass { + t.Fatalf("status = %s, want pass", result.Status) + } + if result.Metrics["cold_median_ms"] != 412.4 || result.Metrics["warm_median_ms"] != 288.1 { + t.Fatalf("metrics = %v", result.Metrics) + } + if result.Metrics["peak_memory_bytes"] != 91234304 { + t.Fatalf("peak memory = %v", result.Metrics["peak_memory_bytes"]) + } + if len(result.Details) != 2 { + t.Fatalf("details = %+v", result.Details) + } +} + +func TestConvertBenchmarkRunFailedGate(t *testing.T) { + data := []byte(`{"schema_version":"bomly.benchmark-run/v1","case":{"name":"c","samples_per_mode":5}, + "summaries":[],"gates":{"passed":false,"failure_reason":"normalized output changed between samples"}}`) + result, err := ConvertBenchmarkRun(data, baseFor("perf-samples", StagePostRelease, LevelAdvisory)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if result.Status != StatusFail { + t.Fatalf("status = %s, want fail", result.Status) + } + if !strings.Contains(result.Summary, "normalized output changed") { + t.Fatalf("summary = %q", result.Summary) + } +} + +func TestConvertSBOMAssurance(t *testing.T) { + data := []byte(`{"schema_version":"bomly.sbom-assurance-run/v1", + "validators":[{"name":"spdx-tools-java","version":"2.0.7","sha256":"2dc63c3399c5178058b1be8a3de6f13b9f24981cd86c4292ef98f4a7e90de36d"}, + {"name":"cyclonedx-cli","version":"0.32.0","sha256":"454879e6a4a405c8a13bff49b8982adcb0596f3019b26b0811c66e4d7f0783e1"}], + "artifacts":[{"format":"spdx-2.3-json","path":"a.json","sha256":"aa","bytes":120}, + {"format":"cyclonedx-1.7-json","path":"b.json","sha256":"bb","bytes":140}], + "commands":[{"executable":"/usr/bin/java","args":["-jar","tools.jar","Verify"],"exit_code":0,"duration_ms":900}]}`) + result, err := ConvertSBOMAssurance(data, baseFor("sbom-interoperability", StagePostRelease, LevelGate)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if err := result.Validate(); err != nil { + t.Fatalf("converted result invalid: %v", err) + } + if result.Status != StatusPass { + t.Fatalf("status = %s, want pass", result.Status) + } + if len(result.Artifacts) != 2 { + t.Fatalf("artifacts = %+v", result.Artifacts) + } + if !strings.Contains(result.Summary, "spdx-tools-java 2.0.7") { + t.Fatalf("summary = %q", result.Summary) + } +} + +func TestConvertSBOMAssuranceRecordsFailure(t *testing.T) { + data := []byte(`{"schema_version":"bomly.sbom-assurance-run/v1","validators":[],"artifacts":[], + "commands":[{"executable":"cyclonedx-cli","args":["validate"],"exit_code":1,"duration_ms":10}], + "failure":"validator cyclonedx-cli failed with exit 1"}`) + result, err := ConvertSBOMAssurance(data, baseFor("sbom-interoperability", StagePostRelease, LevelGate)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if result.Status != StatusFail || !strings.Contains(result.Summary, "validator cyclonedx-cli failed") { + t.Fatalf("result = %+v", result) + } +} + +func TestConvertRejectsWrongSchema(t *testing.T) { + if _, err := ConvertBenchmarkRun([]byte(`{"schema_version":"other/v1"}`), CheckResult{}); err == nil { + t.Fatal("expected the wrong schema to be rejected") + } + if _, err := ConvertSBOMAssurance([]byte(`{"schema_version":"other/v1"}`), CheckResult{}); err == nil { + t.Fatal("expected the wrong schema to be rejected") + } +} + +func TestExpectedAssetsCoverEveryPlatform(t *testing.T) { + assets := ExpectedAssets("0.23.0") + if len(assets) != 23 { + t.Fatalf("expected 23 release assets, got %d: %v", len(assets), assets) + } + for _, required := range []string{ + "SHA256SUMS", "SHA256SUMS.sigstore.json", "multiple.intoto.jsonl", + "bomly_0.23.0_linux_amd64.tar.gz", "bomly-lite_0.23.0_windows_arm64.zip", + "bomly_0.23.0_linux_arm64.pkg.tar.zst", + } { + if !contains(assets, required) { + t.Fatalf("expected asset list to contain %s", required) + } + } +} + +func contains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func TestParseSHA256SUMS(t *testing.T) { + entries, err := ParseSHA256SUMS([]byte( + "aa2c19f8a17ad4c65c6b6a41c9de9b8bbd9d3d3b1f1d6b0b5b0a67f6d4f1a2b3 bomly_1.0.0_linux_amd64.tar.gz\n" + + "bb2c19f8a17ad4c65c6b6a41c9de9b8bbd9d3d3b1f1d6b0b5b0a67f6d4f1a2b3 *bomly_1.0.0_windows_amd64.zip\n")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(entries) != 2 { + t.Fatalf("entries = %v", entries) + } + for _, broken := range []string{ + "not-a-hash file.tar.gz\n", + "aa2c19f8a17ad4c65c6b6a41c9de9b8bbd9d3d3b1f1d6b0b5b0a67f6d4f1a2b3 ../escape.tar.gz\n", + "onlyonefield\n", + "", + } { + if _, err := ParseSHA256SUMS([]byte(broken)); err == nil { + t.Fatalf("expected %q to be rejected", broken) + } + } +} + +func TestVerifyChecksumsDetectsProblems(t *testing.T) { + dir := t.TempDir() + payload := []byte("binary\n") + if err := os.WriteFile(filepath.Join(dir, "bomly_1.0.0_linux_amd64.tar.gz"), payload, 0o644); err != nil { + t.Fatalf("write asset: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "stray.txt"), payload, 0o644); err != nil { + t.Fatalf("write stray: %v", err) + } + entries := ChecksumEntries{ + "bomly_1.0.0_linux_amd64.tar.gz": "1111111111111111111111111111111111111111111111111111111111111111", + "bomly_1.0.0_darwin_arm64.tar.gz": "2222222222222222222222222222222222222222222222222222222222222222", + } + outcome, err := VerifyChecksums(dir, entries) + if err != nil { + t.Fatalf("verify: %v", err) + } + if len(outcome.Mismatched) != 1 || len(outcome.Unlisted) != 1 || len(outcome.NotDownloaded) != 1 { + t.Fatalf("outcome = %+v", outcome) + } +} + +func TestArchiveNameUsesZipOnWindows(t *testing.T) { + if got := ArchiveName("bomly", "1.2.3", "windows", "arm64"); got != "bomly_1.2.3_windows_arm64.zip" { + t.Fatalf("archive name = %s", got) + } + if got := ArchiveName("bomly-lite", "1.2.3", runtime.GOOS, "amd64"); !strings.Contains(got, "1.2.3") { + t.Fatalf("archive name = %s", got) + } +} diff --git a/internal/assurance/gotest.go b/internal/assurance/gotest.go new file mode 100644 index 00000000..4ddbb3cf --- /dev/null +++ b/internal/assurance/gotest.go @@ -0,0 +1,230 @@ +package assurance + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "sort" + "strings" +) + +// MaxGoTestDetails caps how many individual test outcomes a check result +// records, so one slice with thousands of subtests cannot bloat the report. +const MaxGoTestDetails = 200 + +// maxGoTestLine bounds one line of `go test -json` output. +const maxGoTestLine = 1 << 20 + +// GoTestSummary is the outcome of one `go test -json` stream. +type GoTestSummary struct { + Packages int + Total int + Passed int + Failed int + Skipped int + ElapsedSec float64 + Details []Detail + Truncated int + // FailedTests names the tests that failed, in report order. + FailedTests []string + // Anomalies records non-JSON lines, which usually mean a build failure. + Anomalies []string +} + +type goTestEvent struct { + Action string `json:"Action"` + Package string `json:"Package"` + Test string `json:"Test"` + Elapsed float64 `json:"Elapsed"` + Output string `json:"Output"` +} + +// ParseGoTestEvents reads a `go test -json` stream and summarises it. Lines that +// are not JSON events (build errors, toolchain notices) are recorded as +// anomalies instead of failing the parse. When echo is non-nil, test output is +// replayed to it so CI logs stay readable. +func ParseGoTestEvents(reader io.Reader, echo io.Writer) (GoTestSummary, error) { + summary := GoTestSummary{} + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 64*1024), maxGoTestLine) + + type outcome struct { + status Status + elapsed float64 + } + tests := map[string]outcome{} + packages := map[string]struct{}{} + + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + var event goTestEvent + if err := json.Unmarshal(line, &event); err != nil || event.Action == "" { + text := strings.TrimSpace(string(line)) + if text != "" && len(summary.Anomalies) < 20 { + summary.Anomalies = append(summary.Anomalies, truncate(text, 300)) + } + if echo != nil { + fmt.Fprintln(echo, text) + } + continue + } + if event.Package != "" { + packages[event.Package] = struct{}{} + } + if event.Action == "output" { + if echo != nil { + fmt.Fprint(echo, event.Output) + } + continue + } + if event.Test == "" { + if event.Action == "fail" && event.Elapsed > summary.ElapsedSec { + summary.ElapsedSec = event.Elapsed + } + if event.Action == "pass" && event.Elapsed > summary.ElapsedSec { + summary.ElapsedSec = event.Elapsed + } + continue + } + name := event.Test + if event.Package != "" { + name = shortPackage(event.Package) + "." + event.Test + } + switch event.Action { + case "pass": + tests[name] = outcome{status: StatusPass, elapsed: event.Elapsed} + case "fail": + tests[name] = outcome{status: StatusFail, elapsed: event.Elapsed} + case "skip": + tests[name] = outcome{status: StatusSkip, elapsed: event.Elapsed} + } + } + if err := scanner.Err(); err != nil { + return summary, fmt.Errorf("read go test output: %w", err) + } + + summary.Packages = len(packages) + names := make([]string, 0, len(tests)) + for name := range tests { + names = append(names, name) + } + sort.Strings(names) + + var reportable []string + for _, name := range names { + result := tests[name] + summary.Total++ + switch result.status { + case StatusPass: + summary.Passed++ + case StatusFail: + summary.Failed++ + summary.FailedTests = append(summary.FailedTests, name) + case StatusSkip: + summary.Skipped++ + } + if result.status == StatusFail || testDepth(name) <= 2 { + reportable = append(reportable, name) + } + } + for _, name := range reportable { + if len(summary.Details) >= MaxGoTestDetails { + summary.Truncated = len(reportable) - len(summary.Details) + break + } + result := tests[name] + summary.Details = append(summary.Details, Detail{ + Name: name, Status: result.status, DurationMS: result.elapsed * 1000, + }) + } + return summary, nil +} + +// testDepth counts the subtest levels in a `Package.TestName/sub/case` label. +func testDepth(name string) int { + if index := strings.Index(name, "."); index >= 0 { + name = name[index+1:] + } + return strings.Count(name, "/") + 1 +} + +func shortPackage(pkg string) string { + parts := strings.Split(pkg, "/") + if len(parts) == 0 { + return pkg + } + return parts[len(parts)-1] +} + +func truncate(value string, limit int) string { + if len(value) <= limit { + return value + } + return value[:limit] + "…" +} + +// ToCheckResult turns a `go test -json` summary into a check result. exitCode is +// the test command's exit status, which catches build failures that produce no +// test events at all. +func (s GoTestSummary) ToCheckResult(base CheckResult, exitCode int) CheckResult { + result := base + result.SchemaVersion = CheckSchema + result.Status = StatusPass + switch { + case s.Failed > 0 || exitCode != 0: + result.Status = StatusFail + case s.Total == 0: + // A test command that ran cleanly but executed nothing proves nothing: + // usually a -run pattern that no longer matches. Reporting that as a + // skip would let it slide past a gate, so it fails instead. + result.Status = StatusFail + } + result.Metrics = map[string]float64{ + "tests_total": float64(s.Total), + "tests_passed": float64(s.Passed), + "tests_failed": float64(s.Failed), + "tests_skipped": float64(s.Skipped), + "packages": float64(s.Packages), + } + result.Details = s.Details + if result.DurationMS == 0 && s.ElapsedSec > 0 { + result.DurationMS = s.ElapsedSec * 1000 + } + result.Summary = s.summaryLine(exitCode) + return result +} + +func (s GoTestSummary) summaryLine(exitCode int) string { + if s.Total == 0 { + detail := "" + if len(s.Anomalies) > 0 { + detail = " " + s.Anomalies[0] + } + if exitCode != 0 { + return fmt.Sprintf("No tests ran and the command exited with code %d.%s", exitCode, detail) + } + return "No tests ran, so this check proved nothing. The test selection most likely no longer matches any test." + detail + } + parts := []string{fmt.Sprintf("%d of %d tests passed", s.Passed, s.Total)} + if s.Failed > 0 { + failed := s.FailedTests + if len(failed) > 3 { + failed = failed[:3] + } + parts = append(parts, fmt.Sprintf("%d failed (%s)", s.Failed, strings.Join(failed, ", "))) + } + if s.Skipped > 0 { + parts = append(parts, fmt.Sprintf("%d skipped", s.Skipped)) + } + if s.Failed == 0 && exitCode != 0 { + parts = append(parts, fmt.Sprintf("the command exited with code %d", exitCode)) + } + if s.Truncated > 0 { + parts = append(parts, fmt.Sprintf("%d further results are not listed", s.Truncated)) + } + return strings.Join(parts, ", ") + "." +} diff --git a/internal/assurance/joburl.go b/internal/assurance/joburl.go new file mode 100644 index 00000000..aca6b864 --- /dev/null +++ b/internal/assurance/joburl.go @@ -0,0 +1,52 @@ +package assurance + +import ( + "encoding/json" + "fmt" +) + +// maxJobsPayloadBytes bounds the workflow-jobs response this package parses. +const maxJobsPayloadBytes = 8 << 20 + +type jobsPayload struct { + Jobs []struct { + Name string `json:"name"` + Status string `json:"status"` + RunnerName string `json:"runner_name"` + HTMLURL string `json:"html_url"` + } `json:"jobs"` +} + +// MatchJobURL picks the URL of the job a check is running inside, so a reported +// count links to the exact log that produced it rather than to the whole run. +// +// A runner executes one job at a time, so the running job on this runner is the +// caller's own job — that holds for matrix legs and for jobs contributed by a +// called workflow, which is why the runner name is matched rather than the job +// name (display names differ between direct and nested invocations). Returns an +// empty string when there is no confident match; callers fall back to the run. +func MatchJobURL(payload []byte, runnerName string) (string, error) { + if len(payload) > maxJobsPayloadBytes { + return "", fmt.Errorf("workflow jobs payload is %d bytes, limit is %d", len(payload), maxJobsPayloadBytes) + } + if runnerName == "" { + return "", nil + } + var jobs jobsPayload + if err := json.Unmarshal(payload, &jobs); err != nil { + return "", fmt.Errorf("decode workflow jobs: %w", err) + } + match := "" + for _, job := range jobs.Jobs { + if job.RunnerName != runnerName || job.Status != "in_progress" || job.HTMLURL == "" { + continue + } + if match != "" { + // Two running jobs claiming the same runner should not happen; if + // it does, a wrong link is worse than the run-level one. + return "", nil + } + match = job.HTMLURL + } + return match, nil +} diff --git a/internal/assurance/parse_fuzz_test.go b/internal/assurance/parse_fuzz_test.go new file mode 100644 index 00000000..643a9405 --- /dev/null +++ b/internal/assurance/parse_fuzz_test.go @@ -0,0 +1,130 @@ +package assurance + +import ( + "bytes" + "testing" + + testutil "github.com/bomly-dev/bomly-sdk/testkit" +) + +func FuzzParseCheckResult(f *testing.F) { + for _, seed := range []string{ + "", + "{}", + `{"schema_version":"bomly.assurance-check/v1","id":"smoke","instance":"go","stage":"prerequisites",` + + `"level":"gate","status":"pass","summary":"18 of 18 tests passed.","metrics":{"tests_total":18},` + + `"details":[{"name":"smoke.TestScan","status":"pass","duration_ms":41000}]}`, + `{"schema_version":"bomly.assurance-check/v1","id":"SMOKE","stage":"whenever","status":"green"}`, + `{"schema_version":"bomly.assurance-check/v1","id":"smoke","stage":"prerequisites","status":"pass"`, + `{"schema_version":"bomly.assurance-check/v1","id":"smoke","stage":"prerequisites","status":"pass","summary":"ok"} {}`, + } { + f.Add([]byte(seed)) + } + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > testutil.MaxFuzzInputSize { + return + } + first, firstErr := ParseCheckResult(data) + second, secondErr := ParseCheckResult(data) + if (firstErr == nil) != (secondErr == nil) { + t.Fatalf("ParseCheckResult changed success state: first=%v second=%v", firstErr, secondErr) + } + if firstErr != nil { + if firstErr.Error() != secondErr.Error() { + t.Fatalf("ParseCheckResult changed error: first=%v second=%v", firstErr, secondErr) + } + return + } + if err := first.Validate(); err != nil { + t.Fatalf("accepted an invalid check result: %v", err) + } + encodedFirst, err := first.Encode() + if err != nil { + t.Fatalf("encode accepted result: %v", err) + } + encodedSecond, err := second.Encode() + if err != nil { + t.Fatalf("encode accepted result: %v", err) + } + if !bytes.Equal(encodedFirst, encodedSecond) { + t.Fatal("ParseCheckResult produced different documents for identical input") + } + if _, err := ParseCheckResult(encodedFirst); err != nil { + t.Fatalf("re-parsing an accepted result failed: %v", err) + } + }) +} + +func FuzzParseCatalog(f *testing.F) { + valid := `{"schema_version":"bomly.assurance-catalog/v1",` + + `"areas":[{"id":"end-to-end","title":"End to end","description":"Real runs."}],` + + `"checks":[{"id":"smoke","title":"Smoke","area":"end-to-end","stage":"prerequisites","level":"gate",` + + `"description":"Runs scans.","source":{"workflow":"smoke.yml","job":"smoke"},` + + `"proves":["It scans."],"limitations":["One project."]}],` + + `"evidence":[{"id":"graph-go","title":"Go graph","area":"end-to-end","evidence_level":"release-artifact",` + + `"check_id":"smoke","inputs":[{"kind":"release","location":"https://example.test"}],` + + `"reproduce":[["make","smoke"]],"proves":["It resolves."],"limitations":["One toolchain."]}]}` + for _, seed := range []string{"", "{}", valid, valid[:len(valid)/2], `{"schema_version":"bomly.assurance-catalog/v1","areas":[]}`} { + f.Add([]byte(seed)) + } + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > testutil.MaxFuzzInputSize { + return + } + first, firstErr := ParseCatalog(data) + _, secondErr := ParseCatalog(data) + if (firstErr == nil) != (secondErr == nil) { + t.Fatalf("ParseCatalog changed success state: first=%v second=%v", firstErr, secondErr) + } + if firstErr != nil { + if firstErr.Error() != secondErr.Error() { + t.Fatalf("ParseCatalog changed error: first=%v second=%v", firstErr, secondErr) + } + return + } + if err := first.Validate(); err != nil { + t.Fatalf("accepted an invalid catalog: %v", err) + } + // An accepted catalog must always produce a report, whatever it contains. + BuildReport(first, nil, BuildOptions{Release: Release{Tag: "v0.0.0"}, IncludeEvidence: true}) + }) +} + +func FuzzParseGoTestEvents(f *testing.F) { + for _, seed := range []string{ + "", + `{"Action":"pass","Package":"p","Test":"TestOne","Elapsed":1}`, + "{\"Action\":\"output\",\"Package\":\"p\",\"Test\":\"TestOne\",\"Output\":\"ok\\n\"}\n" + + "{\"Action\":\"fail\",\"Package\":\"p\",\"Test\":\"TestOne/sub\",\"Elapsed\":2}\n", + "# github.com/example\nsyntax error\n", + "{\"Action\":\"pass\"", + } { + f.Add([]byte(seed)) + } + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > testutil.MaxFuzzInputSize { + return + } + first, firstErr := ParseGoTestEvents(bytes.NewReader(data), nil) + second, secondErr := ParseGoTestEvents(bytes.NewReader(data), nil) + if (firstErr == nil) != (secondErr == nil) { + t.Fatalf("ParseGoTestEvents changed success state: first=%v second=%v", firstErr, secondErr) + } + if firstErr != nil { + return + } + if first.Total != second.Total || first.Failed != second.Failed { + t.Fatal("ParseGoTestEvents produced different counts for identical input") + } + base := CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Stage: StagePrerequisites, Level: LevelGate, + } + result := first.ToCheckResult(base, 0) + if err := result.Validate(); err != nil { + t.Fatalf("converted result is invalid: %v", err) + } + if len(result.Details) > MaxGoTestDetails { + t.Fatalf("detail count %d exceeds the cap", len(result.Details)) + } + }) +} diff --git a/internal/tools/benchmarkrun/main.go b/internal/assurance/perfrun/main.go similarity index 99% rename from internal/tools/benchmarkrun/main.go rename to internal/assurance/perfrun/main.go index ee77d463..4e48ec2b 100644 --- a/internal/tools/benchmarkrun/main.go +++ b/internal/assurance/perfrun/main.go @@ -1,4 +1,4 @@ -// Command benchmarkrun repeatedly executes one deterministic assurance case +// Command perfrun repeatedly executes one deterministic assurance case // and records cold/warm samples in a versioned machine-readable manifest. package main @@ -124,7 +124,7 @@ func main() { flag.Parse() command := flag.Args() if len(command) == 0 { - fmt.Fprintln(os.Stderr, "benchmarkrun requires a command after --") + fmt.Fprintln(os.Stderr, "perfrun requires a command after --") os.Exit(2) } if samples < 1 { diff --git a/internal/tools/benchmarkrun/main_test.go b/internal/assurance/perfrun/main_test.go similarity index 100% rename from internal/tools/benchmarkrun/main_test.go rename to internal/assurance/perfrun/main_test.go diff --git a/internal/assurance/releaseassets.go b/internal/assurance/releaseassets.go new file mode 100644 index 00000000..56882e59 --- /dev/null +++ b/internal/assurance/releaseassets.go @@ -0,0 +1,365 @@ +package assurance + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" +) + +// maxAssetBytes bounds one release asset read during verification. +const maxAssetBytes = 512 << 20 + +// maxExtractedBytes bounds one file extracted from a release archive. +const maxExtractedBytes = 512 << 20 + +// releasePlatforms are the operating system and architecture pairs every +// release ships binaries for. +var releasePlatforms = []struct{ OS, Arch string }{ + {"darwin", "amd64"}, {"darwin", "arm64"}, + {"linux", "amd64"}, {"linux", "arm64"}, + {"windows", "amd64"}, {"windows", "arm64"}, +} + +// linuxPackageFormats are the Linux package file extensions a release ships. +var linuxPackageFormats = []string{"apk", "deb", "pkg.tar.zst", "rpm"} + +// ArchiveName returns the release archive file name for one binary, version, +// and platform. version is the tag without its leading "v". +func ArchiveName(binary, version, goos, goarch string) string { + extension := "tar.gz" + if goos == "windows" { + extension = "zip" + } + return fmt.Sprintf("%s_%s_%s_%s.%s", binary, version, goos, goarch, extension) +} + +// ExpectedAssets lists every file a published release must carry, sorted. +// version is the release tag without its leading "v". +func ExpectedAssets(version string) []string { + assets := []string{"SHA256SUMS", "SHA256SUMS.sigstore.json", "multiple.intoto.jsonl"} + for _, platform := range releasePlatforms { + assets = append(assets, + ArchiveName("bomly", version, platform.OS, platform.Arch), + ArchiveName("bomly-lite", version, platform.OS, platform.Arch), + ) + } + for _, arch := range []string{"amd64", "arm64"} { + for _, format := range linuxPackageFormats { + assets = append(assets, fmt.Sprintf("bomly_%s_linux_%s.%s", version, arch, format)) + } + } + sort.Strings(assets) + return assets +} + +// ChecksumEntries maps asset file names to their recorded SHA-256 hashes. +type ChecksumEntries map[string]string + +// ParseSHA256SUMS reads a GoReleaser SHA256SUMS document. +func ParseSHA256SUMS(data []byte) (ChecksumEntries, error) { + entries := ChecksumEntries{} + for index, line := range strings.Split(string(data), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + fields := strings.Fields(trimmed) + if len(fields) != 2 { + return nil, fmt.Errorf("SHA256SUMS line %d is malformed", index+1) + } + hash := fields[0] + name := strings.TrimPrefix(fields[1], "*") + if !hashPattern.MatchString(hash) { + return nil, fmt.Errorf("SHA256SUMS line %d has an invalid hash", index+1) + } + if name == "" || strings.ContainsAny(name, "/\\") { + return nil, fmt.Errorf("SHA256SUMS line %d has an invalid file name", index+1) + } + entries[name] = hash + } + if len(entries) == 0 { + return nil, errCatalog("SHA256SUMS lists no files") + } + return entries, nil +} + +// AssetPresence reports which expected assets a directory holds. +type AssetPresence struct { + Present []string + Missing []string + Extra []string +} + +// InspectAssets compares the files in dir against the expected asset list. +func InspectAssets(dir, version string) (AssetPresence, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return AssetPresence{}, fmt.Errorf("read release asset directory: %w", err) + } + found := map[string]struct{}{} + for _, entry := range entries { + if entry.IsDir() { + continue + } + found[entry.Name()] = struct{}{} + } + presence := AssetPresence{} + expected := map[string]struct{}{} + for _, name := range ExpectedAssets(version) { + expected[name] = struct{}{} + if _, ok := found[name]; ok { + presence.Present = append(presence.Present, name) + continue + } + presence.Missing = append(presence.Missing, name) + } + for name := range found { + if _, ok := expected[name]; !ok { + presence.Extra = append(presence.Extra, name) + } + } + sort.Strings(presence.Extra) + return presence, nil +} + +// ChecksumOutcome is the result of hashing the assets present in a directory. +type ChecksumOutcome struct { + Verified []string + Mismatched []string + Unlisted []string + // NotDownloaded names files SHA256SUMS lists that are absent locally, which + // is expected when only the platform-native assets were fetched. + NotDownloaded []string +} + +// VerifyChecksums hashes every file in dir and compares it to SHA256SUMS. +func VerifyChecksums(dir string, entries ChecksumEntries) (ChecksumOutcome, error) { + outcome := ChecksumOutcome{} + present := map[string]struct{}{} + files, err := os.ReadDir(dir) + if err != nil { + return outcome, fmt.Errorf("read release asset directory: %w", err) + } + for _, file := range files { + if file.IsDir() { + continue + } + name := file.Name() + if name == "SHA256SUMS" || name == "SHA256SUMS.sigstore.json" || name == "multiple.intoto.jsonl" { + continue + } + present[name] = struct{}{} + want, listed := entries[name] + if !listed { + outcome.Unlisted = append(outcome.Unlisted, name) + continue + } + actual, hashErr := hashFile(filepath.Join(dir, name)) + if hashErr != nil { + return outcome, hashErr + } + if actual != want { + outcome.Mismatched = append(outcome.Mismatched, name) + continue + } + outcome.Verified = append(outcome.Verified, name) + } + for name := range entries { + if _, ok := present[name]; !ok { + outcome.NotDownloaded = append(outcome.NotDownloaded, name) + } + } + sort.Strings(outcome.Verified) + sort.Strings(outcome.Mismatched) + sort.Strings(outcome.Unlisted) + sort.Strings(outcome.NotDownloaded) + return outcome, nil +} + +func hashFile(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open %s: %w", path, err) + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, io.LimitReader(file, maxAssetBytes)); err != nil { + return "", fmt.Errorf("hash %s: %w", path, err) + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// BinaryProbe is the outcome of extracting one release archive and asking the +// binary inside it for its version. +type BinaryProbe struct { + Archive string + Binary string + Output string + Status Status + Note string +} + +// UntrustedNativeArchives returns the archives for the running platform that +// this release's checksum verification did not vouch for — mismatched, or +// absent from SHA256SUMS entirely. Their contents must not be executed. +func UntrustedNativeArchives(outcome ChecksumOutcome, version string) []string { + suspect := map[string]struct{}{} + for _, name := range outcome.Mismatched { + suspect[name] = struct{}{} + } + verified := map[string]struct{}{} + for _, name := range outcome.Verified { + verified[name] = struct{}{} + } + var untrusted []string + for _, binary := range []string{"bomly", "bomly-lite"} { + archive := ArchiveName(binary, version, runtime.GOOS, runtime.GOARCH) + if _, bad := suspect[archive]; bad { + untrusted = append(untrusted, archive) + continue + } + if _, ok := verified[archive]; !ok { + untrusted = append(untrusted, archive) + } + } + sort.Strings(untrusted) + return untrusted +} + +// ProbeNativeBinaries extracts the archives built for the host platform and +// checks that each binary reports the released version. +func ProbeNativeBinaries(ctx context.Context, dir, version, workDir string) ([]BinaryProbe, error) { + var probes []BinaryProbe + for _, binary := range []string{"bomly", "bomly-lite"} { + archive := ArchiveName(binary, version, runtime.GOOS, runtime.GOARCH) + probe := BinaryProbe{Archive: archive, Binary: binary, Status: StatusFail} + target := filepath.Join(workDir, binary) + if err := os.MkdirAll(target, 0o755); err != nil { + return nil, fmt.Errorf("create extraction directory: %w", err) + } + executable, err := extractBinary(filepath.Join(dir, archive), target, binary) + if err != nil { + probe.Note = err.Error() + probes = append(probes, probe) + continue + } + output, err := runVersion(ctx, executable) + probe.Output = output + switch { + case err != nil: + probe.Note = err.Error() + case !strings.Contains(output, version): + probe.Note = fmt.Sprintf("reported %q, want version %s", output, version) + default: + probe.Status = StatusPass + } + probes = append(probes, probe) + } + return probes, nil +} + +func runVersion(ctx context.Context, executable string) (string, error) { + runCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + command := exec.CommandContext(runCtx, executable, "version") + output, err := command.CombinedOutput() + text := strings.TrimSpace(string(output)) + if err != nil { + return text, fmt.Errorf("run %s version: %w", filepath.Base(executable), err) + } + return text, nil +} + +// extractBinary pulls one named binary out of a release archive into target. +func extractBinary(archivePath, target, binary string) (string, error) { + name := binary + if runtime.GOOS == "windows" { + name += ".exe" + } + if strings.HasSuffix(archivePath, ".zip") { + return extractFromZip(archivePath, target, name) + } + return extractFromTarGz(archivePath, target, name) +} + +func extractFromZip(archivePath, target, name string) (string, error) { + reader, err := zip.OpenReader(archivePath) + if err != nil { + return "", fmt.Errorf("open %s: %w", filepath.Base(archivePath), err) + } + defer reader.Close() + for _, file := range reader.File { + if filepath.Base(file.Name) != name || file.FileInfo().IsDir() { + continue + } + source, openErr := file.Open() + if openErr != nil { + return "", fmt.Errorf("read %s from %s: %w", name, filepath.Base(archivePath), openErr) + } + defer source.Close() + return writeExecutable(filepath.Join(target, name), source) + } + return "", fmt.Errorf("%s does not contain %s", filepath.Base(archivePath), name) +} + +func extractFromTarGz(archivePath, target, name string) (string, error) { + file, err := os.Open(archivePath) + if err != nil { + return "", fmt.Errorf("open %s: %w", filepath.Base(archivePath), err) + } + defer file.Close() + gzipReader, err := gzip.NewReader(file) + if err != nil { + return "", fmt.Errorf("read %s: %w", filepath.Base(archivePath), err) + } + defer gzipReader.Close() + tarReader := tar.NewReader(gzipReader) + for { + header, readErr := tarReader.Next() + if readErr == io.EOF { + break + } + if readErr != nil { + return "", fmt.Errorf("read %s: %w", filepath.Base(archivePath), readErr) + } + if header.Typeflag != tar.TypeReg || filepath.Base(header.Name) != name { + continue + } + return writeExecutable(filepath.Join(target, name), tarReader) + } + return "", fmt.Errorf("%s does not contain %s", filepath.Base(archivePath), name) +} + +// writeExecutable copies a bounded amount of archive content to an executable +// file. Destinations are always names this package chose, never archive paths, +// so an archive cannot direct a write outside the extraction directory. +func writeExecutable(destination string, source io.Reader) (string, error) { + file, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o755) + if err != nil { + return "", fmt.Errorf("create %s: %w", filepath.Base(destination), err) + } + written, copyErr := io.Copy(file, io.LimitReader(source, maxExtractedBytes+1)) + closeErr := file.Close() + if copyErr != nil { + return "", fmt.Errorf("write %s: %w", filepath.Base(destination), copyErr) + } + if closeErr != nil { + return "", fmt.Errorf("close %s: %w", filepath.Base(destination), closeErr) + } + if written > maxExtractedBytes { + return "", fmt.Errorf("%s exceeds the %d byte extraction limit", filepath.Base(destination), maxExtractedBytes) + } + return destination, nil +} diff --git a/internal/assurance/render_markdown.go b/internal/assurance/render_markdown.go new file mode 100644 index 00000000..cc781de0 --- /dev/null +++ b/internal/assurance/render_markdown.go @@ -0,0 +1,243 @@ +package assurance + +import ( + "fmt" + "strings" +) + +// StatusIcon returns the emoji shown next to a status in markdown summaries. +func StatusIcon(status Status) string { + switch status { + case StatusPass: + return "✅" + case StatusFail: + return "❌" + case StatusDegraded: + return "⚠️" + case StatusSkip: + return "⏭️" + case StatusMissing: + return "❓" + default: + return "•" + } +} + +// MarkdownOptions controls how a report renders as markdown. +type MarkdownOptions struct { + // Heading is the top-level title; a default is used when empty. + Heading string + // IncludeChecks lists every check in a per-stage table. + IncludeChecks bool + // IncludeTrends adds the comparison against the previous release. + IncludeTrends bool +} + +// RenderMarkdown renders a report as the markdown used for job summaries and +// the per-release tracking issue. +func RenderMarkdown(report Report, opts MarkdownOptions) string { + var out strings.Builder + heading := opts.Heading + if heading == "" { + heading = "Release assurance" + if report.Release.Tag != "" { + heading += " " + report.Release.Tag + } + } + fmt.Fprintf(&out, "## %s %s\n\n", StatusIcon(report.Verdict.Overall), heading) + fmt.Fprintf(&out, "%s\n\n", verdictSentence(report.Verdict)) + + if len(report.Stages) > 0 { + out.WriteString("| Stage | Result | Checks | Passed | Failed | Missing |\n") + out.WriteString("| --- | --- | --- | --- | --- | --- |\n") + for _, stage := range report.Stages { + fmt.Fprintf(&out, "| %s | %s %s | %d | %d | %d | %d |\n", + stage.Title, StatusIcon(stage.Status), stage.Status, + stage.Verdict.Checks, stage.Verdict.Passed, stage.Verdict.Failed, stage.Verdict.Missing) + } + out.WriteString("\n") + } + + if opts.IncludeChecks { + for _, stage := range report.Stages { + if len(stage.CheckIDs) == 0 { + continue + } + fmt.Fprintf(&out, "### %s\n\n", stage.Title) + out.WriteString("| Check | Level | Result | Summary |\n") + out.WriteString("| --- | --- | --- | --- |\n") + for _, id := range stage.CheckIDs { + check, found := report.Check(id) + if !found { + continue + } + fmt.Fprintf(&out, "| %s | %s | %s %s | %s |\n", + check.Title, check.Level, StatusIcon(check.Status), check.Status, + markdownCell(check.Summary)) + } + out.WriteString("\n") + } + } + + if attention := attentionLines(report); len(attention) > 0 { + out.WriteString("### Needs attention\n\n") + for _, line := range attention { + out.WriteString("- " + line + "\n") + } + out.WriteString("\n") + } + + if opts.IncludeTrends && report.Trends != nil { + out.WriteString(renderTrends(*report.Trends)) + } + return out.String() +} + +func verdictSentence(verdict Verdict) string { + switch verdict.Overall { + case StatusPass: + return fmt.Sprintf("All %d checks passed.", verdict.Checks) + case StatusDegraded: + return fmt.Sprintf("%d of %d checks passed; advisory checks reported problems that do not block the release.", + verdict.Passed, verdict.Checks) + case StatusMissing: + return fmt.Sprintf("%d of %d checks reported nothing, so the release cannot be judged complete.", + verdict.Missing, verdict.Checks) + default: + return fmt.Sprintf("%d of %d checks passed; %d failed and %d reported nothing.", + verdict.Passed, verdict.Checks, verdict.Failed, verdict.Missing) + } +} + +func attentionLines(report Report) []string { + var lines []string + for _, check := range report.Checks { + if check.Status == StatusPass || check.Status == StatusSkip { + continue + } + line := fmt.Sprintf("%s **%s** (%s, %s) — %s", + StatusIcon(check.Status), check.Title, check.Level, check.Stage, markdownCell(check.Summary)) + if len(check.MissingInstances) > 0 { + line += fmt.Sprintf(" Missing: %s.", strings.Join(check.MissingInstances, ", ")) + } + // Only the parts that need attention are linked: a reader of a red + // summary should reach the failing log in one click, and linking the + // passing rows too would bury it. + if logs := failingLogs(check); logs != "" { + line += " Logs: " + logs + "." + } + lines = append(lines, line) + } + for _, unknown := range report.Unknown { + lines = append(lines, fmt.Sprintf("❓ Result `%s` is not declared in the assurance catalog.", unknown.ID)) + } + return lines +} + +// failingLogs renders links to the jobs behind a check's failing instances. +func failingLogs(check ReportCheck) string { + var links []string + for _, instance := range check.Instances { + if instance.Status == StatusPass || instance.RunURL == "" { + continue + } + name := instance.Name + if name == "default" { + name = "job" + } + links = append(links, fmt.Sprintf("[%s](%s)", name, instance.RunURL)) + } + if len(links) == 0 { + return "" + } + if len(links) > 5 { + remaining := len(links) - 5 + links = append(links[:5], fmt.Sprintf("and %d more", remaining)) + } + return strings.Join(links, ", ") +} + +func renderTrends(trends Trends) string { + if len(trends.Changed) == 0 && len(trends.Metrics) == 0 { + return "" + } + var out strings.Builder + fmt.Fprintf(&out, "### Compared with %s\n\n", trends.PreviousTag) + for _, change := range trends.Changed { + fmt.Fprintf(&out, "- `%s`: %s → %s\n", change.CheckID, change.Previous, change.Current) + } + shown := 0 + for _, metric := range trends.Metrics { + if shown >= 8 { + break + } + if metric.Better == betterNeutral && metric.DeltaPct < 5 && metric.DeltaPct > -5 { + continue + } + fmt.Fprintf(&out, "- `%s` %s: %.2f → %.2f (%+.1f%%)\n", + metric.CheckID, metric.Metric, metric.Previous, metric.Current, metric.DeltaPct) + shown++ + } + out.WriteString("\n") + return out.String() +} + +// RenderResultMarkdown renders one check result for a workflow step summary. +// reproduce is the catalog's local reproduction command for the check, shown so +// a reader can run the same thing without hunting for it. +func RenderResultMarkdown(result CheckResult, reproduce [][]string) string { + var out strings.Builder + title := result.ID + if result.Instance != "" { + title += " (" + result.Instance + ")" + } + fmt.Fprintf(&out, "### %s %s\n\n%s\n\n", StatusIcon(result.Status), title, result.Summary) + if len(result.Details) > 0 { + out.WriteString("| Item | Result | Note |\n| --- | --- | --- |\n") + shown := 0 + for _, detail := range result.Details { + if shown >= 30 { + fmt.Fprintf(&out, "| … | | %d further items are recorded in the check result |\n", + len(result.Details)-shown) + break + } + fmt.Fprintf(&out, "| %s | %s %s | %s |\n", + markdownCell(detail.Name), StatusIcon(detail.Status), detail.Status, markdownCell(detail.Note)) + shown++ + } + out.WriteString("\n") + } + if len(reproduce) > 0 { + out.WriteString("Reproduce locally:\n\n```sh\n") + for _, command := range reproduce { + out.WriteString(shellCommand(command) + "\n") + } + out.WriteString("```\n\n") + } + return out.String() +} + +// shellCommand renders one argument list as a copyable shell command. +func shellCommand(command []string) string { + quoted := make([]string, len(command)) + for index, argument := range command { + if argument != "" && strings.IndexFunc(argument, func(r rune) bool { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + return false + } + return !strings.ContainsRune("@%_+=:,./-", r) + }) == -1 { + quoted[index] = argument + continue + } + quoted[index] = "'" + strings.ReplaceAll(argument, "'", "'\"'\"'") + "'" + } + return strings.Join(quoted, " ") +} + +func markdownCell(value string) string { + cleaned := strings.ReplaceAll(strings.TrimSpace(value), "|", "\\|") + cleaned = strings.ReplaceAll(cleaned, "\r", " ") + cleaned = strings.ReplaceAll(cleaned, "\n", " ") + return cleaned +} diff --git a/internal/assurance/report.go b/internal/assurance/report.go new file mode 100644 index 00000000..4e299a41 --- /dev/null +++ b/internal/assurance/report.go @@ -0,0 +1,313 @@ +package assurance + +import ( + "encoding/json" + "fmt" + "strings" +) + +// ReportSchema is the schema identifier of a per-release assurance report. +const ReportSchema = "bomly.assurance-report/v1" + +// IndexSchema is the schema identifier of the release index. +const IndexSchema = "bomly.assurance-index/v1" + +// MaxReportBytes bounds a report document. +const MaxReportBytes = 16 << 20 + +// Release identifies the release a report describes. +type Release struct { + Tag string `json:"tag"` + Version string `json:"version"` + Commit string `json:"commit,omitempty"` + URL string `json:"url,omitempty"` + PublishedAt string `json:"published_at,omitempty"` +} + +// Verdict counts outcomes and names the checks that need attention. +type Verdict struct { + Overall Status `json:"overall"` + Checks int `json:"checks"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Degraded int `json:"degraded"` + Skipped int `json:"skipped"` + Missing int `json:"missing"` + GatesFailed []string `json:"gates_failed,omitempty"` + AdvisoriesFailed []string `json:"advisories_failed,omitempty"` + MissingChecks []string `json:"missing_checks,omitempty"` +} + +// Blocking reports whether the verdict must stop a release. +func (v Verdict) Blocking() bool { return len(v.GatesFailed) > 0 || len(v.MissingChecks) > 0 } + +// StageReport is one release stage and the checks it contributed. +type StageReport struct { + ID Stage `json:"id"` + Title string `json:"title"` + Status Status `json:"status"` + Verdict Verdict `json:"verdict"` + RunURL string `json:"run_url,omitempty"` + CheckIDs []string `json:"check_ids"` +} + +// AreaReport summarises one subject area: the checks that cover it and the +// evidence claims made about it. Areas are what the published report is +// organised by, because "what does this tell me about Bomly" is a more useful +// question for a reader than "when in the release did this run". +type AreaReport struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Status Status `json:"status"` + Verdict Verdict `json:"verdict"` + CheckIDs []string `json:"check_ids,omitempty"` + EvidenceIDs []string `json:"evidence_ids,omitempty"` +} + +// InstanceReport is one reported leg of a check. +type InstanceReport struct { + Name string `json:"name"` + Status Status `json:"status"` + Summary string `json:"summary,omitempty"` + DurationMS float64 `json:"duration_ms,omitempty"` + RunURL string `json:"run_url,omitempty"` + Runner Runner `json:"runner,omitempty"` + Metrics map[string]float64 `json:"metrics,omitempty"` + Details []Detail `json:"details,omitempty"` + Artifacts []Artifact `json:"artifacts,omitempty"` + Links []Link `json:"links,omitempty"` +} + +// ReportCheck is one catalog check merged with the results it received. +type ReportCheck struct { + ID string `json:"id"` + Title string `json:"title"` + Area string `json:"area"` + Stage Stage `json:"stage"` + Level Level `json:"level"` + Description string `json:"description"` + Status Status `json:"status"` + Summary string `json:"summary,omitempty"` + DurationMS float64 `json:"duration_ms,omitempty"` + Source Source `json:"source"` + Instances []InstanceReport `json:"instances,omitempty"` + MissingInstances []string `json:"missing_instances,omitempty"` + Metrics map[string]float64 `json:"metrics,omitempty"` + Reproduce [][]string `json:"reproduce,omitempty"` + Proves []string `json:"proves"` + Limitations []string `json:"limitations"` +} + +// ReportEvidence is one public claim with the status of the check backing it. +type ReportEvidence struct { + ID string `json:"id"` + Title string `json:"title"` + Area string `json:"area"` + Description string `json:"description"` + EvidenceLevel EvidenceLevel `json:"evidence_level"` + CheckID string `json:"check_id"` + Instance string `json:"instance,omitempty"` + Status Status `json:"status"` + Inputs []Input `json:"inputs"` + RequiredTools []string `json:"required_tools,omitempty"` + Reproduce [][]string `json:"reproduce"` + Artifacts []EvidenceArtifact `json:"artifacts"` + Proves []string `json:"proves"` + Limitations []string `json:"limitations"` +} + +// EcosystemCoverage is whether one language or package format was exercised +// for this release. Which check did the exercising is deliberately not part of +// it: the reader's question is "was my ecosystem covered", and answering it +// with a grid of checks invites the false conclusion that a blank cell is a +// gap when the ecosystem was covered by another check. +type EcosystemCoverage struct { + Name string `json:"name"` + Status Status `json:"status"` +} + +// Coverage lists every ecosystem any check exercised, in name order. Each +// carries the worst status among the checks that covered it. +type Coverage struct { + Ecosystems []EcosystemCoverage `json:"ecosystems"` +} + +// MetricTrend compares one metric against the previous release's report. +type MetricTrend struct { + CheckID string `json:"check_id"` + Metric string `json:"metric"` + Previous float64 `json:"previous"` + Current float64 `json:"current"` + Delta float64 `json:"delta"` + DeltaPct float64 `json:"delta_pct,omitempty"` + Better string `json:"better"` +} + +// StatusChange records a check whose status moved between releases. +type StatusChange struct { + CheckID string `json:"check_id"` + Previous Status `json:"previous"` + Current Status `json:"current"` +} + +// Trends compares this report against the previous release's report. +type Trends struct { + PreviousTag string `json:"previous_tag"` + Metrics []MetricTrend `json:"metrics,omitempty"` + Changed []StatusChange `json:"changed,omitempty"` +} + +// UnknownResult records a reported check that the catalog does not declare. +type UnknownResult struct { + ID string `json:"id"` + Instance string `json:"instance,omitempty"` + Stage Stage `json:"stage"` + Status Status `json:"status"` +} + +// Environment records where the checks ran. +type Environment struct { + Runners []Runner `json:"runners,omitempty"` + GeneratedBy string `json:"generated_by,omitempty"` +} + +// Report is the per-release document the public assurance page renders. +type Report struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt string `json:"generated_at"` + Release Release `json:"release"` + Verdict Verdict `json:"verdict"` + Stages []StageReport `json:"stages"` + Areas []AreaReport `json:"areas"` + Checks []ReportCheck `json:"checks"` + Evidence []ReportEvidence `json:"evidence"` + Coverage Coverage `json:"coverage"` + Trends *Trends `json:"trends,omitempty"` + Unknown []UnknownResult `json:"unknown_results,omitempty"` + Environment Environment `json:"environment"` +} + +// Encode renders the report as indented JSON with a trailing newline. +func (r Report) Encode() ([]byte, error) { + data, err := json.MarshalIndent(r, "", " ") + if err != nil { + return nil, fmt.Errorf("encode assurance report: %w", err) + } + return append(data, '\n'), nil +} + +// Check looks up one reported check by ID. +func (r Report) Check(id string) (ReportCheck, bool) { + for _, check := range r.Checks { + if check.ID == id { + return check, true + } + } + return ReportCheck{}, false +} + +// ParseReport decodes and structurally validates a report document. +func ParseReport(data []byte) (Report, error) { + if len(data) > MaxReportBytes { + return Report{}, fmt.Errorf("assurance report is %d bytes, limit is %d", len(data), MaxReportBytes) + } + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + var report Report + if err := decoder.Decode(&report); err != nil { + return Report{}, fmt.Errorf("decode assurance report: %w", err) + } + if err := ensureEOF(decoder, "assurance report"); err != nil { + return Report{}, err + } + if report.SchemaVersion != ReportSchema { + return Report{}, fmt.Errorf("unsupported assurance report schema %q", report.SchemaVersion) + } + if strings.TrimSpace(report.Release.Tag) == "" { + return Report{}, errCatalog("assurance report is missing its release tag") + } + if !report.Verdict.Overall.Valid() { + return Report{}, fmt.Errorf("assurance report has unsupported verdict %q", report.Verdict.Overall) + } + for _, check := range report.Checks { + if !idPattern.MatchString(check.ID) { + return Report{}, fmt.Errorf("assurance report has invalid check id %q", check.ID) + } + if !check.Status.Valid() { + return Report{}, fmt.Errorf("check %q has unsupported status %q", check.ID, check.Status) + } + } + // Claims carry the same fields whichever way they are asserted, because the + // published page renders one shape for all of them; a claim with no + // description would render as an empty card. + for _, evidence := range report.Evidence { + if !idPattern.MatchString(evidence.ID) { + return Report{}, fmt.Errorf("assurance report has invalid evidence id %q", evidence.ID) + } + if !evidence.Status.Valid() { + return Report{}, fmt.Errorf("evidence %q has unsupported status %q", evidence.ID, evidence.Status) + } + if strings.TrimSpace(evidence.Description) == "" { + return Report{}, fmt.Errorf("evidence %q is missing its description", evidence.ID) + } + } + return report, nil +} + +// LoadReport reads and validates the report at path. +func LoadReport(path string) (Report, error) { + data, err := readBounded(path, MaxReportBytes) + if err != nil { + return Report{}, err + } + return ParseReport(data) +} + +// IndexEntry is one release listed in the assurance index. +type IndexEntry struct { + Tag string `json:"tag"` + Version string `json:"version"` + PublishedAt string `json:"published_at,omitempty"` + GeneratedAt string `json:"generated_at"` + Verdict Status `json:"verdict"` + Gates int `json:"gates_failed"` + Path string `json:"path"` +} + +// Index lists every release that has a published assurance report, newest first. +type Index struct { + SchemaVersion string `json:"schema_version"` + Latest string `json:"latest"` + GeneratedAt string `json:"generated_at"` + Releases []IndexEntry `json:"releases"` +} + +// Encode renders the index as indented JSON with a trailing newline. +func (i Index) Encode() ([]byte, error) { + data, err := json.MarshalIndent(i, "", " ") + if err != nil { + return nil, fmt.Errorf("encode assurance index: %w", err) + } + return append(data, '\n'), nil +} + +// ParseIndex decodes and validates an index document. +func ParseIndex(data []byte) (Index, error) { + if len(data) > MaxReportBytes { + return Index{}, fmt.Errorf("assurance index is %d bytes, limit is %d", len(data), MaxReportBytes) + } + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + var index Index + if err := decoder.Decode(&index); err != nil { + return Index{}, fmt.Errorf("decode assurance index: %w", err) + } + if err := ensureEOF(decoder, "assurance index"); err != nil { + return Index{}, err + } + if index.SchemaVersion != IndexSchema { + return Index{}, fmt.Errorf("unsupported assurance index schema %q", index.SchemaVersion) + } + return index, nil +} diff --git a/internal/tools/sbomassurance/main.go b/internal/assurance/sbominterop/main.go similarity index 97% rename from internal/tools/sbomassurance/main.go rename to internal/assurance/sbominterop/main.go index c0131d89..387410b1 100644 --- a/internal/tools/sbomassurance/main.go +++ b/internal/assurance/sbominterop/main.go @@ -1,6 +1,6 @@ -// Command sbomassurance generates canonical SBOMs and validates them with -// checksum-pinned upstream tools. It is intended only for the explicit -// interoperability assurance workflow. +// Command sbominterop generates canonical SBOMs and validates them with +// checksum-pinned upstream tools. It backs the SBOM interoperability check of +// Bomly's release assurance framework. package main import ( @@ -44,6 +44,7 @@ type runManifest struct { Validators []validatorInfo `json:"validators"` Artifacts []artifactInfo `json:"artifacts"` Commands []commandResult `json:"commands"` + Failure string `json:"failure,omitempty"` } type hostInfo struct { @@ -301,6 +302,7 @@ func describeArtifact(format, path string) (artifactInfo, error) { func writeFailure(outputDir string, manifest *runManifest, cause error) error { manifest.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano) + manifest.Failure = cause.Error() if err := writeManifest(outputDir, *manifest); err != nil { return fmt.Errorf("%v; additionally write run manifest: %w", cause, err) } diff --git a/internal/tools/sbomassurance/main_test.go b/internal/assurance/sbominterop/main_test.go similarity index 100% rename from internal/tools/sbomassurance/main_test.go rename to internal/assurance/sbominterop/main_test.go diff --git a/internal/assurance/testdata/catalog.json b/internal/assurance/testdata/catalog.json new file mode 100644 index 00000000..98993f3d --- /dev/null +++ b/internal/assurance/testdata/catalog.json @@ -0,0 +1,91 @@ +{ + "schema_version": "bomly.assurance-catalog/v1", + "areas": [ + { + "id": "end-to-end", + "title": "End-to-end behavior", + "description": "Running the real command line against real projects." + }, + { + "id": "release-integrity", + "title": "Release integrity", + "description": "Whether the published files are complete and unmodified." + }, + { + "id": "performance", + "title": "Speed and stability", + "description": "How long a repeated scan takes and whether it returns the same answer." + } + ], + "checks": [ + { + "id": "perf-samples", + "title": "Repeated scan speed and stability", + "area": "performance", + "stage": "post-release", + "level": "advisory", + "description": "Runs the same scan several times and records timing.", + "source": { "workflow": "assurance-assessment.yml", "job": "perf-samples" }, + "reproduce": [["make", "benchmark-samples"]], + "proves": ["Repeated runs produced identical normalized output."], + "limitations": ["Timings are observations from one machine, not limits."] + }, + { + "id": "release-checksums", + "title": "Release files match their checksums", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Hashes every release file and compares it with SHA256SUMS.", + "source": { "workflow": "release.yml", "job": "verify-draft" }, + "reproduce": [["sha256sum", "--check", "SHA256SUMS"]], + "proves": ["Every published file hashes to the recorded value."], + "limitations": ["Checksums prove the files were not altered after they were built."] + }, + { + "id": "smoke", + "title": "End-to-end scans of real projects", + "area": "end-to-end", + "stage": "prerequisites", + "level": "gate", + "description": "Runs scan against pinned public repositories.", + "source": { "workflow": "smoke.yml", "job": "smoke" }, + "expected_instances": [ + { "name": "go", "ecosystems": ["Go"] }, + { "name": "node", "ecosystems": ["JavaScript"] } + ], + "reproduce": [["make", "smoke"]], + "proves": ["Each ecosystem produces the dependency graph recorded in its golden file."], + "limitations": ["Golden files pin one revision of one example project per ecosystem."] + } + ], + "evidence": [ + { + "id": "graph-go", + "title": "Go module graph", + "area": "end-to-end", + "description": "Scans a pinned Go example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "go", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": ["git", "go"], + "reproduce": [["make", "smoke"]], + "artifacts": [ + { + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", + "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + } + ], + "proves": ["The Go detector resolves a build-tool-backed module graph."], + "limitations": ["The result depends on a compatible Go toolchain."] + } + ] +} diff --git a/internal/assurance/testdata/fixtures/all-pass/results/perf-samples.json b/internal/assurance/testdata/fixtures/all-pass/results/perf-samples.json new file mode 100644 index 00000000..1e728acf --- /dev/null +++ b/internal/assurance/testdata/fixtures/all-pass/results/perf-samples.json @@ -0,0 +1,21 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "perf-samples", + "stage": "post-release", + "level": "advisory", + "status": "pass", + "duration_ms": 52000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/3", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 412 ms, warm median 288 ms).", + "metrics": { + "cold_median_ms": 412, + "warm_median_ms": 288, + "samples_per_mode": 5, + "peak_memory_bytes": 91234304 + } +} diff --git a/internal/assurance/testdata/fixtures/all-pass/results/release-checksums.json b/internal/assurance/testdata/fixtures/all-pass/results/release-checksums.json new file mode 100644 index 00000000..7249bc62 --- /dev/null +++ b/internal/assurance/testdata/fixtures/all-pass/results/release-checksums.json @@ -0,0 +1,20 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "release-checksums", + "stage": "pre-release", + "level": "gate", + "status": "pass", + "duration_ms": 4200, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/2", + "summary": "23 downloaded assets match SHA256SUMS, which lists 23 files.", + "metrics": { + "verified": 23, + "mismatched": 0, + "listed": 23 + } +} diff --git a/internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json b/internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json new file mode 100644 index 00000000..6d623e5a --- /dev/null +++ b/internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json @@ -0,0 +1,29 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "smoke", + "instance": "go", + "stage": "prerequisites", + "level": "gate", + "status": "pass", + "duration_ms": 61000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/1", + "summary": "18 of 18 tests passed.", + "metrics": { + "tests_total": 18, + "tests_passed": 18, + "tests_failed": 0, + "tests_skipped": 0 + }, + "details": [ + { + "name": "smoke.TestScan/scan-go", + "status": "pass", + "duration_ms": 41000 + } + ] +} diff --git a/internal/assurance/testdata/fixtures/all-pass/results/smoke.node.json b/internal/assurance/testdata/fixtures/all-pass/results/smoke.node.json new file mode 100644 index 00000000..0315970b --- /dev/null +++ b/internal/assurance/testdata/fixtures/all-pass/results/smoke.node.json @@ -0,0 +1,22 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "smoke", + "instance": "node", + "stage": "prerequisites", + "level": "gate", + "status": "pass", + "duration_ms": 95000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/1", + "summary": "24 of 24 tests passed.", + "metrics": { + "tests_total": 24, + "tests_passed": 24, + "tests_failed": 0, + "tests_skipped": 0 + } +} diff --git a/internal/assurance/testdata/fixtures/mixed-failure/results/mystery-check.json b/internal/assurance/testdata/fixtures/mixed-failure/results/mystery-check.json new file mode 100644 index 00000000..6874d2f0 --- /dev/null +++ b/internal/assurance/testdata/fixtures/mixed-failure/results/mystery-check.json @@ -0,0 +1,14 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "mystery-check", + "stage": "post-release", + "level": "gate", + "status": "pass", + "duration_ms": 10, + "summary": "A check that the catalog does not declare." +} diff --git a/internal/assurance/testdata/fixtures/mixed-failure/results/perf-samples.json b/internal/assurance/testdata/fixtures/mixed-failure/results/perf-samples.json new file mode 100644 index 00000000..80a2bb7c --- /dev/null +++ b/internal/assurance/testdata/fixtures/mixed-failure/results/perf-samples.json @@ -0,0 +1,21 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "perf-samples", + "stage": "post-release", + "level": "advisory", + "status": "degraded", + "duration_ms": 61000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/11", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms).", + "metrics": { + "cold_median_ms": 705, + "warm_median_ms": 402, + "samples_per_mode": 5, + "peak_memory_bytes": 120586240 + } +} diff --git a/internal/assurance/testdata/fixtures/mixed-failure/results/release-checksums.json b/internal/assurance/testdata/fixtures/mixed-failure/results/release-checksums.json new file mode 100644 index 00000000..609841bf --- /dev/null +++ b/internal/assurance/testdata/fixtures/mixed-failure/results/release-checksums.json @@ -0,0 +1,27 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "release-checksums", + "stage": "pre-release", + "level": "gate", + "status": "fail", + "duration_ms": 3900, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/10", + "summary": "1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz.", + "metrics": { + "verified": 22, + "mismatched": 1, + "listed": 23 + }, + "details": [ + { + "name": "bomly_9.9.9_linux_amd64.tar.gz", + "status": "fail", + "note": "hash does not match SHA256SUMS" + } + ] +} diff --git a/internal/assurance/testdata/fixtures/mixed-failure/results/smoke.go.json b/internal/assurance/testdata/fixtures/mixed-failure/results/smoke.go.json new file mode 100644 index 00000000..56259e85 --- /dev/null +++ b/internal/assurance/testdata/fixtures/mixed-failure/results/smoke.go.json @@ -0,0 +1,22 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "smoke", + "instance": "go", + "stage": "prerequisites", + "level": "gate", + "status": "pass", + "duration_ms": 60000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/9", + "summary": "18 of 18 tests passed.", + "metrics": { + "tests_total": 18, + "tests_passed": 18, + "tests_failed": 0, + "tests_skipped": 0 + } +} diff --git a/internal/assurance/testdata/golden/all-pass.report.json b/internal/assurance/testdata/golden/all-pass.report.json new file mode 100644 index 00000000..6b07becd --- /dev/null +++ b/internal/assurance/testdata/golden/all-pass.report.json @@ -0,0 +1,379 @@ +{ + "schema_version": "bomly.assurance-report/v1", + "generated_at": "2026-08-19T12:00:00Z", + "release": { + "tag": "v9.9.9", + "version": "9.9.9", + "commit": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + }, + "verdict": { + "overall": "pass", + "checks": 3, + "passed": 3, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "stages": [ + { + "id": "prerequisites", + "title": "Release prerequisites", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "check_ids": [ + "smoke" + ] + }, + { + "id": "pre-release", + "title": "Final pre-release checks", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "check_ids": [ + "release-checksums" + ] + }, + { + "id": "post-release", + "title": "Post-release assessment", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "check_ids": [ + "perf-samples" + ] + } + ], + "areas": [ + { + "id": "end-to-end", + "title": "End-to-end behavior", + "description": "Running the real command line against real projects.", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "check_ids": [ + "smoke" + ], + "evidence_ids": [ + "graph-go" + ] + }, + { + "id": "release-integrity", + "title": "Release integrity", + "description": "Whether the published files are complete and unmodified.", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "check_ids": [ + "release-checksums" + ] + }, + { + "id": "performance", + "title": "Speed and stability", + "description": "How long a repeated scan takes and whether it returns the same answer.", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "check_ids": [ + "perf-samples" + ] + } + ], + "checks": [ + { + "id": "perf-samples", + "title": "Repeated scan speed and stability", + "area": "performance", + "stage": "post-release", + "level": "advisory", + "description": "Runs the same scan several times and records timing.", + "status": "pass", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 412 ms, warm median 288 ms).", + "duration_ms": 52000, + "source": { + "workflow": "assurance-assessment.yml", + "job": "perf-samples" + }, + "instances": [ + { + "name": "default", + "status": "pass", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 412 ms, warm median 288 ms).", + "duration_ms": 52000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/3", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "cold_median_ms": 412, + "peak_memory_bytes": 91234304, + "samples_per_mode": 5, + "warm_median_ms": 288 + } + } + ], + "metrics": { + "cold_median_ms": 412, + "peak_memory_bytes": 91234304, + "samples_per_mode": 5, + "warm_median_ms": 288 + }, + "reproduce": [ + [ + "make", + "benchmark-samples" + ] + ], + "proves": [ + "Repeated runs produced identical normalized output." + ], + "limitations": [ + "Timings are observations from one machine, not limits." + ] + }, + { + "id": "release-checksums", + "title": "Release files match their checksums", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Hashes every release file and compares it with SHA256SUMS.", + "status": "pass", + "summary": "23 downloaded assets match SHA256SUMS, which lists 23 files.", + "duration_ms": 4200, + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "instances": [ + { + "name": "default", + "status": "pass", + "summary": "23 downloaded assets match SHA256SUMS, which lists 23 files.", + "duration_ms": 4200, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/2", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "listed": 23, + "mismatched": 0, + "verified": 23 + } + } + ], + "metrics": { + "listed": 23, + "mismatched": 0, + "verified": 23 + }, + "reproduce": [ + [ + "sha256sum", + "--check", + "SHA256SUMS" + ] + ], + "proves": [ + "Every published file hashes to the recorded value." + ], + "limitations": [ + "Checksums prove the files were not altered after they were built." + ] + }, + { + "id": "smoke", + "title": "End-to-end scans of real projects", + "area": "end-to-end", + "stage": "prerequisites", + "level": "gate", + "description": "Runs scan against pinned public repositories.", + "status": "pass", + "summary": "2 instances passed.", + "duration_ms": 156000, + "source": { + "workflow": "smoke.yml", + "job": "smoke" + }, + "instances": [ + { + "name": "go", + "status": "pass", + "summary": "18 of 18 tests passed.", + "duration_ms": 61000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "tests_failed": 0, + "tests_passed": 18, + "tests_skipped": 0, + "tests_total": 18 + }, + "details": [ + { + "name": "smoke.TestScan/scan-go", + "status": "pass", + "duration_ms": 41000 + } + ] + }, + { + "name": "node", + "status": "pass", + "summary": "24 of 24 tests passed.", + "duration_ms": 95000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "tests_failed": 0, + "tests_passed": 24, + "tests_skipped": 0, + "tests_total": 24 + } + } + ], + "metrics": { + "tests_failed": 0, + "tests_passed": 42, + "tests_skipped": 0, + "tests_total": 42 + }, + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "proves": [ + "Each ecosystem produces the dependency graph recorded in its golden file." + ], + "limitations": [ + "Golden files pin one revision of one example project per ecosystem." + ] + } + ], + "evidence": [ + { + "id": "graph-go", + "title": "Go module graph", + "area": "end-to-end", + "description": "Scans a pinned Go example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "go", + "status": "pass", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "artifacts": [ + { + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", + "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + } + ], + "proves": [ + "The Go detector resolves a build-tool-backed module graph." + ], + "limitations": [ + "The result depends on a compatible Go toolchain." + ] + } + ], + "coverage": { + "ecosystems": [ + { + "name": "Go", + "status": "pass" + }, + { + "name": "JavaScript", + "status": "pass" + } + ] + }, + "environment": { + "runners": [ + { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + } + ], + "generated_by": "assurance-test" + } +} diff --git a/internal/assurance/testdata/golden/all-pass.summary.md b/internal/assurance/testdata/golden/all-pass.summary.md new file mode 100644 index 00000000..334e6ef4 --- /dev/null +++ b/internal/assurance/testdata/golden/all-pass.summary.md @@ -0,0 +1,28 @@ +## ✅ Release assurance v9.9.9 + +All 3 checks passed. + +| Stage | Result | Checks | Passed | Failed | Missing | +| --- | --- | --- | --- | --- | --- | +| Release prerequisites | ✅ pass | 1 | 1 | 0 | 0 | +| Final pre-release checks | ✅ pass | 1 | 1 | 0 | 0 | +| Post-release assessment | ✅ pass | 1 | 1 | 0 | 0 | + +### Release prerequisites + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| End-to-end scans of real projects | gate | ✅ pass | 2 instances passed. | + +### Final pre-release checks + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| Release files match their checksums | gate | ✅ pass | 23 downloaded assets match SHA256SUMS, which lists 23 files. | + +### Post-release assessment + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| Repeated scan speed and stability | advisory | ✅ pass | canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 412 ms, warm median 288 ms). | + diff --git a/internal/assurance/testdata/golden/mixed-failure.report.json b/internal/assurance/testdata/golden/mixed-failure.report.json new file mode 100644 index 00000000..7b7aa4e7 --- /dev/null +++ b/internal/assurance/testdata/golden/mixed-failure.report.json @@ -0,0 +1,482 @@ +{ + "schema_version": "bomly.assurance-report/v1", + "generated_at": "2026-08-19T12:00:00Z", + "release": { + "tag": "v9.9.9", + "version": "9.9.9", + "commit": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + }, + "verdict": { + "overall": "fail", + "checks": 3, + "passed": 0, + "failed": 1, + "degraded": 1, + "skipped": 0, + "missing": 1, + "gates_failed": [ + "release-checksums" + ], + "advisories_failed": [ + "perf-samples" + ], + "missing_checks": [ + "smoke" + ] + }, + "stages": [ + { + "id": "prerequisites", + "title": "Release prerequisites", + "status": "missing", + "verdict": { + "overall": "missing", + "checks": 1, + "passed": 0, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 1, + "missing_checks": [ + "smoke" + ] + }, + "check_ids": [ + "smoke" + ] + }, + { + "id": "pre-release", + "title": "Final pre-release checks", + "status": "fail", + "verdict": { + "overall": "fail", + "checks": 1, + "passed": 0, + "failed": 1, + "degraded": 0, + "skipped": 0, + "missing": 0, + "gates_failed": [ + "release-checksums" + ] + }, + "check_ids": [ + "release-checksums" + ] + }, + { + "id": "post-release", + "title": "Post-release assessment", + "status": "degraded", + "verdict": { + "overall": "degraded", + "checks": 1, + "passed": 0, + "failed": 0, + "degraded": 1, + "skipped": 0, + "missing": 0, + "advisories_failed": [ + "perf-samples" + ] + }, + "check_ids": [ + "perf-samples" + ] + } + ], + "areas": [ + { + "id": "end-to-end", + "title": "End-to-end behavior", + "description": "Running the real command line against real projects.", + "status": "missing", + "verdict": { + "overall": "missing", + "checks": 1, + "passed": 0, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 1, + "missing_checks": [ + "smoke" + ] + }, + "check_ids": [ + "smoke" + ], + "evidence_ids": [ + "graph-go" + ] + }, + { + "id": "release-integrity", + "title": "Release integrity", + "description": "Whether the published files are complete and unmodified.", + "status": "fail", + "verdict": { + "overall": "fail", + "checks": 1, + "passed": 0, + "failed": 1, + "degraded": 0, + "skipped": 0, + "missing": 0, + "gates_failed": [ + "release-checksums" + ] + }, + "check_ids": [ + "release-checksums" + ] + }, + { + "id": "performance", + "title": "Speed and stability", + "description": "How long a repeated scan takes and whether it returns the same answer.", + "status": "degraded", + "verdict": { + "overall": "degraded", + "checks": 1, + "passed": 0, + "failed": 0, + "degraded": 1, + "skipped": 0, + "missing": 0, + "advisories_failed": [ + "perf-samples" + ] + }, + "check_ids": [ + "perf-samples" + ] + } + ], + "checks": [ + { + "id": "perf-samples", + "title": "Repeated scan speed and stability", + "area": "performance", + "stage": "post-release", + "level": "advisory", + "description": "Runs the same scan several times and records timing.", + "status": "degraded", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms).", + "duration_ms": 61000, + "source": { + "workflow": "assurance-assessment.yml", + "job": "perf-samples" + }, + "instances": [ + { + "name": "default", + "status": "degraded", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms).", + "duration_ms": 61000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/11", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "cold_median_ms": 705, + "peak_memory_bytes": 120586240, + "samples_per_mode": 5, + "warm_median_ms": 402 + } + } + ], + "metrics": { + "cold_median_ms": 705, + "peak_memory_bytes": 120586240, + "samples_per_mode": 5, + "warm_median_ms": 402 + }, + "reproduce": [ + [ + "make", + "benchmark-samples" + ] + ], + "proves": [ + "Repeated runs produced identical normalized output." + ], + "limitations": [ + "Timings are observations from one machine, not limits." + ] + }, + { + "id": "release-checksums", + "title": "Release files match their checksums", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Hashes every release file and compares it with SHA256SUMS.", + "status": "fail", + "summary": "1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz.", + "duration_ms": 3900, + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "instances": [ + { + "name": "default", + "status": "fail", + "summary": "1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz.", + "duration_ms": 3900, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/10", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "listed": 23, + "mismatched": 1, + "verified": 22 + }, + "details": [ + { + "name": "bomly_9.9.9_linux_amd64.tar.gz", + "status": "fail", + "note": "hash does not match SHA256SUMS" + } + ] + } + ], + "metrics": { + "listed": 23, + "mismatched": 1, + "verified": 22 + }, + "reproduce": [ + [ + "sha256sum", + "--check", + "SHA256SUMS" + ] + ], + "proves": [ + "Every published file hashes to the recorded value." + ], + "limitations": [ + "Checksums prove the files were not altered after they were built." + ] + }, + { + "id": "smoke", + "title": "End-to-end scans of real projects", + "area": "end-to-end", + "stage": "prerequisites", + "level": "gate", + "description": "Runs scan against pinned public repositories.", + "status": "missing", + "summary": "1 instance passed, 1 instance reported nothing.", + "duration_ms": 60000, + "source": { + "workflow": "smoke.yml", + "job": "smoke" + }, + "instances": [ + { + "name": "go", + "status": "pass", + "summary": "18 of 18 tests passed.", + "duration_ms": 60000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/9", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "tests_failed": 0, + "tests_passed": 18, + "tests_skipped": 0, + "tests_total": 18 + } + } + ], + "missing_instances": [ + "node" + ], + "metrics": { + "tests_failed": 0, + "tests_passed": 18, + "tests_skipped": 0, + "tests_total": 18 + }, + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "proves": [ + "Each ecosystem produces the dependency graph recorded in its golden file." + ], + "limitations": [ + "Golden files pin one revision of one example project per ecosystem." + ] + } + ], + "evidence": [ + { + "id": "graph-go", + "title": "Go module graph", + "area": "end-to-end", + "description": "Scans a pinned Go example project and compares the whole dependency graph with a recorded one.", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "go", + "status": "pass", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "artifacts": [ + { + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", + "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + } + ], + "proves": [ + "The Go detector resolves a build-tool-backed module graph." + ], + "limitations": [ + "The result depends on a compatible Go toolchain." + ] + } + ], + "coverage": { + "ecosystems": [ + { + "name": "Go", + "status": "pass" + }, + { + "name": "JavaScript", + "status": "missing" + } + ] + }, + "trends": { + "previous_tag": "v9.9.9", + "metrics": [ + { + "check_id": "perf-samples", + "metric": "cold_median_ms", + "previous": 412, + "current": 705, + "delta": 293, + "delta_pct": 71.11650485436894, + "better": "lower" + }, + { + "check_id": "perf-samples", + "metric": "peak_memory_bytes", + "previous": 91234304, + "current": 120586240, + "delta": 29351936, + "delta_pct": 32.172039148783334, + "better": "lower" + }, + { + "check_id": "perf-samples", + "metric": "warm_median_ms", + "previous": 288, + "current": 402, + "delta": 114, + "delta_pct": 39.58333333333333, + "better": "lower" + }, + { + "check_id": "release-checksums", + "metric": "mismatched", + "previous": 0, + "current": 1, + "delta": 1, + "better": "neutral" + }, + { + "check_id": "release-checksums", + "metric": "verified", + "previous": 23, + "current": 22, + "delta": -1, + "delta_pct": -4.3478260869565215, + "better": "neutral" + }, + { + "check_id": "smoke", + "metric": "tests_passed", + "previous": 42, + "current": 18, + "delta": -24, + "delta_pct": -57.14285714285714, + "better": "higher" + }, + { + "check_id": "smoke", + "metric": "tests_total", + "previous": 42, + "current": 18, + "delta": -24, + "delta_pct": -57.14285714285714, + "better": "higher" + } + ], + "changed": [ + { + "check_id": "perf-samples", + "previous": "pass", + "current": "degraded" + }, + { + "check_id": "release-checksums", + "previous": "pass", + "current": "fail" + }, + { + "check_id": "smoke", + "previous": "pass", + "current": "missing" + } + ] + }, + "unknown_results": [ + { + "id": "mystery-check", + "stage": "post-release", + "status": "pass" + } + ], + "environment": { + "runners": [ + { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + } + ], + "generated_by": "assurance-test" + } +} diff --git a/internal/assurance/testdata/golden/mixed-failure.summary.md b/internal/assurance/testdata/golden/mixed-failure.summary.md new file mode 100644 index 00000000..dfebf9b8 --- /dev/null +++ b/internal/assurance/testdata/golden/mixed-failure.summary.md @@ -0,0 +1,46 @@ +## ❌ Release assurance v9.9.9 + +0 of 3 checks passed; 1 failed and 1 reported nothing. + +| Stage | Result | Checks | Passed | Failed | Missing | +| --- | --- | --- | --- | --- | --- | +| Release prerequisites | ❓ missing | 1 | 0 | 0 | 1 | +| Final pre-release checks | ❌ fail | 1 | 0 | 1 | 0 | +| Post-release assessment | ⚠️ degraded | 1 | 0 | 0 | 0 | + +### Release prerequisites + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| End-to-end scans of real projects | gate | ❓ missing | 1 instance passed, 1 instance reported nothing. | + +### Final pre-release checks + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| Release files match their checksums | gate | ❌ fail | 1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz. | + +### Post-release assessment + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| Repeated scan speed and stability | advisory | ⚠️ degraded | canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms). | + +### Needs attention + +- ⚠️ **Repeated scan speed and stability** (advisory, post-release) — canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms). Logs: [job](https://github.com/bomly-dev/bomly-cli/actions/runs/11). +- ❌ **Release files match their checksums** (gate, pre-release) — 1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz. Logs: [job](https://github.com/bomly-dev/bomly-cli/actions/runs/10). +- ❓ **End-to-end scans of real projects** (gate, prerequisites) — 1 instance passed, 1 instance reported nothing. Missing: node. +- ❓ Result `mystery-check` is not declared in the assurance catalog. + +### Compared with v9.9.9 + +- `perf-samples`: pass → degraded +- `release-checksums`: pass → fail +- `smoke`: pass → missing +- `perf-samples` cold_median_ms: 412.00 → 705.00 (+71.1%) +- `perf-samples` peak_memory_bytes: 91234304.00 → 120586240.00 (+32.2%) +- `perf-samples` warm_median_ms: 288.00 → 402.00 (+39.6%) +- `smoke` tests_passed: 42.00 → 18.00 (-57.1%) +- `smoke` tests_total: 42.00 → 18.00 (-57.1%) + diff --git a/internal/assurance/trends.go b/internal/assurance/trends.go new file mode 100644 index 00000000..998b36cb --- /dev/null +++ b/internal/assurance/trends.go @@ -0,0 +1,85 @@ +package assurance + +import ( + "sort" + "strings" +) + +// Direction names whether a rising metric is an improvement. +const ( + betterLower = "lower" + betterHigher = "higher" + betterNeutral = "neutral" +) + +// lowerIsBetterSuffixes name metrics where a smaller number is an improvement. +var lowerIsBetterSuffixes = []string{"_ms", "_bytes", "_seconds", "_failed"} + +// higherIsBetterSuffixes name metrics where a larger number is an improvement. +var higherIsBetterSuffixes = []string{"_passed", "_completed", "_total", "targets", "cases", "assets", "checks"} + +// metricDirection reports whether higher or lower is better for a metric. +func metricDirection(name string) string { + for _, suffix := range lowerIsBetterSuffixes { + if strings.HasSuffix(name, suffix) { + return betterLower + } + } + for _, suffix := range higherIsBetterSuffixes { + if strings.HasSuffix(name, suffix) { + return betterHigher + } + } + return betterNeutral +} + +// buildTrends compares the current report against the previous release's report +// and records metric deltas plus every check whose status changed. +func buildTrends(previous, current Report) *Trends { + trends := &Trends{PreviousTag: previous.Release.Tag} + previousChecks := make(map[string]ReportCheck, len(previous.Checks)) + for _, check := range previous.Checks { + previousChecks[check.ID] = check + } + for _, check := range current.Checks { + before, exists := previousChecks[check.ID] + if !exists { + continue + } + if before.Status != check.Status { + trends.Changed = append(trends.Changed, StatusChange{ + CheckID: check.ID, Previous: before.Status, Current: check.Status, + }) + } + names := make([]string, 0, len(check.Metrics)) + for name := range check.Metrics { + if _, present := before.Metrics[name]; present { + names = append(names, name) + } + } + sort.Strings(names) + for _, name := range names { + was := before.Metrics[name] + now := check.Metrics[name] + if was == now { + continue + } + trend := MetricTrend{ + CheckID: check.ID, Metric: name, Previous: was, Current: now, + Delta: now - was, Better: metricDirection(name), + } + if was != 0 { + trend.DeltaPct = (now - was) / was * 100 + } + trends.Metrics = append(trends.Metrics, trend) + } + } + sort.Slice(trends.Changed, func(i, j int) bool { return trends.Changed[i].CheckID < trends.Changed[j].CheckID }) + sort.Slice(trends.Metrics, func(i, j int) bool { + if trends.Metrics[i].CheckID != trends.Metrics[j].CheckID { + return trends.Metrics[i].CheckID < trends.Metrics[j].CheckID + } + return trends.Metrics[i].Metric < trends.Metrics[j].Metric + }) + return trends +} diff --git a/internal/tools/publicevidence/main.go b/internal/tools/publicevidence/main.go deleted file mode 100644 index 441cbc7d..00000000 --- a/internal/tools/publicevidence/main.go +++ /dev/null @@ -1,349 +0,0 @@ -// Command publicevidence validates and displays Bomly's public evidence -// catalog. -package main - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "os" - "path/filepath" - "regexp" - "sort" - "strings" -) - -const ( - catalogSchema = "bomly.public-evidence/v1" - defaultCatalog = "test/evidence/cases.json" -) - -var ( - caseIDPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) - revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) - hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) - containerDigestPattern = regexp.MustCompile(`@sha256:[0-9a-f]{64}$`) -) - -type catalog struct { - SchemaVersion string `json:"schema_version"` - Cases []evidenceCase `json:"cases"` -} - -type evidenceCase struct { - ID string `json:"id"` - Title string `json:"title"` - Area string `json:"area"` - EvidenceLevel string `json:"evidence_level"` - Inputs []input `json:"inputs"` - RequiredTools []string `json:"required_tools,omitempty"` - Reproduce [][]string `json:"reproduce"` - Evidence []artifact `json:"evidence"` - Proves []string `json:"proves"` - Limitations []string `json:"limitations"` -} - -type input struct { - Kind string `json:"kind"` - Location string `json:"location"` - Ref string `json:"ref,omitempty"` - Revision string `json:"revision,omitempty"` - SHA256 string `json:"sha256,omitempty"` -} - -type artifact struct { - Path string `json:"path"` - SHA256 string `json:"sha256"` -} - -func main() { - catalogPath := flag.String("catalog", defaultCatalog, "path to the public evidence catalog") - caseID := flag.String("case", "", "show one evidence case") - flag.Parse() - - root, err := repositoryRoot() - if err != nil { - exitError(err) - } - resolvedCatalog := resolveCatalogPath(root, *catalogPath) - loaded, err := loadCatalog(resolvedCatalog) - if err != nil { - exitError(err) - } - if err := validateCatalog(root, loaded); err != nil { - exitError(err) - } - - selected := loaded.Cases - if *caseID != "" { - selected = nil - for _, current := range loaded.Cases { - if current.ID == *caseID { - selected = []evidenceCase{current} - break - } - } - if len(selected) == 0 { - exitError(fmt.Errorf("unknown evidence case %q", *caseID)) - } - } - printCases(selected) -} - -func repositoryRoot() (string, error) { - current, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("resolve working directory: %w", err) - } - for { - if info, statErr := os.Stat(filepath.Join(current, "go.mod")); statErr == nil && !info.IsDir() { - return current, nil - } - parent := filepath.Dir(current) - if parent == current { - return "", errors.New("find repository root: go.mod not found") - } - current = parent - } -} - -func resolveCatalogPath(root, catalogPath string) string { - resolved := filepath.FromSlash(catalogPath) - if filepath.IsAbs(resolved) { - return resolved - } - return filepath.Join(root, resolved) -} - -func loadCatalog(path string) (catalog, error) { - file, err := os.Open(path) - if err != nil { - return catalog{}, fmt.Errorf("open evidence catalog: %w", err) - } - defer file.Close() - - decoder := json.NewDecoder(io.LimitReader(file, 2<<20)) - decoder.DisallowUnknownFields() - var loaded catalog - if err := decoder.Decode(&loaded); err != nil { - return catalog{}, fmt.Errorf("decode evidence catalog: %w", err) - } - if err := ensureJSONEOF(decoder); err != nil { - return catalog{}, err - } - return loaded, nil -} - -func ensureJSONEOF(decoder *json.Decoder) error { - var extra any - err := decoder.Decode(&extra) - if errors.Is(err, io.EOF) { - return nil - } - if err == nil { - return errors.New("decode evidence catalog: multiple JSON values") - } - return fmt.Errorf("decode evidence catalog trailing data: %w", err) -} - -func validateCatalog(root string, loaded catalog) error { - if loaded.SchemaVersion != catalogSchema { - return fmt.Errorf("unsupported evidence catalog schema %q", loaded.SchemaVersion) - } - if len(loaded.Cases) == 0 { - return errors.New("evidence catalog contains no cases") - } - seen := make(map[string]struct{}, len(loaded.Cases)) - previous := "" - for index, current := range loaded.Cases { - if !caseIDPattern.MatchString(current.ID) { - return fmt.Errorf("case %d has invalid id %q", index+1, current.ID) - } - if _, exists := seen[current.ID]; exists { - return fmt.Errorf("duplicate evidence case %q", current.ID) - } - seen[current.ID] = struct{}{} - if previous != "" && current.ID < previous { - return fmt.Errorf("evidence cases are not sorted: %q follows %q", current.ID, previous) - } - previous = current.ID - } - for _, current := range loaded.Cases { - if err := validateCase(root, current); err != nil { - return fmt.Errorf("case %q: %w", current.ID, err) - } - } - return nil -} - -func validateCase(root string, current evidenceCase) error { - if strings.TrimSpace(current.Title) == "" || strings.TrimSpace(current.Area) == "" { - return errors.New("title and area are required") - } - switch current.EvidenceLevel { - case "deterministic", "pinned-input", "live-service", "manual-assurance", "snapshot": - default: - return fmt.Errorf("unsupported evidence level %q", current.EvidenceLevel) - } - if len(current.Inputs) == 0 { - return errors.New("at least one input is required") - } - for _, item := range current.Inputs { - if err := validateInput(root, current.EvidenceLevel, item); err != nil { - return err - } - } - if len(current.Reproduce) == 0 { - return errors.New("at least one reproduction command is required") - } - for index, command := range current.Reproduce { - if len(command) == 0 { - return fmt.Errorf("reproduction command %d is empty", index+1) - } - for _, argument := range command { - if argument == "" { - return fmt.Errorf("reproduction command %d contains an empty argument", index+1) - } - } - } - if len(current.Evidence) == 0 { - return errors.New("at least one evidence artifact is required") - } - for _, item := range current.Evidence { - if err := validateArtifact(root, item); err != nil { - return err - } - } - if len(current.Proves) == 0 || len(current.Limitations) == 0 { - return errors.New("proves and limitations must both be explicit") - } - for _, claim := range current.Proves { - if strings.TrimSpace(claim) == "" { - return errors.New("proves and limitations cannot contain blank entries") - } - } - for _, limitation := range current.Limitations { - if strings.TrimSpace(limitation) == "" { - return errors.New("proves and limitations cannot contain blank entries") - } - } - return nil -} - -func validateInput(root, evidenceLevel string, current input) error { - if strings.TrimSpace(current.Location) == "" { - return errors.New("input location is required") - } - switch current.Kind { - case "git": - if !revisionPattern.MatchString(current.Revision) { - return errors.New("git input requires a full lowercase commit revision") - } - case "fixture": - if !hashPattern.MatchString(current.SHA256) { - return errors.New("fixture input requires a SHA-256 hash") - } - return validateArtifact(root, artifact{Path: current.Location, SHA256: current.SHA256}) - case "container": - if current.Ref == "" { - return errors.New("container input requires an image reference") - } - if evidenceLevel == "pinned-input" && !containerDigestPattern.MatchString(current.Ref) { - return errors.New("pinned container input requires an immutable sha256 digest") - } - case "workflow": - if !hashPattern.MatchString(current.SHA256) { - return errors.New("workflow input requires a SHA-256 hash") - } - return validateArtifact(root, artifact{Path: current.Location, SHA256: current.SHA256}) - default: - return fmt.Errorf("unsupported input kind %q", current.Kind) - } - return nil -} - -func validateArtifact(root string, item artifact) error { - if !hashPattern.MatchString(item.SHA256) { - return fmt.Errorf("artifact %q has an invalid SHA-256 hash", item.Path) - } - clean := filepath.Clean(filepath.FromSlash(item.Path)) - if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { - return fmt.Errorf("artifact path %q must stay inside the repository", item.Path) - } - resolvedRoot, err := filepath.EvalSymlinks(root) - if err != nil { - return fmt.Errorf("resolve repository root: %w", err) - } - path := filepath.Join(root, clean) - resolvedPath, err := filepath.EvalSymlinks(path) - if err != nil { - return fmt.Errorf("resolve artifact %q: %w", item.Path, err) - } - relative, err := filepath.Rel(resolvedRoot, resolvedPath) - if err != nil { - return fmt.Errorf("resolve artifact %q relative to repository: %w", item.Path, err) - } - if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { - return fmt.Errorf("artifact path %q resolves outside the repository", item.Path) - } - info, err := os.Stat(resolvedPath) - if err != nil { - return fmt.Errorf("inspect artifact %q: %w", item.Path, err) - } - if !info.Mode().IsRegular() { - return fmt.Errorf("artifact %q is not a regular file", item.Path) - } - data, err := os.ReadFile(resolvedPath) - if err != nil { - return fmt.Errorf("read artifact %q: %w", item.Path, err) - } - sum := sha256.Sum256(data) - actual := hex.EncodeToString(sum[:]) - if actual != item.SHA256 { - return fmt.Errorf("artifact %q hash is %s, want %s", item.Path, actual, item.SHA256) - } - return nil -} - -func printCases(cases []evidenceCase) { - sort.Slice(cases, func(i, j int) bool { return cases[i].ID < cases[j].ID }) - fmt.Printf("Verified %d public evidence case(s).\n", len(cases)) - for _, current := range cases { - fmt.Printf("\n%s — %s\n", current.ID, current.Title) - fmt.Printf(" Area: %s; evidence: %s\n", current.Area, current.EvidenceLevel) - for _, command := range current.Reproduce { - fmt.Printf(" Reproduce: %s\n", shellCommand(command)) - } - for _, limitation := range current.Limitations { - fmt.Printf(" Limitation: %s\n", limitation) - } - } -} - -func shellCommand(command []string) string { - quoted := make([]string, len(command)) - for index, argument := range command { - if argument != "" && strings.IndexFunc(argument, func(r rune) bool { - if (r >= 'a' && r <= 'z') || - (r >= 'A' && r <= 'Z') || - (r >= '0' && r <= '9') { - return false - } - return !strings.ContainsRune("@%_+=:,./-", r) - }) == -1 { - quoted[index] = argument - continue - } - quoted[index] = "'" + strings.ReplaceAll(argument, "'", "'\"'\"'") + "'" - } - return strings.Join(quoted, " ") -} - -func exitError(err error) { - fmt.Fprintln(os.Stderr, "public evidence:", err) - os.Exit(1) -} diff --git a/internal/tools/publicevidence/main_test.go b/internal/tools/publicevidence/main_test.go deleted file mode 100644 index 4b76beb4..00000000 --- a/internal/tools/publicevidence/main_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package main - -import ( - "crypto/sha256" - "encoding/hex" - "os" - "path/filepath" - "runtime" - "strings" - "testing" -) - -func TestLoadAndValidateCatalog(t *testing.T) { - root := t.TempDir() - artifactPath := filepath.Join(root, "result.json") - data := []byte(`{"ok":true}`) - if err := os.WriteFile(artifactPath, data, 0o600); err != nil { - t.Fatal(err) - } - sum := sha256.Sum256(data) - hash := hex.EncodeToString(sum[:]) - catalogPath := filepath.Join(root, "cases.json") - document := `{ - "schema_version": "bomly.public-evidence/v1", - "cases": [{ - "id": "example-case", - "title": "Example", - "area": "graph", - "evidence_level": "deterministic", - "inputs": [{"kind": "fixture", "location": "result.json", "sha256": "` + hash + `"}], - "reproduce": [["go", "test", "./..."]], - "evidence": [{"path": "result.json", "sha256": "` + hash + `"}], - "proves": ["The example succeeds."], - "limitations": ["The example covers one input."] - }] -}` - if err := os.WriteFile(catalogPath, []byte(document), 0o600); err != nil { - t.Fatal(err) - } - - loaded, err := loadCatalog(catalogPath) - if err != nil { - t.Fatal(err) - } - if err := validateCatalog(root, loaded); err != nil { - t.Fatal(err) - } - - loaded.Cases[0].Proves = []string{" "} - if err := validateCatalog(root, loaded); err == nil || !strings.Contains(err.Error(), "blank entries") { - t.Fatalf("validateCatalog() blank proof error = %v", err) - } - loaded.Cases[0].Proves = []string{"The example succeeds."} - loaded.Cases[0].Limitations = []string{"\t"} - if err := validateCatalog(root, loaded); err == nil || !strings.Contains(err.Error(), "blank entries") { - t.Fatalf("validateCatalog() blank limitation error = %v", err) - } -} - -func TestValidateCatalogRejectsUnpinnedGitAndChangedArtifact(t *testing.T) { - root := t.TempDir() - if err := os.WriteFile(filepath.Join(root, "result.json"), []byte("{}"), 0o600); err != nil { - t.Fatal(err) - } - current := catalog{ - SchemaVersion: catalogSchema, - Cases: []evidenceCase{{ - ID: "git-case", - Title: "Git case", - Area: "graph", - EvidenceLevel: "pinned-input", - Inputs: []input{{ - Kind: "git", - Location: "https://github.com/example/project", - Revision: "main", - }}, - Reproduce: [][]string{{"go", "test", "./..."}}, - Evidence: []artifact{{Path: "result.json", SHA256: strings.Repeat("0", 64)}}, - Proves: []string{"A result."}, - Limitations: []string{"One input."}, - }}, - } - if err := validateCatalog(root, current); err == nil || !strings.Contains(err.Error(), "full lowercase commit") { - t.Fatalf("validateCatalog() error = %v", err) - } - - current.Cases[0].Inputs[0].Revision = strings.Repeat("a", 40) - if err := validateCatalog(root, current); err == nil || !strings.Contains(err.Error(), "artifact") { - t.Fatalf("validateCatalog() error = %v", err) - } -} - -func TestValidateCatalogRejectsUnsortedAndUnknownFields(t *testing.T) { - root := t.TempDir() - current := catalog{ - SchemaVersion: catalogSchema, - Cases: []evidenceCase{ - {ID: "z-case"}, - {ID: "a-case"}, - }, - } - if err := validateCatalog(root, current); err == nil || !strings.Contains(err.Error(), "not sorted") { - t.Fatalf("validateCatalog() error = %v", err) - } - - path := filepath.Join(root, "cases.json") - if err := os.WriteFile(path, []byte(`{"schema_version":"bomly.public-evidence/v1","cases":[],"extra":true}`), 0o600); err != nil { - t.Fatal(err) - } - if _, err := loadCatalog(path); err == nil || !strings.Contains(err.Error(), "unknown field") { - t.Fatalf("loadCatalog() error = %v", err) - } -} - -func TestValidateInputRequiresDigestForPinnedContainer(t *testing.T) { - tagged := input{Kind: "container", Location: "Docker Hub", Ref: "alpine:3.20"} - if err := validateInput(t.TempDir(), "pinned-input", tagged); err == nil || - !strings.Contains(err.Error(), "immutable sha256 digest") { - t.Fatalf("validateInput() tagged pinned container error = %v", err) - } - if err := validateInput(t.TempDir(), "snapshot", tagged); err != nil { - t.Fatalf("validateInput() snapshot tag error = %v", err) - } - digested := tagged - digested.Ref = "alpine@sha256:" + strings.Repeat("a", 64) - if err := validateInput(t.TempDir(), "pinned-input", digested); err != nil { - t.Fatalf("validateInput() digest error = %v", err) - } -} - -func TestValidateArtifactRejectsSymlinkOutsideRepository(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("symlink creation requires elevated privileges on Windows") - } - root := t.TempDir() - data := []byte("outside") - outside := filepath.Join(t.TempDir(), "result.json") - if err := os.WriteFile(outside, data, 0o600); err != nil { - t.Fatal(err) - } - link := filepath.Join(root, "result.json") - if err := os.Symlink(outside, link); err != nil { - t.Fatal(err) - } - sum := sha256.Sum256(data) - err := validateArtifact(root, artifact{ - Path: "result.json", - SHA256: hex.EncodeToString(sum[:]), - }) - if err == nil || !strings.Contains(err.Error(), "resolves outside the repository") { - t.Fatalf("validateArtifact() error = %v", err) - } -} - -func TestResolveCatalogPathHonorsAbsolutePath(t *testing.T) { - root := t.TempDir() - absolute := filepath.Join(t.TempDir(), "cases.json") - if got := resolveCatalogPath(root, absolute); got != absolute { - t.Fatalf("resolveCatalogPath() absolute = %q, want %q", got, absolute) - } - wantRelative := filepath.Join(root, "test", "evidence", "cases.json") - if got := resolveCatalogPath(root, "test/evidence/cases.json"); got != wantRelative { - t.Fatalf("resolveCatalogPath() relative = %q, want %q", got, wantRelative) - } -} - -func TestShellCommandQuotesPatterns(t *testing.T) { - got := shellCommand([]string{"go", "test", "-run", "TestScan$/scan-npm$"}) - want := "go test -run 'TestScan$/scan-npm$'" - if got != want { - t.Fatalf("shellCommand() = %q, want %q", got, want) - } -} diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index c5d3f8e6..a2945d2c 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -3,10 +3,19 @@ set -euo pipefail FUZZTIME="${FUZZTIME:-60s}" +# When FUZZ_RESULTS_JSONL is set, every target is attempted and one JSON line +# per target is written to that file, so the release assurance framework can +# record the whole run instead of stopping at the first failure. The script +# still exits non-zero when any target failed. +FUZZ_RESULTS_JSONL="${FUZZ_RESULTS_JSONL:-}" + # The SDK's own fuzz targets (package URL canonicalization, graph/registry # transport JSON) moved with the sdk package to the bomly-sdk repository and # run there. targets=( + "github.com/bomly-dev/bomly-cli/internal/assurance FuzzParseCatalog" + "github.com/bomly-dev/bomly-cli/internal/assurance FuzzParseCheckResult" + "github.com/bomly-dev/bomly-cli/internal/assurance FuzzParseGoTestEvents" "github.com/bomly-dev/bomly-cli/internal/config FuzzLoadFile" "github.com/bomly-dev/bomly-cli/internal/detectors/cargo FuzzDepGraphFromCargoLock" "github.com/bomly-dev/bomly-cli/internal/detectors/cargo FuzzDepGraphFromCargoLockWorkspace" @@ -38,9 +47,33 @@ targets=( "github.com/bomly-dev/bomly-cli/internal/plugin FuzzPluginPathSanitizers" ) +if [ -n "${FUZZ_RESULTS_JSONL}" ]; then + : > "${FUZZ_RESULTS_JSONL}" +fi + +failures=0 for target in "${targets[@]}"; do package="${target%% *}" fuzz="${target#* }" echo "==> go test ${package} -run=^$ -fuzz=^${fuzz}$ -fuzztime=${FUZZTIME}" - go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" + started="$(date -u +%s)" + status=0 + if [ -n "${FUZZ_RESULTS_JSONL}" ]; then + go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" || status=$? + else + go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" + fi + if [ "${status}" -ne 0 ]; then + failures=$((failures + 1)) + fi + if [ -n "${FUZZ_RESULTS_JSONL}" ]; then + printf '{"name":"%s %s","exit_code":%s,"duration_s":%s}\n' \ + "${package##*/}" "${fuzz}" "${status}" "$(( $(date -u +%s) - started ))" \ + >> "${FUZZ_RESULTS_JSONL}" + fi done + +if [ "${failures}" -ne 0 ]; then + echo "${failures} fuzz target(s) failed" >&2 + exit 1 +fi diff --git a/test/assurance/BENCHMARK_RUNS.md b/test/assurance/BENCHMARK_RUNS.md index 10ee3702..b0f7fe6d 100644 --- a/test/assurance/BENCHMARK_RUNS.md +++ b/test/assurance/BENCHMARK_RUNS.md @@ -1,6 +1,7 @@ # Measuring speed and stability -`make benchmark-samples` measures the same offline Bomly scan ten times: +`make benchmark-samples` (the `perfrun` tool under `internal/assurance/`) +measures the same offline Bomly scan ten times: - five runs start with an empty cache; - five runs share a cache, like repeated scans normally do. @@ -38,12 +39,13 @@ checksum. This comparison format is named ## Checking supported systems -The `Portable stability assurance` workflow runs only when someone starts it -from GitHub Actions. It: +The `Portable stability assurance` workflow runs when someone starts it from +GitHub Actions, and as part of the `Release prerequisites` stage before a +version is tagged. It: - runs the Go unit tests twice on Linux, macOS, and Windows; -- runs the Java-related unit tests ten times to catch intermittent failures; -- runs all Go unit tests on Linux five more times; +- runs the Java-related unit tests ten times to catch intermittent failures, + because those are the suites where intermittent failures have appeared; - builds both Bomly binaries for every supported Linux, macOS, and Windows processor target. @@ -57,10 +59,12 @@ when investigating platform-specific or intermittent failures. ## Reading a portable run -Open the workflow run's **Summary** page first. The overall section explains -what ran and whether each area passed. Each platform also has a short section -showing how many test runs completed and which run failed, if any. The Linux -section does the same for repeated tests and release builds. +Open the workflow run's **Summary** page first. Each check writes its own +section there — the repeated suites per platform, the repeated Java suites, the +repeated complete suite, and the cross-build matrix — with the number of runs +planned and completed and the exact point of failure. Those sections are +rendered from the same check results the release assurance report is built +from, so the summary and the published report always agree. If something fails, open the named job and failed step for the test or build output. To show only failed logs with the GitHub CLI, run: diff --git a/test/assurance/PARSER_FUZZING.md b/test/assurance/PARSER_FUZZING.md index 7d69d143..8aa28bba 100644 --- a/test/assurance/PARSER_FUZZING.md +++ b/test/assurance/PARSER_FUZZING.md @@ -25,6 +25,7 @@ the reader inventory, cache behavior, and intentional exclusions. | Other lockfiles and manifests | Cargo, CocoaPods, Composer, Conan, Go list, Mix, NuGet lock and packages.config, Pub, Bundler, SwiftPM | | Workflow manifests | GitHub Actions workflow references | | Matcher evidence | vulnerability consolidation and advisory aliases | +| Release assurance | check-result documents, the assurance catalog, and `go test -json` streams | Seeds include valid minimal documents and malformed/truncated structures. The fuzz engine supplies invalid encodings, deep nesting, duplicate values, diff --git a/test/assurance/SBOM_INTEROPERABILITY.md b/test/assurance/SBOM_INTEROPERABILITY.md index b265a88b..0c115872 100644 --- a/test/assurance/SBOM_INTEROPERABILITY.md +++ b/test/assurance/SBOM_INTEROPERABILITY.md @@ -5,27 +5,39 @@ read the SBOM files that Bomly creates. This catches compatibility problems that Bomly's own tests might miss. The workflow uses the same checked-in sample input each time. Bomly creates an -SPDX 2.3 file and a CycloneDX 1.6 file from that input. The workflow then asks +SPDX 2.3 file and a CycloneDX 1.7 file from that input. The workflow then asks the official SPDX and CycloneDX validators to check those files. -This workflow runs only when someone starts it from GitHub Actions. It is kept -separate from normal tests because it downloads the validators and takes +It runs three ways: on a weekly schedule, when someone starts it from GitHub +Actions, and as part of the post-release assessment. In the post-release run it +downloads the binary the release actually shipped, verifies it against the +published checksum list, and validates the SBOMs that binary produces. It is +kept separate from normal tests because it downloads the validators and takes longer to run. Bomly never downloads or installs these tools during normal CLI use. +The download, generation, and validation steps live in +`internal/assurance/sbominterop`, so the same run can be reproduced locally: + +```sh +make build-full +go run ./internal/assurance/sbominterop -bomly ./bin/bomly +``` + ## Reading the result Open the workflow run's **Summary** page first. It shows: - whether validation passed; -- the version and result of each validator; -- the size and checksum of each generated file; -- any messages returned by the validators. - -If validation fails, open the **Generate and validate canonical SBOMs** step -to see its full output. The summary also provides a command that downloads the -saved evidence. The downloaded `run-manifest.json` records every command, exit -code, duration, validator message, version, and checksum. +- the version and checksum of each validator; +- the result of each command that ran; +- the size and checksum of each generated file. + +If validation fails, open the **Generate and validate SBOMs** step to see its +full output. The saved evidence is attached to the run as an artifact. The +downloaded `run-manifest.json` records every command, exit code, duration, +validator message, version, and checksum, and is also what the release +assurance report is built from. ## What the workflow saves diff --git a/test/evidence/README.md b/test/evidence/README.md deleted file mode 100644 index 1c642b0a..00000000 --- a/test/evidence/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Public evidence catalog - -This directory maps Bomly claims to repeatable tests and checked-in evidence. -It is not a second test suite. The catalog points to the smoke tests, focused -unit tests, fixtures, and manually started assurance workflows that already -own each behavior. - -Run the catalog check from the repository root: - -```sh -make evidence -``` - -Show one case and its exact reproduction command: - -```sh -make evidence CASE=graph-npm -``` - -The checker verifies that: - -- every remote Git input has a full commit revision; -- every fixture, workflow, and result file has the recorded SHA-256 checksum; -- every case says what it proves and what it does not prove; -- case IDs are unique and stable. - -## Evidence levels - -- `deterministic` uses checked-in inputs or local services and should produce - the same normalized result. -- `pinned-input` uses a public repository revision. Package-manager tools or - registries can still affect build-tool-backed resolution. -- `snapshot` records the normalized result of an input that can move, such as - a container tag. -- `live-service` combines a pinned project with current advisory data. It is a - dated observation because advisory services change. -- `manual-assurance` starts a GitHub Actions workflow and stores the detailed - run report as a workflow artifact. - -The catalog schema belongs only to repository assurance metadata. It does not -change Bomly's CLI, MCP, SDK, or plugin schemas. diff --git a/test/evidence/cases.json b/test/evidence/cases.json deleted file mode 100644 index 8125919e..00000000 --- a/test/evidence/cases.json +++ /dev/null @@ -1,1021 +0,0 @@ -{ - "schema_version": "bomly.public-evidence/v1", - "cases": [ - { - "id": "baseline-policy", - "title": "Finding baseline lifecycle", - "area": "policy", - "evidence_level": "deterministic", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-go-gomod", - "ref": "v1.0.0", - "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" - } - ], - "required_tools": [ - "git", - "go" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestFindingBaselineWorkflow$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/finding-baseline-workflow.golden.json", - "sha256": "8fa2b220af563a89d1fbaaab03a75ccc0952ae43251438bc27a2314301315aa0" - } - ], - "proves": [ - "A project baseline can keep a denied package finding visible with suppressed policy status." - ], - "limitations": [ - "The local matcher fixture is deterministic but does not measure advisory freshness." - ] - }, - { - "id": "container-inventory", - "title": "Container package inventory", - "area": "targets", - "evidence_level": "snapshot", - "inputs": [ - { - "kind": "container", - "location": "Docker Hub", - "ref": "alpine:3.20" - } - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestContainerScan/container-scan-alpine$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/container-scan-alpine.golden.json", - "sha256": "50f7d4426ead3af9ea5a07df7eeb77743e7aa1412260e1a665a7301f3c22ecc5" - } - ], - "proves": [ - "Bomly can inventory the operating-system packages in the checked Alpine image." - ], - "limitations": [ - "The upstream image tag can move; the checked-in golden is a snapshot, not an immutable image claim." - ] - }, - { - "id": "degraded-detector-fallback", - "title": "Visible detector fallback", - "area": "dependency-graph", - "evidence_level": "deterministic", - "inputs": [ - { - "kind": "fixture", - "location": "internal/engine/pipeline_fallback_test.go", - "sha256": "7751d6137dd39c8616857f2e0891ed4928bf2b37e72475d567c1e38262755a6d" - } - ], - "required_tools": [ - "go" - ], - "reproduce": [ - [ - "go", - "test", - "./internal/engine", - "-run", - "TestPipeline_RunRecordsFallbackWarning|TestPipeline_Run_TypesFallbackWarningsAsDegradedCoverage", - "-count=1" - ] - ], - "evidence": [ - { - "path": "internal/engine/pipeline_fallback_test.go", - "sha256": "7751d6137dd39c8616857f2e0891ed4928bf2b37e72475d567c1e38262755a6d" - } - ], - "proves": [ - "A detector fallback preserves its origin and produces a typed degraded-coverage warning." - ], - "limitations": [ - "The synthetic detector case proves orchestration behavior, not the fidelity of every concrete fallback." - ] - }, - { - "id": "graph-bun", - "title": "Bun lockfile graph", - "area": "dependency-graph", - "evidence_level": "pinned-input", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-javascript-bun", - "ref": "v1.0.0", - "revision": "358a2a920fe9c2f7e596c514f36df1c30c1ab185" - } - ], - "required_tools": [ - "git" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-bun$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-bun.golden.json", - "sha256": "c61ccae15f3d36028ec1e741436abbf036236b9e2f018393b863e98d14360fb4" - } - ], - "proves": [ - "The native Bun detector preserves the package inventory and dependency placement represented by the pinned lockfile." - ], - "limitations": [ - "This is one Bun lockfile shape and does not prove behavior for every historical Bun format." - ] - }, - { - "id": "graph-go", - "title": "Go module graph", - "area": "dependency-graph", - "evidence_level": "pinned-input", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-go-gomod", - "ref": "v1.0.0", - "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" - } - ], - "required_tools": [ - "git", - "go" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-go$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-go.golden.json", - "sha256": "728f949898446657c4987bc1202a8d00c78b118868b4945141e4db8c4c54cdea" - } - ], - "proves": [ - "The Go detector resolves a build-tool-backed module graph for the pinned repository." - ], - "limitations": [ - "The result depends on a compatible Go toolchain and the evidence available from that toolchain." - ] - }, - { - "id": "graph-maven", - "title": "Maven dependency graph", - "area": "dependency-graph", - "evidence_level": "pinned-input", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-java-maven", - "ref": "v1.0.0", - "revision": "93bb3aae614e2f2c6cb65f5ea2315846f5234150" - } - ], - "required_tools": [ - "git", - "java", - "mvn" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-maven$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-maven.golden.json", - "sha256": "a1077544af4bd637311e1767abe2a140e3ba8f0f65a64e662378e5ccca07de9b" - } - ], - "proves": [ - "The Maven detector resolves a build-tool-backed dependency graph for the pinned repository." - ], - "limitations": [ - "Artifact resolution depends on Maven repositories and a compatible Java and Maven installation." - ] - }, - { - "id": "graph-npm", - "title": "npm lockfile graph", - "area": "dependency-graph", - "evidence_level": "pinned-input", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-javascript-npm", - "ref": "v1.0.0", - "revision": "559a762aeef68b0e5c818f62dfba67abc369912f" - } - ], - "required_tools": [ - "git", - "npm" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-npm$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-npm.golden.json", - "sha256": "da252e623cdeaf88db7874f528fd9a2f204c1136c4ab0bb26e4faa943dee79c1" - } - ], - "proves": [ - "The npm detector preserves lockfile package inventory, versions, scopes, and dependency placement." - ], - "limitations": [ - "This case covers the lockfile versions present in one pinned repository." - ] - }, - { - "id": "graph-pnpm", - "title": "pnpm lockfile graph", - "area": "dependency-graph", - "evidence_level": "pinned-input", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-javascript-pnpm", - "ref": "v1.0.0", - "revision": "f1b0959f916dfb91db70c54a75b15e5b7f3d16af" - } - ], - "required_tools": [ - "git", - "npm" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-pnpm$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-pnpm.golden.json", - "sha256": "07991083b233acb7bc856e41275517a30cd5793c4b1afb94a467502374545467" - } - ], - "proves": [ - "The pnpm detector preserves lockfile package inventory and dependency placement." - ], - "limitations": [ - "This case does not cover every pnpm lockfile generation." - ] - }, - { - "id": "graph-python", - "title": "Python requirements graph", - "area": "dependency-graph", - "evidence_level": "pinned-input", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-python-pip", - "revision": "fe04c758134b95dab102e1fce10275f7d18c0cf2" - } - ], - "required_tools": [ - "git", - "pip" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-python-pip$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-python-pip.golden.json", - "sha256": "025d14ad1a3c93f3af22a4a83a358ee85b42d782f50798b285048e35e8632bf2" - } - ], - "proves": [ - "The pip detector reads the pinned requirements lock and preserves the resolved Python package graph." - ], - "limitations": [ - "Unpinned requirements and environment inspection have different fidelity and are not proven by this case." - ] - }, - { - "id": "graph-yarn", - "title": "Yarn lockfile graph", - "area": "dependency-graph", - "evidence_level": "pinned-input", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-javascript-yarn", - "ref": "v1.0.0", - "revision": "c84017b43bf0f6ea74281f4174a0c89b88b8cddf" - } - ], - "required_tools": [ - "git", - "npm" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-yarn$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-yarn.golden.json", - "sha256": "707e1ab208812a53527cc1de888d3e76c410d9b963b49c14076f023ceb5f7b90" - } - ], - "proves": [ - "The Yarn detector preserves package inventory and dependency placement represented by the pinned lockfile." - ], - "limitations": [ - "This case covers one Yarn lockfile family; fallback behavior is documented separately." - ] - }, - { - "id": "license-policy", - "title": "Complex and invalid SPDX policy", - "area": "policy", - "evidence_level": "deterministic", - "inputs": [ - { - "kind": "fixture", - "location": "internal/auditors/license/spdx_policy_matrix_test.go", - "sha256": "04d8e3528b33687a09d317b867f7c24068081c1ac60bb53cee1e0a03c0b143f2" - } - ], - "required_tools": [ - "go" - ], - "reproduce": [ - [ - "go", - "test", - "./internal/auditors/license", - "-run", - "TestLicenseAuditorComplexSPDX|TestLicenseAuditorInvalidSPDXExpressionMatrix", - "-count=1" - ] - ], - "evidence": [ - { - "path": "internal/auditors/license/spdx_policy_matrix_test.go", - "sha256": "04d8e3528b33687a09d317b867f7c24068081c1ac60bb53cee1e0a03c0b143f2" - } - ], - "proves": [ - "The license auditor evaluates AND, OR, nested, exception, custom-reference, and invalid SPDX expressions under allow and deny policy." - ], - "limitations": [ - "The tests prove policy evaluation after license data is present; they do not prove the accuracy of every upstream license source." - ] - }, - { - "id": "performance-stability", - "title": "Repeated cold and warm scan measurements", - "area": "operations", - "evidence_level": "deterministic", - "inputs": [ - { - "kind": "fixture", - "location": "test/smoke/testdata/sboms/go.spdx.json", - "sha256": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" - } - ], - "required_tools": [ - "go" - ], - "reproduce": [ - [ - "make", - "benchmark-samples" - ] - ], - "evidence": [ - { - "path": "internal/tools/benchmarkrun/main.go", - "sha256": "d99c52b52065a95b0cda323f082bbdcb04276ab7cef96f21678550154f340bef" - } - ], - "proves": [ - "The runner records five isolated cold and five shared-cache warm samples with output hashes, timing, memory, and dispersion." - ], - "limitations": [ - "Wall time and memory are machine-specific observations and are not fixed pass-or-fail limits." - ] - }, - { - "id": "persisted-risk", - "title": "Risk that persists across a version change", - "area": "diff", - "evidence_level": "deterministic", - "inputs": [ - { - "kind": "fixture", - "location": "internal/engine/diff/diff_test.go", - "sha256": "abe87f7b95e55e5cf70bd0684c36027757308f7f405a9db370a894238f16f195" - } - ], - "required_tools": [ - "go" - ], - "reproduce": [ - [ - "go", - "test", - "./internal/engine/diff", - "-run", - "TestRun_SameVulnerabilityAcrossVersionBumpPersists|TestRun_SameLicenseIssueAcrossVersionBumpPersists", - "-count=1" - ] - ], - "evidence": [ - { - "path": "internal/engine/diff/diff_test.go", - "sha256": "abe87f7b95e55e5cf70bd0684c36027757308f7f405a9db370a894238f16f195" - } - ], - "proves": [ - "A vulnerability or license finding that remains across a package version change is classified as persisted instead of one resolved and one introduced finding." - ], - "limitations": [ - "Persistence uses the canonical package-finding identity; a genuinely different advisory or rule remains a distinct finding." - ] - }, - { - "id": "portable-platforms", - "title": "Repeated unit tests and release builds", - "area": "operations", - "evidence_level": "manual-assurance", - "inputs": [ - { - "kind": "workflow", - "location": ".github/workflows/portable-assurance.yml", - "sha256": "fadc2ff37a55c3b0dbaed91d3db67745b32b6ca517075e55734cea49a231b4d2" - } - ], - "required_tools": [ - "gh" - ], - "reproduce": [ - [ - "gh", - "workflow", - "run", - "portable-assurance.yml", - "--ref", - "v0.20.0" - ] - ], - "evidence": [ - { - "path": ".github/workflows/portable-assurance.yml", - "sha256": "fadc2ff37a55c3b0dbaed91d3db67745b32b6ca517075e55734cea49a231b4d2" - } - ], - "proves": [ - "The manual workflow repeats Go unit tests on Linux, macOS, and Windows and cross-builds every release target." - ], - "limitations": [ - "This workflow runs unit tests and builds; it does not run smoke tests against remote repositories or services." - ] - }, - { - "id": "reachability-go", - "title": "Go vulnerability reachability", - "area": "reachability", - "evidence_level": "live-service", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-go-gomod", - "ref": "v1.0.0", - "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" - } - ], - "required_tools": [ - "git", - "go" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-go-reachability$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-go-reachability.golden.json", - "sha256": "2d1db6318d75c491b8866ff9643ae80635702591046e4be89c362ca1b828a4cd" - } - ], - "proves": [ - "The Go analyzer attaches named analyzer, tier, status, and reason evidence to matched vulnerabilities." - ], - "limitations": [ - "Advisories come from live services and may change; an unreachable result is not proof that a package is safe." - ] - }, - { - "id": "reachability-java", - "title": "Java package reachability", - "area": "reachability", - "evidence_level": "live-service", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-java-maven", - "ref": "v1.0.0", - "revision": "93bb3aae614e2f2c6cb65f5ea2315846f5234150" - } - ], - "required_tools": [ - "git", - "java", - "mvn" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-java-maven-reachability$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-java-maven-reachability.golden.json", - "sha256": "b3fa7485ebcee27a4836f7ad6b318ca53c238b24fa303059dd518007fc8ecb0f" - } - ], - "proves": [ - "The Java analyzer separates package-reachable and package-unreachable vulnerability evidence." - ], - "limitations": [ - "This is package-tier evidence, not symbol-level proof, and live advisory results may change." - ] - }, - { - "id": "reachability-node", - "title": "JavaScript package reachability", - "area": "reachability", - "evidence_level": "live-service", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-javascript-npm", - "ref": "v1.0.0", - "revision": "559a762aeef68b0e5c818f62dfba67abc369912f" - } - ], - "required_tools": [ - "git", - "npm" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-npm-reachability$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-npm-reachability.golden.json", - "sha256": "2bb379a7e0b301357b24a8a1a169b805461530dac392af1908525e3ce5366760" - } - ], - "proves": [ - "The JavaScript analyzer separates package-reachable and package-unreachable vulnerability evidence." - ], - "limitations": [ - "This is package-tier evidence, dynamic loading can reduce confidence, and live advisory results may change." - ] - }, - { - "id": "reachability-python", - "title": "Python package reachability", - "area": "reachability", - "evidence_level": "live-service", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-python-pip", - "revision": "fe04c758134b95dab102e1fce10275f7d18c0cf2" - } - ], - "required_tools": [ - "git", - "pip" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-python-pip-reachability$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-python-pip-reachability.golden.json", - "sha256": "4fbf082566a5562308edd71fd86f155e3eaf1fc7ed0f6dc96c0f2f7778ca3f90" - } - ], - "proves": [ - "The Python analyzer separates package-reachable and package-unreachable vulnerability evidence." - ], - "limitations": [ - "This is package-tier evidence, reflective imports can reduce confidence, and live advisory results may change." - ] - }, - { - "id": "remediation-read-only", - "title": "Canonical read-only remediation guidance", - "area": "remediation", - "evidence_level": "deterministic", - "inputs": [ - { - "kind": "fixture", - "location": "internal/remediation/derive_test.go", - "sha256": "4ab715423343aad70cff5ea4a406b642eb687f2552afb894a9edfba4244c7582" - } - ], - "required_tools": [ - "go" - ], - "reproduce": [ - [ - "go", - "test", - "./internal/remediation", - "-run", - "TestDerivePackageRemediation|TestDeriveBuildsCanonicalOccurrenceSuggestions", - "-count=1" - ] - ], - "evidence": [ - { - "path": "internal/remediation/derive_test.go", - "sha256": "4ab715423343aad70cff5ea4a406b642eb687f2552afb894a9edfba4244c7582" - } - ], - "proves": [ - "One central component derives fix status, recommended versions, occurrence actions, and detector advice without applying changes." - ], - "limitations": [ - "Suggestions are read-only evidence and are not guaranteed to work in every checkout." - ] - }, - { - "id": "sbom-cyclonedx-ingest", - "title": "CycloneDX 1.6 ingestion", - "area": "sbom", - "evidence_level": "deterministic", - "inputs": [ - { - "kind": "fixture", - "location": "test/smoke/testdata/sboms/go.cdx.json", - "sha256": "78454a7207f98ba0c8d35df8a257f9185c313d22691cc90a06f11809021bed93" - } - ], - "required_tools": [ - "go" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-sbom-cyclonedx$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json", - "sha256": "54f5e33a47d884e1705d22102dd4745b4474295d0cc56da827bc3f172792dabb" - } - ], - "proves": [ - "Bomly ingests the checked CycloneDX 1.6 dependency graph." - ], - "limitations": [ - "The fixture covers one document shape and does not imply lossless conversion from every producer." - ] - }, - { - "id": "sbom-interoperability", - "title": "External SBOM validation", - "area": "sbom", - "evidence_level": "manual-assurance", - "inputs": [ - { - "kind": "workflow", - "location": ".github/workflows/sbom-interoperability.yml", - "sha256": "3d379300ab8e7bc02f6859c088ba06153fcbd2dc3cf159a62d890d1abda24f1a" - } - ], - "required_tools": [ - "gh" - ], - "reproduce": [ - [ - "gh", - "workflow", - "run", - "sbom-interoperability.yml", - "--ref", - "v0.20.0" - ] - ], - "evidence": [ - { - "path": ".github/workflows/sbom-interoperability.yml", - "sha256": "3d379300ab8e7bc02f6859c088ba06153fcbd2dc3cf159a62d890d1abda24f1a" - } - ], - "proves": [ - "Checksum-pinned official SPDX and CycloneDX validators can check Bomly's generated canonical SBOMs." - ], - "limitations": [ - "The workflow validates canonical fixtures, not every possible receiving tool or document conversion." - ] - }, - { - "id": "sbom-spdx-ingest", - "title": "SPDX 2.3 ingestion", - "area": "sbom", - "evidence_level": "deterministic", - "inputs": [ - { - "kind": "fixture", - "location": "test/smoke/testdata/sboms/go.spdx.json", - "sha256": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" - } - ], - "required_tools": [ - "go" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestScan$/scan-sbom-spdx$" - ] - ], - "evidence": [ - { - "path": "test/smoke/testdata/golden/scan-sbom-spdx.golden.json", - "sha256": "d49e4fc3f9c61fdb4c533116d9a5988e1841c7ccacc3675b93fb069d5ec14a00" - } - ], - "proves": [ - "Bomly ingests the checked SPDX 2.3 dependency graph." - ], - "limitations": [ - "The fixture covers one document shape and does not imply lossless conversion from every producer." - ] - }, - { - "id": "source-change-policy", - "title": "Registry-to-Git source-change review", - "area": "policy", - "evidence_level": "pinned-input", - "inputs": [ - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-javascript-npm", - "ref": "assurance/dependency-source-registry-v1", - "revision": "f6127099dad7f8b6fbaa7ed1ceb8e7b0d1e8c864" - }, - { - "kind": "git", - "location": "https://github.com/bomly-dev/example-javascript-npm", - "ref": "assurance/dependency-source-git-v1", - "revision": "96ffda21548628ce36201741b56164ca5c4405b6" - } - ], - "required_tools": [ - "git", - "npm" - ], - "reproduce": [ - [ - "go", - "test", - "-tags", - "smoke", - "./test/smoke/", - "-v", - "-count=1", - "-timeout", - "15m", - "-run", - "TestDependencyDetailRiskPolicy$" - ] - ], - "evidence": [ - { - "path": "test/smoke/audit_test.go", - "sha256": "363c2ab0ecd4bec931c18abb32706c528e22f0dd3331db2691822851d5eb5ae2" - } - ], - "proves": [ - "Diff marks a same-version registry-to-Git move for review and the package auditor can fail it when source-change policy is enabled." - ], - "limitations": [ - "A source change is a review signal, not proof of malicious intent, and unknown source evidence cannot be classified." - ] - }, - { - "id": "vulnerability-policy", - "title": "Vulnerability policy constraints", - "area": "policy", - "evidence_level": "deterministic", - "inputs": [ - { - "kind": "fixture", - "location": "internal/auditors/vulnerability/policy_matrix_test.go", - "sha256": "79b98c8c367eebe8933328980611b6d8497c369f6be5b55e57ef25405697c7a8" - } - ], - "required_tools": [ - "go" - ], - "reproduce": [ - [ - "go", - "test", - "./internal/auditors/vulnerability", - "-run", - "TestAuditorSeverityReachabilityExploitabilityAndAllowlistMatrix", - "-count=1" - ] - ], - "evidence": [ - { - "path": "internal/auditors/vulnerability/policy_matrix_test.go", - "sha256": "79b98c8c367eebe8933328980611b6d8497c369f6be5b55e57ef25405697c7a8" - } - ], - "proves": [ - "The vulnerability auditor composes severity, reachability, known exploitation, and advisory allowlists with the documented repeated-constraint behavior." - ], - "limitations": [ - "Policy can only evaluate vulnerability, reachability, and exploitation evidence that is present in the registry." - ] - } - ] -}