From f4cc3d89e2afb995ca97f21a9be1600c1a5a5368 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 19 Aug 2026 22:01:50 -0400 Subject: [PATCH 01/13] feat(assurance): framework core, catalog, and consolidation Bomly's quality checks were scattered: smoke goldens, fuzz, portable stability, SBOM interoperability, a separate public evidence catalog, and local-only performance sampling, each with its own output shape and no shared verdict. Nothing verified published release artifacts, and no release got a single readable answer to "did this one pass?". This adds the framework those checks will report into: - internal/assurance: the bomly.assurance-check/v1 result contract, the bomly.assurance-catalog/v1 catalog, report and index generation, `go test -json` and tool-manifest converters, release-asset verification, trends against the previous release, and markdown rendering for job summaries. - internal/assurance/cmd: emit, gotest, convert, verify-release, verdict, report, and catalog-validate, so no workflow step ever hand-writes a result document. - docs/assurance/catalog.json: 17 checks across the three release stages plus the 24 public evidence claims absorbed from test/evidence, each now naming the check that backs it. Consolidation: internal/tools/sbomassurance and internal/tools/benchmarkrun move under internal/assurance as sbominterop and perfrun, internal/tools/publicevidence is replaced by catalog-validate, and docs/EVIDENCE.md plus docs/evidence/* are replaced by docs/ASSURANCE.md. No workflow changes yet: this commit stands on its own under `make assurance-catalog`, `make assurance-report`, and `make test`. Co-Authored-By: Claude Fable 5 --- .gitignore | 3 + AGENTS.md | 23 +- CLAUDE.md | 23 +- Makefile | 12 +- README.md | 6 +- dev-docs/RELEASE_ASSURANCE.md | 149 +++ ...ance-is-a-catalog-and-a-result-contract.md | 52 + dev-docs/adr/README.md | 1 + docs/ASSURANCE.md | 91 ++ docs/EVIDENCE.md | 92 -- docs/README.md | 2 +- .../cases.json => docs/assurance/catalog.json | 936 ++++++++++++++++-- docs/evidence/DEPENDENCY_GRAPHS.md | 86 -- docs/evidence/POLICY_AND_GUIDANCE.md | 117 --- docs/evidence/TARGETS_AND_OPERATIONS.md | 126 --- docs/manifest.json | 13 +- internal/assurance/aggregate.go | 402 ++++++++ internal/assurance/assurance_test.go | 333 +++++++ internal/assurance/catalog.go | 495 +++++++++ internal/assurance/catalog_test.go | 187 ++++ internal/assurance/cmd/commands.go | 546 ++++++++++ internal/assurance/cmd/main.go | 265 +++++ internal/assurance/cmd/report.go | 364 +++++++ internal/assurance/contract.go | 348 +++++++ internal/assurance/convert.go | 220 ++++ internal/assurance/convert_test.go | 191 ++++ internal/assurance/gotest.go | 227 +++++ internal/assurance/parse_fuzz_test.go | 130 +++ .../perfrun}/main.go | 4 +- .../perfrun}/main_test.go | 0 internal/assurance/releaseassets.go | 338 +++++++ internal/assurance/render_markdown.go | 187 ++++ internal/assurance/report.go | 290 ++++++ .../sbominterop}/main.go | 8 +- .../sbominterop}/main_test.go | 0 internal/assurance/testdata/catalog.json | 90 ++ .../all-pass/results/perf-samples.json | 21 + .../all-pass/results/release-checksums.json | 20 + .../fixtures/all-pass/results/smoke.go.json | 29 + .../fixtures/all-pass/results/smoke.node.json | 22 + .../mixed-failure/results/mystery-check.json | 14 + .../mixed-failure/results/perf-samples.json | 21 + .../results/release-checksums.json | 27 + .../mixed-failure/results/smoke.go.json | 22 + .../testdata/golden/all-pass.report.json | 370 +++++++ .../testdata/golden/all-pass.summary.md | 28 + .../testdata/golden/mixed-failure.report.json | 473 +++++++++ .../testdata/golden/mixed-failure.summary.md | 46 + internal/assurance/trends.go | 85 ++ internal/tools/publicevidence/main.go | 349 ------- internal/tools/publicevidence/main_test.go | 173 ---- scripts/run-fuzz.sh | 3 + test/assurance/PARSER_FUZZING.md | 1 + test/evidence/README.md | 41 - 54 files changed, 7035 insertions(+), 1067 deletions(-) create mode 100644 dev-docs/RELEASE_ASSURANCE.md create mode 100644 dev-docs/adr/0035-release-assurance-is-a-catalog-and-a-result-contract.md create mode 100644 docs/ASSURANCE.md delete mode 100644 docs/EVIDENCE.md rename test/evidence/cases.json => docs/assurance/catalog.json (50%) delete mode 100644 docs/evidence/DEPENDENCY_GRAPHS.md delete mode 100644 docs/evidence/POLICY_AND_GUIDANCE.md delete mode 100644 docs/evidence/TARGETS_AND_OPERATIONS.md create mode 100644 internal/assurance/aggregate.go create mode 100644 internal/assurance/assurance_test.go create mode 100644 internal/assurance/catalog.go create mode 100644 internal/assurance/catalog_test.go create mode 100644 internal/assurance/cmd/commands.go create mode 100644 internal/assurance/cmd/main.go create mode 100644 internal/assurance/cmd/report.go create mode 100644 internal/assurance/contract.go create mode 100644 internal/assurance/convert.go create mode 100644 internal/assurance/convert_test.go create mode 100644 internal/assurance/gotest.go create mode 100644 internal/assurance/parse_fuzz_test.go rename internal/{tools/benchmarkrun => assurance/perfrun}/main.go (99%) rename internal/{tools/benchmarkrun => assurance/perfrun}/main_test.go (100%) create mode 100644 internal/assurance/releaseassets.go create mode 100644 internal/assurance/render_markdown.go create mode 100644 internal/assurance/report.go rename internal/{tools/sbomassurance => assurance/sbominterop}/main.go (97%) rename internal/{tools/sbomassurance => assurance/sbominterop}/main_test.go (100%) create mode 100644 internal/assurance/testdata/catalog.json create mode 100644 internal/assurance/testdata/fixtures/all-pass/results/perf-samples.json create mode 100644 internal/assurance/testdata/fixtures/all-pass/results/release-checksums.json create mode 100644 internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json create mode 100644 internal/assurance/testdata/fixtures/all-pass/results/smoke.node.json create mode 100644 internal/assurance/testdata/fixtures/mixed-failure/results/mystery-check.json create mode 100644 internal/assurance/testdata/fixtures/mixed-failure/results/perf-samples.json create mode 100644 internal/assurance/testdata/fixtures/mixed-failure/results/release-checksums.json create mode 100644 internal/assurance/testdata/fixtures/mixed-failure/results/smoke.go.json create mode 100644 internal/assurance/testdata/golden/all-pass.report.json create mode 100644 internal/assurance/testdata/golden/all-pass.summary.md create mode 100644 internal/assurance/testdata/golden/mixed-failure.report.json create mode 100644 internal/assurance/testdata/golden/mixed-failure.summary.md create mode 100644 internal/assurance/trends.go delete mode 100644 internal/tools/publicevidence/main.go delete mode 100644 internal/tools/publicevidence/main_test.go delete mode 100644 test/evidence/README.md diff --git a/.gitignore b/.gitignore index 55c34326..e749bd48 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ # Ignore generated SBOM files /sboms +# Ignore local assurance report previews +/.assurance + # Ignore local dependency graph benchmark artifacts /.benchmark-runs diff --git a/AGENTS.md b/AGENTS.md index 7ddd5b0d..732b0be0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,7 @@ make smoke ARGS="-update" # regenerate smoke golden files make fuzz FUZZTIME=5s # run every registered fuzz target with a short per-target budget make benchmark # run the hidden local dependency-graph benchmark make benchmark-report # analyze local benchmark artifacts with Copilot CLI -make evidence # verify the public evidence catalog (test/evidence/cases.json) +make assurance-catalog # validate the release assurance catalog (docs/assurance/catalog.json) make run ARGS="scan" # go run ./cmd/bomly make generate # regenerate config reference, JSON schemas, schema docs, support matrix, and component docs (binary-driven) ``` @@ -54,6 +54,7 @@ See [`dev-docs/ARCHITECTURE.md`](dev-docs/ARCHITECTURE.md) for full detail (the | `internal/baseline` | Portable package-finding baseline codec and audit-integrated policy-status resolver | | `internal/remediation` | Canonical vulnerability fix status, version, detector-hint validation, and occurrence suggestions | | `internal/sbom` | SBOM codec (SPDX 2.3, CycloneDX) | +| `internal/assurance` | Release assurance framework: check-result contract, catalog, report generation, release-asset verification, and the `sbominterop` and `perfrun` check tools | | `internal/benchmark` | Hidden local dependency-graph benchmark, baseline comparison, scoring, and embedded presets | | `internal/output` | Output rendering plus structured command payloads and schema generation for `scan`, `diff`, `explain`, JSON, and SARIF 2.1.0 | | `internal/plugin` | Plugin discovery, protocol, handshake, and pooled subprocess execution | @@ -95,6 +96,7 @@ Runtime preparation is owned by `internal/engine`: build the filtered registry o - `internal/baseline` owns the baseline document and matching implementation. It depends on the SDK policy contracts and must not be imported by `internal/engine`. - `internal/remediation` owns canonical vulnerability remediation decisions. Detectors may supply validated read-only strategy hints, but they do not choose final actions or versions. - `internal/registry` owns package-manager discovery, support lookups, and built-in registry wiring in `internal/registry/builder.go`. Do not create or reintroduce a separate `registrybuilder` package. +- `internal/assurance` owns the release assurance framework and must not be imported by any package under `cmd/`, `internal/cli`, or `internal/engine`: it is repository tooling, not shipped CLI behavior. It may read repository files and run downloaded release binaries, which no shipped package may do. - `internal/engine` may import `internal/detectors` and `internal/registry`, but detector packages must not point back into `internal/engine`. Runtime planning, prepared subprojects, and detector-chain reuse belong in `internal/engine`. ## Non-Negotiable @@ -231,6 +233,24 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai - Register new tests in both slice matrices (`smoke.yml` and exactly one slice in `update-smoke-goldens.yml`); `go test -run` elements are unanchored regexes — use `$` anchors to keep slice ownership exact. - `TestExamplePluginFixtureCompiles` runs in `make test` and must keep compiling against the pinned `bomly-dev/bomly-sdk` release; update the fixture source when the SDK contract changes. +## Release assurance + +Every quality check belongs to one of three release stages and is declared in `docs/assurance/catalog.json` (schema `bomly.assurance-catalog/v1`): + +- **prerequisites** — run on the source tree before a tag exists (smoke, portable stability, cross-builds, fuzz, catalog validation). +- **pre-release** — run inside `release.yml` against the still-draft release (asset completeness, checksums, cosign signature, SLSA provenance, released binaries). +- **post-release** — run against the shipped binaries after publication (install scripts, public download, released-binary scans, SBOM interoperability, performance samples). + +Rules: + +- Every check writes one `bomly.assurance-check/v1` document per instance through `go run ./internal/assurance/cmd` (`emit`, `gotest`, `convert`, or `verify-release`) and uploads it as an `assurance-*` artifact. Never hand-write that JSON in a workflow. +- Adding a check means adding a catalog entry **and** emitting its result; a declared check with no result is reported as `missing` and blocks its stage when it is a gate. +- `proves` and `limitations` are mandatory, public, and written in plain language — they are rendered on bomly.dev/assurance. +- Public evidence claims live in the same catalog (`evidence[]`), keep their pinned Git revisions and checksummed artifacts, and name the check that backs them. `make assurance-catalog` re-hashes every file they reference. +- The per-release report (`docs/assurance/reports/.json`) and `docs/assurance/index.json` are written by the post-release assessment and are the only data source for the public page. + +See [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) for the contracts and how to add a check. + ## Feature Checklist When adding a new user-visible feature (new CLI flag, new component class, new pipeline stage, new analyzer, etc.), walk this checklist before requesting review. The surfaces forgotten most often are **MCP**, **plugin command**, and **smoke test**. @@ -316,6 +336,7 @@ Draft releases are created automatically after merges to `main` from commit pref | [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | Public, user-facing architecture overview | | [`dev-docs/MODELS.md`](dev-docs/MODELS.md) | Domain model reference: Dependency, Package, Vulnerability, Finding, PackageRegistry | | [`dev-docs/CI.md`](dev-docs/CI.md) | CI setup and workflow (GitHub Actions) | +| [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) | Release assurance framework: stages, check contract, catalog, reports | | [`docs/CONFIG_REFERENCE.md`](docs/CONFIG_REFERENCE.md) | Generated config reference (all keys, env vars, defaults) | | [`docs/SUPPORT_MATRIX.md`](docs/SUPPORT_MATRIX.md) | Ecosystem detector coverage | | `docs/schemas/*.json`, `docs/schemas/*.md` | Generated JSON schemas and human-readable output docs for `scan`, `diff`, and `explain` | diff --git a/CLAUDE.md b/CLAUDE.md index 072530cb..c3ad3a22 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ make smoke ARGS="-update" # regenerate smoke golden files make fuzz FUZZTIME=5s # run every registered fuzz target with a short per-target budget make benchmark # run the hidden local dependency-graph benchmark make benchmark-report # analyze local benchmark artifacts with Copilot CLI -make evidence # verify the public evidence catalog (test/evidence/cases.json) +make assurance-catalog # validate the release assurance catalog (docs/assurance/catalog.json) make run ARGS="scan" # go run ./cmd/bomly make generate # regenerate config reference, JSON schemas, schema docs, support matrix, and component docs (binary-driven) ``` @@ -54,6 +54,7 @@ See [`dev-docs/ARCHITECTURE.md`](dev-docs/ARCHITECTURE.md) for full detail (the | `internal/baseline` | Portable package-finding baseline codec and audit-integrated policy-status resolver | | `internal/remediation` | Canonical vulnerability fix status, version, detector-hint validation, and occurrence suggestions | | `internal/sbom` | SBOM codec (SPDX 2.3, CycloneDX) | +| `internal/assurance` | Release assurance framework: check-result contract, catalog, report generation, release-asset verification, and the `sbominterop` and `perfrun` check tools | | `internal/benchmark` | Hidden local dependency-graph benchmark, baseline comparison, scoring, and embedded presets | | `internal/output` | Output rendering plus structured command payloads and schema generation for `scan`, `diff`, `explain`, JSON, and SARIF 2.1.0 | | `internal/plugin` | Plugin discovery, protocol, handshake, and pooled subprocess execution | @@ -95,6 +96,7 @@ Runtime preparation is owned by `internal/engine`: build the filtered registry o - `internal/baseline` owns the baseline document and matching implementation. It depends on the SDK policy contracts and must not be imported by `internal/engine`. - `internal/remediation` owns canonical vulnerability remediation decisions. Detectors may supply validated read-only strategy hints, but they do not choose final actions or versions. - `internal/registry` owns package-manager discovery, support lookups, and built-in registry wiring in `internal/registry/builder.go`. Do not create or reintroduce a separate `registrybuilder` package. +- `internal/assurance` owns the release assurance framework and must not be imported by any package under `cmd/`, `internal/cli`, or `internal/engine`: it is repository tooling, not shipped CLI behavior. It may read repository files and run downloaded release binaries, which no shipped package may do. - `internal/engine` may import `internal/detectors` and `internal/registry`, but detector packages must not point back into `internal/engine`. Runtime planning, prepared subprojects, and detector-chain reuse belong in `internal/engine`. ## Non-Negotiable @@ -231,6 +233,24 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai - Register new tests in both slice matrices (`smoke.yml` and exactly one slice in `update-smoke-goldens.yml`); `go test -run` elements are unanchored regexes — use `$` anchors to keep slice ownership exact. - `TestExamplePluginFixtureCompiles` runs in `make test` and must keep compiling against the pinned `bomly-dev/bomly-sdk` release; update the fixture source when the SDK contract changes. +## Release assurance + +Every quality check belongs to one of three release stages and is declared in `docs/assurance/catalog.json` (schema `bomly.assurance-catalog/v1`): + +- **prerequisites** — run on the source tree before a tag exists (smoke, portable stability, cross-builds, fuzz, catalog validation). +- **pre-release** — run inside `release.yml` against the still-draft release (asset completeness, checksums, cosign signature, SLSA provenance, released binaries). +- **post-release** — run against the shipped binaries after publication (install scripts, public download, released-binary scans, SBOM interoperability, performance samples). + +Rules: + +- Every check writes one `bomly.assurance-check/v1` document per instance through `go run ./internal/assurance/cmd` (`emit`, `gotest`, `convert`, or `verify-release`) and uploads it as an `assurance-*` artifact. Never hand-write that JSON in a workflow. +- Adding a check means adding a catalog entry **and** emitting its result; a declared check with no result is reported as `missing` and blocks its stage when it is a gate. +- `proves` and `limitations` are mandatory, public, and written in plain language — they are rendered on bomly.dev/assurance. +- Public evidence claims live in the same catalog (`evidence[]`), keep their pinned Git revisions and checksummed artifacts, and name the check that backs them. `make assurance-catalog` re-hashes every file they reference. +- The per-release report (`docs/assurance/reports/.json`) and `docs/assurance/index.json` are written by the post-release assessment and are the only data source for the public page. + +See [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) for the contracts and how to add a check. + ## Feature Checklist When adding a new user-visible feature (new CLI flag, new component class, new pipeline stage, new analyzer, etc.), walk this checklist before requesting review. The surfaces forgotten most often are **MCP**, **plugin command**, and **smoke test**. @@ -316,6 +336,7 @@ Draft releases are created automatically after merges to `main` from commit pref | [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | Public, user-facing architecture overview | | [`dev-docs/MODELS.md`](dev-docs/MODELS.md) | Domain model reference: Dependency, Package, Vulnerability, Finding, PackageRegistry | | [`dev-docs/CI.md`](dev-docs/CI.md) | CI setup and workflow (GitHub Actions) | +| [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) | Release assurance framework: stages, check contract, catalog, reports | | [`docs/CONFIG_REFERENCE.md`](docs/CONFIG_REFERENCE.md) | Generated config reference (all keys, env vars, defaults) | | [`docs/SUPPORT_MATRIX.md`](docs/SUPPORT_MATRIX.md) | Ecosystem detector coverage | | `docs/schemas/*.json`, `docs/schemas/*.md` | Generated JSON schemas and human-readable output docs for `scan`, `diff`, and `explain` | diff --git a/Makefile b/Makefile index cb45a1ce..078da8cc 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ EXE_SUFFIX=$(if $(filter Windows_NT,$(OS)),.exe,) GOLANGCI_LINT=$(GOPATH_BIN)/golangci-lint$(EXE_SUFFIX) FUZZTIME?=60s -.PHONY: build build-full build-lite fmt fmt-check lint install-hooks test smoke fuzz run generate evidence benchmark benchmark-report licenses +.PHONY: build build-full build-lite fmt fmt-check lint install-hooks test smoke fuzz run generate assurance-catalog assurance-report benchmark benchmark-samples benchmark-report licenses build: build-full build-lite @@ -41,14 +41,18 @@ smoke: fuzz: FUZZTIME="$(FUZZTIME)" scripts/run-fuzz.sh -evidence: - go run ./internal/tools/publicevidence $(if $(CASE),-case $(CASE),) +assurance-catalog: + go run ./internal/assurance/cmd catalog-validate $(if $(CHECK),--check $(CHECK),) $(if $(EVIDENCE),--evidence $(EVIDENCE),) + +assurance-report: + go run ./internal/assurance/cmd report --results internal/assurance/testdata/fixtures/mixed-failure/results \ + --catalog internal/assurance/testdata/catalog.json --out .assurance --tag v0.0.0-preview --previous none --allow-unknown benchmark: build-full bin/$(BINARY_NAME)$(EXE_SUFFIX) benchmark $(if $(ARGS),$(ARGS),) benchmark-samples: build-lite - go run ./internal/tools/benchmarkrun -output .benchmark-runs/performance -case canonical-sbom-scan -samples 5 -network-state offline -- \ + go run ./internal/assurance/perfrun -output .benchmark-runs/performance -case canonical-sbom-scan -samples 5 -network-state offline -- \ ./bin/$(BINARY_NAME)-lite$(EXE_SUFFIX) scan --sbom --path test/smoke/testdata/sboms/go.spdx.json --detectors sbom --format json benchmark-report: diff --git a/README.md b/README.md index 1845468d..c1e9ebc6 100644 --- a/README.md +++ b/README.md @@ -79,9 +79,9 @@ Bomly reads manifests, lockfiles, package-manager output, container layers, or e | Can I triage reachable findings first? | `bomly scan --enrich --audit --analyze --fail-on high --fail-on reachable` | For more recipes, see [Getting Started](docs/GETTING_STARTED.md) and -[Use Cases](docs/USE_CASES.md). To review the public inputs, commands, expected -results, and limitations behind important behavior claims, see -[Reproducible Evidence](docs/EVIDENCE.md). +[Use Cases](docs/USE_CASES.md). To see the checks every release goes through, +what they prove, and where to read the per-release report, see +[Release assurance](docs/ASSURANCE.md). ## Explore Interactively diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md new file mode 100644 index 00000000..287badf0 --- /dev/null +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -0,0 +1,149 @@ +# Release assurance framework + +This document is for maintainers. The user-facing explanation is +[`docs/ASSURANCE.md`](../docs/ASSURANCE.md), and the published reports live at +[bomly.dev/assurance](https://bomly.dev/assurance). + +The framework has three parts: + +1. a **catalog** that declares every check and every public evidence claim; +2. a **check-result contract** every check writes when it finishes; +3. a **report** built by merging results into the catalog, published per release. + +Everything lives in one place. Go code is under `internal/assurance/`, data and +the public document are under `docs/assurance/`, and the workflows that run the +checks are named in each catalog entry's `source`. + +## Stages + +| Stage | Runs | Workflow | +| --- | --- | --- | +| `prerequisites` | Before a tag exists, on the source tree | `assurance-prerequisites.yml`, which calls `smoke.yml`, `portable-assurance.yml`, and `fuzz.yml` | +| `pre-release` | Inside the release pipeline, against the still-draft release | `release.yml` | +| `post-release` | After publication, against the shipped binaries | `assurance-assessment.yml` and `sbom-interoperability.yml` | + +A stage passes when every `gate` check in it passes and no declared check is +missing. `advisory` checks are always reported and never block. + +## The check-result contract + +Every check writes one JSON file per instance, named `[.].json`, +in the schema `bomly.assurance-check/v1`. Workflows upload those files as +`assurance-*` artifacts; later jobs download them all with `merge-multiple` and +hand the directory to the tool. + +Nothing hand-writes that JSON. Four commands produce it: + +```sh +# an ordinary shell step +go run ./internal/assurance/cmd emit --id cross-build --exit-code "$rc" \ + --summary "12 of 12 release targets built." \ + --metric builds_planned=12 --metric builds_completed=12 \ + --detail "linux/amd64 full=pass" --out assurance-results --step-summary +``` + +```sh +# a Go test slice +go test -tags smoke ./test/smoke/ -json ... | tee smoke.jsonl +go run ./internal/assurance/cmd gotest --id smoke --instance go \ + --input smoke.jsonl --exit-code "$rc" --echo --out assurance-results +``` + +```sh +# a tool that already writes a manifest +go run ./internal/assurance/cmd convert benchmark-run --id perf-samples \ + --input .benchmark-runs/performance/run-manifest.json --out assurance-results +``` + +```sh +# downloaded release assets +go run ./internal/assurance/cmd verify-release --dir assets --version 0.24.0 \ + --scope full --out assurance-results +``` + +`emit`, `gotest`, and `convert` read the stage and level from the catalog, so a +step only passes `--stage` when it emits something the catalog does not declare +(which the report will then flag as unknown). `--details-jsonl` reads +sub-results from a file, which keeps Windows command lines short. + +Every command fills the release tag, commit, run URL, job, and runner from the +GitHub Actions environment. Set `BOMLY_ASSURANCE_TAG` when a stage runs for a +tag that is not the checked-out ref. + +## Judging a stage and building the report + +```sh +go run ./internal/assurance/cmd verdict --results assurance-results \ + --stage prerequisites --step-summary +``` + +`verdict` exits non-zero when a gate check failed or a declared check reported +nothing. That is the step that stops a release. + +```sh +go run ./internal/assurance/cmd report --results assurance-results \ + --tag v0.24.0 --commit "$SHA" --url "$RELEASE_URL" --published-at "$PUBLISHED" +``` + +`report` writes `docs/assurance/reports/.json` and updates +`docs/assurance/index.json`, prints a markdown summary, and compares the release +against the previous one listed in the index. It exits with code 3 when a result +arrives for a check the catalog does not declare, so a renamed check cannot +silently disappear from the report. + +## Adding a check + +1. Add the entry to `docs/assurance/catalog.json`: `id`, `title`, `area`, + `stage`, `level`, `description`, `source`, optional `expected_instances`, + `reproduce`, and — required — `proves` and `limitations` in plain language. + Checks are sorted by `id`. +2. Emit a result from the workflow named in `source`, and upload it as an + `assurance-` artifact. +3. If the check backs a public claim, add an `evidence` entry pointing at it + with `check_id` (and `instance`, when one specific leg proves the claim). +4. Run `make assurance-catalog`, then `go test ./internal/assurance/`. + Refresh goldens with `go test ./internal/assurance/ -update` when the report + shape changes. + +Until the workflow actually emits the new result, every report will mark the +check `missing` — that is the intended behavior: gaps are loud. + +## Evidence claims + +Evidence claims are the public "we prove X, we do not prove Y" statements that +used to live in `test/evidence/cases.json`. They keep the same rigor: pinned +Git revisions, checksummed fixtures and expected-result files, explicit +reproduction commands, and mandatory limitations. `make assurance-catalog` +re-hashes every file a claim names, so a golden file cannot drift away from the +claim it supports. + +Claims proven by running a workflow against a release (`release-artifact` and +`platform-matrix` levels) carry no repository checksum, because the per-release +check result is the record that they ran. + +## Re-running things + +```sh +gh workflow run assurance-prerequisites.yml -f ref=main +``` + +```sh +gh workflow run assurance-assessment.yml -f tag=v0.24.0 +``` + +A failed prerequisites run means no tag was created, so the fix is an ordinary +pull request. A failed pre-release gate leaves a draft release and no published +version: fix the cause, delete the tag and the draft, then tag again. A failed +post-release check cannot be undone in the release, so the assessment opens a +tracking issue and the published report records the failure honestly. + +`vars.RELEASE_ASSURANCE_ENFORCE` set to `false` runs everything and publishes +the report without blocking a release. Use it for the first release after a +framework change, then turn it back on. + +## Related documents + +- [`docs/ASSURANCE.md`](../docs/ASSURANCE.md) — the public explanation +- [`dev-docs/SECURITY_ASSURANCE.md`](SECURITY_ASSURANCE.md) — trust boundaries and their regression tests +- [`dev-docs/RELEASE_CHECKLIST.md`](RELEASE_CHECKLIST.md) — the release procedure +- [`test/assurance/`](../test/assurance/) — the narrative notes behind individual checks diff --git a/dev-docs/adr/0035-release-assurance-is-a-catalog-and-a-result-contract.md b/dev-docs/adr/0035-release-assurance-is-a-catalog-and-a-result-contract.md new file mode 100644 index 00000000..40115b43 --- /dev/null +++ b/dev-docs/adr/0035-release-assurance-is-a-catalog-and-a-result-contract.md @@ -0,0 +1,52 @@ +# ADR-0035: Release assurance is a declarative catalog plus a per-check result contract + +- **Date:** 2026-08-20 +- **Status:** Accepted + +## Context + +Quality checks were independent workflows whose evidence was a job status and, +in two cases, hand-written markdown in a step summary. Public claims lived in a +separate `test/evidence/cases.json` catalog with its own checker. Nothing tied a +claim to whether its check actually ran for a given release, nothing verified +the published release artifacts at all, and no release produced a single +readable answer to "did this one pass?". + +## Decision + +Release assurance is built from three pieces: + +- one catalog (`docs/assurance/catalog.json`) declaring every check, its stage, + whether it gates a release, what it proves, and what it does not — plus the + public claims, each naming the check that backs it; +- one check-result contract (`bomly.assurance-check/v1`) that every check emits + through `internal/assurance/cmd`, so shell steps never hand-write JSON; +- one report (`bomly.assurance-report/v1`), written per release into + `docs/assurance/reports/.json`, which is the only data source for the + public assurance page. + +Checks are grouped by *when they can still change the outcome*: prerequisites +run before a tag exists, pre-release checks run while the release is still a +draft, and the exhaustive assessment runs afterwards against the binaries users +actually download. + +## Consequences + +- A declared check with no result is reported as `missing` and blocks its stage + when it is a gate. Silence is never treated as success. +- A result whose id the catalog does not declare fails report generation, so a + renamed check cannot quietly vanish. +- A stale golden file is fixed by an ordinary pull request, because the checks + most prone to drift run before a tag exists. +- Workflow files are no longer pinned by checksum in the catalog. Pinning the + file proved only that the file had not changed; the per-release check result + proves the workflow ran and what it found. Fixture and expected-result files + are still checksummed, because a claim about a golden file is only as good as + that file — and regenerating goldens therefore has to refresh the catalog + (`catalog-validate --refresh`, wired into `Update Smoke Goldens`). +- Reports are committed to the default branch rather than attached to the + release, because the exhaustive stage runs after publication and GitHub + releases are immutable once published. +- The report and index schema versions are a contract with bomly.dev: adding an + optional field keeps the version, and anything else has to teach the site the + new shape first. diff --git a/dev-docs/adr/README.md b/dev-docs/adr/README.md index 231661c4..95cecdd2 100644 --- a/dev-docs/adr/README.md +++ b/dev-docs/adr/README.md @@ -55,3 +55,4 @@ status to `Superseded by [ADR-NNNN](NNNN-slug.md)`; do not rewrite history | ADR-0032 | 2026-08-14 | [SBOM exports carry a synthesized primary component and shared document identity](0032-sbom-exports-carry-synthesized-primary-component.md) | Accepted | | ADR-0033 | 2026-08-24 | [Package origin is detector-asserted; SBOM export only projects it](0033-package-origin-is-detector-asserted.md) | Accepted | | ADR-0034 | 2026-08-24 | [Decisions are recorded as individual ADRs](0034-decisions-are-recorded-as-individual-adrs.md) | Accepted | +| ADR-0035 | 2026-08-20 | [Release assurance is a declarative catalog plus a per-check result contract](0035-release-assurance-is-a-catalog-and-a-result-contract.md) | Accepted | diff --git a/docs/ASSURANCE.md b/docs/ASSURANCE.md new file mode 100644 index 00000000..8f03ab3b --- /dev/null +++ b/docs/ASSURANCE.md @@ -0,0 +1,91 @@ +# Release assurance + +Every Bomly release goes through the same set of quality checks, in the same +order, and the results are published for that exact version. You can read them +at **[bomly.dev/assurance](https://bomly.dev/assurance)**, pick any release from +the selector, and print the page if you need a copy for a review file. + +This page explains what the checks are, when they run, and what they do and do +not prove. + +## Three stages + +| Stage | When it runs | What it decides | +| --- | --- | --- | +| Release prerequisites | On the source tree, before a version is tagged | Whether the code is fit to be released at all | +| Final pre-release checks | After the release files are built, while the release is still a draft | Whether the files about to be published are complete, unmodified, and signed | +| Post-release assessment | After the release is published | How the binaries people actually download behave | + +Splitting the work this way is deliberate. Checks that can be flaky, or that +need a fix in the source tree, run **before** a version number exists, so a +problem is fixed by a normal pull request instead of by a broken release. +Checks that describe the published files can only run once those files exist. + +## What each check covers + +The full list, with what every check proves and what it does not, lives in the +machine-readable catalog at +[`docs/assurance/catalog.json`](assurance/catalog.json). The assurance page +renders the same catalog, so the page and the repository can never disagree. + +Highlights: + +- **End-to-end scans.** Every supported ecosystem is scanned from a pinned + public example project and compared against a checked-in expected result. +- **Platform stability.** The unit tests run repeatedly on Linux, macOS, and + Windows, and every release binary is cross-compiled. +- **Parser safety.** Fuzz targets feed malformed project, configuration, + baseline, and SBOM files to the parsers that read them. +- **Release integrity.** Checksums, the Sigstore signature, and SLSA build + provenance are verified against the release files before publication. +- **Installation.** The published install scripts are run on all three + operating systems against the new release. +- **SBOM interoperability.** The SBOM documents Bomly writes are validated with + the official SPDX and CycloneDX tools, pinned by checksum. +- **Speed and stability.** The same scan is repeated with a cold and a warm + cache to record timing and confirm the output does not change. + +Checks are either **gates**, which stop a release, or **advisory**, which are +reported but never block. The report always says which is which. + +## Evidence claims + +Alongside the checks, the catalog carries **evidence claims**: specific +statements about Bomly's behavior, each with the pinned input, the exact +command to reproduce it, the expected result file, and its stated limitations. +Each claim names the check that backs it, so the report can show whether that +claim still held for the release you are looking at. + +Reproduce any claim from a repository checkout: + +```sh +make assurance-catalog +``` + +```sh +go run ./internal/assurance/cmd catalog-validate --evidence graph-npm +``` + +The first command validates the whole catalog, including the checksums of every +expected-result file it names. The second prints one claim with its reproduction +command. + +## What a green report does not mean + +- A passing report describes the checks listed in the catalog. Software can + still fail in ways nobody has written a check for. +- Checks that reach live advisory services record what those services said on + that day. That answer can change afterwards. +- Timing numbers are observations from one continuous-integration machine, not + guarantees or limits. +- "Unreachable" in a reachability result is a confidence signal, not proof that + a package is safe. +- Release integrity checks prove the published files are the ones this + repository's release workflow built. They are not a review of what the code + does. + +## Related documents + +- [Security and trust boundaries](SECURITY.md) +- [Network and privacy](NETWORK.md) +- [Installation](INSTALLATION.md), including how to verify checksums yourself diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md deleted file mode 100644 index 07d28707..00000000 --- a/docs/EVIDENCE.md +++ /dev/null @@ -1,92 +0,0 @@ -# Reproducible evidence - -Bomly publishes the inputs, commands, expected artifacts, and limitations -behind important behavior claims. The goal is to make a claim reviewable -without private infrastructure or a one-off demonstration. - -The machine-readable catalog is -[`test/evidence/cases.json`](../test/evidence/cases.json). Check it from a -repository checkout: - -```sh -make evidence -``` - -To inspect one case: - -```sh -make evidence CASE=graph-npm -``` - -This verifies the recorded checksums and prints the exact command to reproduce -the case. - -## Evidence levels - -| Level | Meaning | -| --- | --- | -| Deterministic | Uses checked-in inputs or local services and compares a stable normalized result | -| Pinned input | Uses a public repository at a recorded commit; local tools or artifact registries can still affect build-tool-backed resolution | -| Snapshot | Records the normalized result of an input that can move, such as a container tag | -| Live service | Uses a pinned project with current advisory data; the result is a dated observation | -| Manual assurance | Runs a separately started GitHub Actions workflow and saves its detailed report as an artifact | - -Each catalog case must state both what it proves and what it does not prove. -Remote Git inputs include a full commit revision. Fixtures, workflows, and -expected results include SHA-256 checksums. - -## Case studies - -- [Dependency graph evidence](evidence/DEPENDENCY_GRAPHS.md) covers npm, pnpm, - Yarn, Bun, Go, Python, and Maven graphs. A separate deterministic - `degraded-detector-fallback` case covers visible fallback orchestration. -- [Policy and vulnerability-guidance evidence](evidence/POLICY_AND_GUIDANCE.md) - covers vulnerability and SPDX policy, baselines, source changes, persisted - findings, reachability tiers, and read-only remediation suggestions. -- [Targets and operational assurance](evidence/TARGETS_AND_OPERATIONS.md) - covers local and Git projects, containers, SBOM ingestion and validation, - repeated unit tests across supported systems, release builds, and repeatable - performance measurements. - -## Dated workflow evidence - -The -[SBOM interoperability run from July 24, 2026](https://github.com/bomly-dev/bomly-cli/actions/runs/30057587653) -completed successfully at commit -`9530b9f3bfcb3fe1d2748fa2bcfadb5e53e3346c`. The workflow generated canonical -SPDX 2.3 and CycloneDX 1.6 documents and checked them with its recorded -checksum-pinned validators. - -The -[portable stability run from July 24, 2026](https://github.com/bomly-dev/bomly-cli/actions/runs/30065452505) -completed successfully at commit -`e6bc5235f85dc909b6bc73f6ba9eb82c22c44ac4`. It repeated Go unit tests on -Linux, macOS, and Windows, repeated the Java-related and full Linux unit-test -suites, and built every release target. It did not run remote smoke tests. - -Workflow summaries explain what ran, the result, and where to inspect failures. -Their downloadable artifacts retain detailed commands, versions, diagnostics, -and hashes where the workflow produces a run manifest. - -## How to read the results - -- A checked golden proves the normalized result for its recorded input. It is - not a promise that every project in that ecosystem has the same fidelity. -- A live enrichment golden can change when advisory services add, correct, or - withdraw records. Bomly does not claim an immutable offline advisory view. -- A package count alone is not graph proof. Review identities, versions, - relationships, scopes, sources, and occurrence paths. -- A fallback warning means useful evidence may still exist, but coverage can - be lower than the preferred detector. -- `unreachable` does not mean safe. The analyzer and tier define what was - checked. -- Remediation output is read-only guidance. It does not apply or validate a - package change. -- SBOM validation proves acceptance by the named validator versions for the - canonical fixtures, not lossless conversion for every producer or consumer. - -## Provenance - -The tracked cases use Bomly-owned fixtures, Bomly-owned example repositories, -or complete public input repositories. The descriptions and test structure -are written for Bomly's own graph, pipeline, and output contracts. diff --git a/docs/README.md b/docs/README.md index 36ec6aac..fc100987 100644 --- a/docs/README.md +++ b/docs/README.md @@ -54,7 +54,7 @@ Specifications, matrices, and design deep dives. The generated pages are regener - [Architecture](ARCHITECTURE.md) — the scan pipeline, domain model, and network behavior - [Network and Privacy](NETWORK.md) — every network trigger, what it transmits, and how to stay offline - [Security and Trust Boundaries](SECURITY.md) — permissions, network behavior, plugins, input limits, and residual risks -- [Reproducible Evidence](EVIDENCE.md) — public inputs, commands, results, and limitations behind important behavior claims +- [Release assurance](ASSURANCE.md) — the checks every release goes through, what they prove, and where to read the per-release report - [Glossary](GLOSSARY.md) — every term, one sentence each ## Experimental diff --git a/test/evidence/cases.json b/docs/assurance/catalog.json similarity index 50% rename from test/evidence/cases.json rename to docs/assurance/catalog.json index 8125919e..b83548f9 100644 --- a/test/evidence/cases.json +++ b/docs/assurance/catalog.json @@ -1,11 +1,742 @@ { - "schema_version": "bomly.public-evidence/v1", - "cases": [ + "schema_version": "bomly.assurance-catalog/v1", + "areas": [ + { + "id": "dependency-graph", + "title": "Dependency graphs", + "description": "Resolving the packages a project depends on, and how they got there." + }, + { + "id": "diff", + "title": "Change detection", + "description": "Comparing two scans and classifying what changed." + }, + { + "id": "end-to-end", + "title": "End-to-end behavior", + "description": "Running the real command line against real projects and comparing the whole result." + }, + { + "id": "framework", + "title": "Assurance framework", + "description": "The catalog and contracts this report is built from." + }, + { + "id": "installation", + "title": "Installation", + "description": "Getting the published release onto a machine the documented ways." + }, + { + "id": "parsers", + "title": "Input parsing safety", + "description": "Reading untrusted project, configuration, and SBOM files without crashing." + }, + { + "id": "performance", + "title": "Speed and stability", + "description": "How long a repeated scan takes and whether it returns the same answer." + }, + { + "id": "policy", + "title": "Policy and guidance", + "description": "Evaluating license, vulnerability, baseline, and source-change policy." + }, + { + "id": "reachability", + "title": "Reachability", + "description": "Deciding whether a vulnerable package is actually used." + }, + { + "id": "release-integrity", + "title": "Release integrity", + "description": "Whether the published files are complete, unmodified, and signed." + }, + { + "id": "remediation", + "title": "Remediation", + "description": "The upgrade guidance Bomly derives, without changing your project." + }, + { + "id": "sbom", + "title": "SBOM interoperability", + "description": "Reading and writing SBOM documents other tools accept." + }, + { + "id": "stability", + "title": "Platform stability", + "description": "Behaving the same on every supported operating system and processor." + }, + { + "id": "targets", + "title": "Scan targets", + "description": "Scanning things other than a local source tree, such as container images." + } + ], + "checks": [ + { + "id": "catalog-valid", + "title": "Assurance catalog is valid", + "area": "framework", + "stage": "prerequisites", + "level": "gate", + "description": "Validates this catalog and confirms every fixture and golden file it points at still has the recorded checksum.", + "source": { + "workflow": "assurance-prerequisites.yml", + "job": "catalog" + }, + "reproduce": [ + [ + "make", + "assurance-catalog" + ] + ], + "proves": [ + "Every claim in this report points at a real check and at repository files that have not changed since the claim was written." + ], + "limitations": [ + "A valid catalog says the paperwork is consistent; the checks themselves decide whether the software works." + ] + }, + { + "id": "cross-build", + "title": "Release binaries build for every target", + "area": "stability", + "stage": "prerequisites", + "level": "gate", + "description": "Cross-compiles the full and lite binaries for every supported operating system and processor from the same source tree.", + "source": { + "workflow": "portable-assurance.yml", + "job": "linux-stability" + }, + "reproduce": [ + [ + "gh", + "workflow", + "run", + "assurance-prerequisites.yml", + "-f", + "ref=main" + ] + ], + "proves": [ + "The source tree compiles into both binaries for every Linux, macOS, and Windows target a release ships." + ], + "limitations": [ + "A successful build is not a test: it does not run the binary or check what it produces." + ] + }, + { + "id": "fuzz", + "title": "Parsers survive malformed input", + "area": "parsers", + "stage": "prerequisites", + "level": "advisory", + "description": "Runs every registered Go fuzz target against the parsers that read untrusted project, configuration, baseline, SBOM, and plugin data.", + "source": { + "workflow": "fuzz.yml", + "job": "fuzz" + }, + "reproduce": [ + [ + "make", + "fuzz", + "FUZZTIME=60s" + ] + ], + "proves": [ + "Each registered parser handled its seed corpus and newly generated inputs without crashing during the time budget of this run." + ], + "limitations": [ + "Fuzzing explores a limited time budget. A clean run is evidence of robustness, not proof that no malformed input can cause a failure.", + "Parsers that are command-backed or handled entirely by the standard library are excluded and listed in test/assurance/PARSER_FUZZING.md." + ] + }, + { + "id": "install-script", + "title": "Install scripts install the release", + "area": "installation", + "stage": "post-release", + "level": "gate", + "description": "Runs the published install script on each operating system against this release and asks the installed binary for its version.", + "source": { + "workflow": "assurance-assessment.yml", + "job": "install-scripts" + }, + "expected_instances": [ + { + "name": "ubuntu", + "platform": "linux" + }, + { + "name": "macos", + "platform": "darwin" + }, + { + "name": "windows", + "platform": "windows" + } + ], + "reproduce": [ + [ + "sh", + "-c", + "BOMLY_VERSION= BOMLY_INSTALL_DIR=$PWD/bin sh scripts/install.sh" + ] + ], + "proves": [ + "The documented one-line install works on Linux, macOS, and Windows for this exact release, and the installed binary reports this version." + ], + "limitations": [ + "This installs into a temporary directory on a clean runner. It does not cover every shell, proxy, or locked-down corporate environment." + ] + }, + { + "id": "perf-samples", + "title": "Repeated scan speed and stability", + "area": "performance", + "stage": "post-release", + "level": "advisory", + "description": "Runs the same offline scan with the released binary several times with a cold cache and several times with a warm cache, and records timing, memory, and output stability.", + "source": { + "workflow": "assurance-assessment.yml", + "job": "perf-samples" + }, + "reproduce": [ + [ + "make", + "benchmark-samples" + ] + ], + "proves": [ + "Repeated runs of the same scan produced identical normalized output, and the recorded timings show how long that scan took on this machine." + ], + "limitations": [ + "Timing and memory are observations from one CI machine, not limits. Comparing them across releases is only meaningful because the case and the machine class stay the same." + ] + }, + { + "id": "public-download", + "title": "Published files download and match", + "area": "release-integrity", + "stage": "post-release", + "level": "gate", + "description": "Downloads release files from their public URLs after publication and checks them against the published checksum list.", + "source": { + "workflow": "assurance-assessment.yml", + "job": "public-download" + }, + "reproduce": [ + [ + "gh", + "release", + "download", + "", + "--repo", + "bomly-dev/bomly-cli" + ] + ], + "proves": [ + "The files a user downloads from the public release page are present and match the checksums published with them." + ], + "limitations": [ + "This checks availability and integrity at one moment from one network location." + ] + }, + { + "id": "release-assets", + "title": "Every release file is attached", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Compares the files attached to the draft release against the full list a release is required to ship.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "reproduce": [ + [ + "go", + "run", + "./internal/assurance/cmd", + "verify-release", + "--dir", + "assets", + "--version", + "" + ] + ], + "proves": [ + "The release carries an archive for every supported platform, both binaries, every Linux package format, the checksum list, its signature, and the provenance file." + ], + "limitations": [ + "This counts files. Whether each archive contains a working binary is checked separately." + ] + }, + { + "id": "release-binaries", + "title": "Released binaries report this version", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Extracts the archives built for each operating system and runs the binaries inside them.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "expected_instances": [ + { + "name": "linux-amd64", + "platform": "linux" + }, + { + "name": "darwin-arm64", + "platform": "darwin" + }, + { + "name": "windows-amd64", + "platform": "windows" + } + ], + "reproduce": [ + [ + "go", + "run", + "./internal/assurance/cmd", + "verify-release", + "--dir", + "assets", + "--version", + "", + "--scope", + "native" + ] + ], + "proves": [ + "Both the full and lite binaries start on Linux, macOS, and Windows and report the version that is being released." + ], + "limitations": [ + "Only the architecture of the CI runner is executed for each operating system; other architectures are verified by checksum and build only." + ] + }, + { + "id": "release-checksums", + "title": "Release files match their checksums", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Hashes every downloaded release file and compares it with the published SHA256SUMS list.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "reproduce": [ + [ + "sha256sum", + "--check", + "SHA256SUMS" + ] + ], + "proves": [ + "Every file the release ships hashes to exactly the value recorded in the checksum list, and the list covers every file." + ], + "limitations": [ + "Checksums prove the files were not altered after they were built; the signature and provenance checks prove who built them." + ] + }, + { + "id": "release-provenance", + "title": "Build provenance verifies", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Verifies the SLSA build provenance attached to the release with slsa-verifier.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "reproduce": [ + [ + "slsa-verifier", + "verify-artifact", + "", + "--provenance-path", + "multiple.intoto.jsonl", + "--source-uri", + "github.com/bomly-dev/bomly-cli" + ] + ], + "proves": [ + "The release archives were produced by this repository's tagged release workflow, as recorded in signed build provenance." + ], + "limitations": [ + "Provenance describes how and where the files were built. It does not review what the source code does." + ] + }, + { + "id": "release-signature", + "title": "Checksum list is signed", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Verifies the Sigstore signature over the release checksum list with cosign.", + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "reproduce": [ + [ + "cosign", + "verify-blob", + "SHA256SUMS", + "--bundle", + "SHA256SUMS.sigstore.json", + "--certificate-oidc-issuer", + "https://token.actions.githubusercontent.com", + "--certificate-identity-regexp", + "^https://github.com/bomly-dev/bomly-cli/" + ] + ], + "proves": [ + "The checksum list carries a valid Sigstore signature issued to this repository's release workflow, so the checksums themselves can be trusted." + ], + "limitations": [ + "The signature covers the checksum list. Individual files are tied to it through their checksums." + ] + }, + { + "id": "released-scan", + "title": "The released binary scans real projects", + "area": "end-to-end", + "stage": "post-release", + "level": "gate", + "description": "Runs scan, diff, and explain with the binary from the published release against pinned example repositories and checks the results against the same golden files the source-tree smoke tests use.", + "source": { + "workflow": "assurance-assessment.yml", + "job": "released-scan" + }, + "expected_instances": [ + { + "name": "go", + "ecosystems": [ + "Go" + ] + }, + { + "name": "node", + "ecosystems": [ + "JavaScript" + ] + }, + { + "name": "sbom", + "ecosystems": [ + "SBOM documents" + ] + } + ], + "reproduce": [ + [ + "sh", + "-c", + "BOMLY_SMOKE_BINARY=$PWD/bin/bomly go test -tags smoke ./test/smoke/ -run 'TestScan$/scan-go$'" + ] + ], + "proves": [ + "The binary users actually download produces the same dependency graphs as the source tree it was built from." + ], + "limitations": [ + "A representative subset of ecosystems runs against the released binary; the full ecosystem matrix runs on the source tree before the release is tagged." + ] + }, + { + "id": "sbom-interoperability", + "title": "Official validators accept our SBOMs", + "area": "sbom", + "stage": "post-release", + "level": "gate", + "description": "Generates SPDX 2.3 and CycloneDX 1.7 documents with the released binary and validates them with checksum-pinned official SPDX and CycloneDX tools.", + "source": { + "workflow": "sbom-interoperability.yml", + "job": "validate" + }, + "reproduce": [ + [ + "go", + "run", + "./internal/assurance/sbominterop", + "-bomly", + "./bin/bomly" + ] + ], + "proves": [ + "The SBOM documents this release writes are accepted by the official SPDX and CycloneDX validators, at the exact validator versions recorded in the result." + ], + "limitations": [ + "Validators check that a document is well formed against its specification. They do not check that every receiving tool interprets it the same way." + ] + }, + { + "id": "smoke", + "title": "End-to-end scans of real projects", + "area": "end-to-end", + "stage": "prerequisites", + "level": "gate", + "description": "Runs scan, diff, and explain against pinned public repositories, one slice per ecosystem, and compares the whole result with checked-in golden files.", + "source": { + "workflow": "smoke.yml", + "job": "smoke" + }, + "expected_instances": [ + { + "name": "go", + "ecosystems": [ + "Go" + ] + }, + { + "name": "go-reachability", + "ecosystems": [ + "Go" + ] + }, + { + "name": "node", + "ecosystems": [ + "JavaScript" + ] + }, + { + "name": "node-detail-policy", + "ecosystems": [ + "JavaScript" + ] + }, + { + "name": "node-reachability", + "ecosystems": [ + "JavaScript" + ] + }, + { + "name": "java", + "ecosystems": [ + "Java" + ] + }, + { + "name": "java-reachability", + "ecosystems": [ + "Java" + ] + }, + { + "name": "python", + "ecosystems": [ + "Python" + ] + }, + { + "name": "python-reachability", + "ecosystems": [ + "Python" + ] + }, + { + "name": "uv", + "ecosystems": [ + "Python" + ] + }, + { + "name": "php", + "ecosystems": [ + "PHP" + ] + }, + { + "name": "ruby", + "ecosystems": [ + "Ruby" + ] + }, + { + "name": "sbom", + "ecosystems": [ + "SBOM documents" + ] + }, + { + "name": "dotnet", + "ecosystems": [ + ".NET" + ] + }, + { + "name": "rust", + "ecosystems": [ + "Rust" + ] + }, + { + "name": "dart", + "ecosystems": [ + "Dart" + ] + }, + { + "name": "swift", + "ecosystems": [ + "Swift" + ] + }, + { + "name": "elixir", + "ecosystems": [ + "Elixir" + ] + }, + { + "name": "scala", + "ecosystems": [ + "Scala" + ] + }, + { + "name": "cpp", + "ecosystems": [ + "C and C++" + ] + }, + { + "name": "container", + "ecosystems": [ + "Container images" + ] + }, + { + "name": "plugin", + "ecosystems": [ + "Plugins" + ] + } + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "proves": [ + "For every supported ecosystem, the command line produces exactly the dependency graph, policy result, and output shape recorded in the golden files." + ], + "limitations": [ + "Golden files pin one revision of one example project per ecosystem. They do not cover every project layout or every version of a package manager.", + "Cases that reach live advisory services can change when upstream data changes." + ] + }, + { + "id": "unit-portable", + "title": "Unit tests pass on every platform", + "area": "stability", + "stage": "prerequisites", + "level": "gate", + "description": "Runs the full Go unit test suite twice on Linux, macOS, and Windows.", + "source": { + "workflow": "portable-assurance.yml", + "job": "portable" + }, + "expected_instances": [ + { + "name": "ubuntu-latest", + "platform": "linux" + }, + { + "name": "macos-latest", + "platform": "darwin" + }, + { + "name": "windows-latest", + "platform": "windows" + } + ], + "reproduce": [ + [ + "go", + "test", + "./...", + "-count=1" + ] + ], + "proves": [ + "Every unit test passes on all three supported operating systems, twice in a row, so a pass is not a one-off." + ], + "limitations": [ + "Unit tests run in process against fixtures. They do not download repositories or call external services." + ] + }, + { + "id": "unit-repeat-full", + "title": "Repeated full test runs are stable", + "area": "stability", + "stage": "prerequisites", + "level": "gate", + "description": "Runs the complete unit test suite five more times on Linux to surface intermittent failures.", + "source": { + "workflow": "portable-assurance.yml", + "job": "linux-stability" + }, + "reproduce": [ + [ + "go", + "test", + "./...", + "-count=5" + ] + ], + "proves": [ + "The full suite passed five consecutive times, so the tests recorded here are not intermittently passing." + ], + "limitations": [ + "Repetition raises confidence in stability; it cannot rule out a rare failure." + ] + }, + { + "id": "unit-repeat-java", + "title": "Java detector tests are stable", + "area": "stability", + "stage": "prerequisites", + "level": "gate", + "description": "Runs the Gradle, Maven, and sbt detector test suites ten times, because build-tool-backed detectors are the most timing sensitive.", + "source": { + "workflow": "portable-assurance.yml", + "job": "linux-stability" + }, + "reproduce": [ + [ + "go", + "test", + "./internal/detectors/gradle", + "./internal/detectors/maven", + "./internal/detectors/sbt", + "-count=10" + ] + ], + "proves": [ + "The build-tool-backed Java detector suites passed ten consecutive times." + ], + "limitations": [ + "This exercises the detectors against local fixtures with the toolchain installed on the runner." + ] + } + ], + "evidence": [ { "id": "baseline-policy", "title": "Finding baseline lifecycle", "area": "policy", "evidence_level": "deterministic", + "check_id": "smoke", + "instance": "go", "inputs": [ { "kind": "git", @@ -33,7 +764,7 @@ "TestFindingBaselineWorkflow$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/finding-baseline-workflow.golden.json", "sha256": "8fa2b220af563a89d1fbaaab03a75ccc0952ae43251438bc27a2314301315aa0" @@ -51,6 +782,8 @@ "title": "Container package inventory", "area": "targets", "evidence_level": "snapshot", + "check_id": "smoke", + "instance": "container", "inputs": [ { "kind": "container", @@ -73,7 +806,7 @@ "TestContainerScan/container-scan-alpine$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/container-scan-alpine.golden.json", "sha256": "50f7d4426ead3af9ea5a07df7eeb77743e7aa1412260e1a665a7301f3c22ecc5" @@ -91,6 +824,7 @@ "title": "Visible detector fallback", "area": "dependency-graph", "evidence_level": "deterministic", + "check_id": "unit-portable", "inputs": [ { "kind": "fixture", @@ -111,7 +845,7 @@ "-count=1" ] ], - "evidence": [ + "artifacts": [ { "path": "internal/engine/pipeline_fallback_test.go", "sha256": "7751d6137dd39c8616857f2e0891ed4928bf2b37e72475d567c1e38262755a6d" @@ -129,6 +863,8 @@ "title": "Bun lockfile graph", "area": "dependency-graph", "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node", "inputs": [ { "kind": "git", @@ -155,7 +891,7 @@ "TestScan$/scan-bun$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-bun.golden.json", "sha256": "c61ccae15f3d36028ec1e741436abbf036236b9e2f018393b863e98d14360fb4" @@ -173,6 +909,8 @@ "title": "Go module graph", "area": "dependency-graph", "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "go", "inputs": [ { "kind": "git", @@ -200,7 +938,7 @@ "TestScan$/scan-go$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-go.golden.json", "sha256": "728f949898446657c4987bc1202a8d00c78b118868b4945141e4db8c4c54cdea" @@ -218,6 +956,8 @@ "title": "Maven dependency graph", "area": "dependency-graph", "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "java", "inputs": [ { "kind": "git", @@ -246,7 +986,7 @@ "TestScan$/scan-maven$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-maven.golden.json", "sha256": "a1077544af4bd637311e1767abe2a140e3ba8f0f65a64e662378e5ccca07de9b" @@ -264,6 +1004,8 @@ "title": "npm lockfile graph", "area": "dependency-graph", "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node", "inputs": [ { "kind": "git", @@ -291,10 +1033,10 @@ "TestScan$/scan-npm$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-npm.golden.json", - "sha256": "da252e623cdeaf88db7874f528fd9a2f204c1136c4ab0bb26e4faa943dee79c1" + "sha256": "82f741b8f7291cb1f885cba788a2a7e3505b143a9f2ded646d01334b8abfaa8a" } ], "proves": [ @@ -309,6 +1051,8 @@ "title": "pnpm lockfile graph", "area": "dependency-graph", "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node", "inputs": [ { "kind": "git", @@ -336,10 +1080,10 @@ "TestScan$/scan-pnpm$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-pnpm.golden.json", - "sha256": "07991083b233acb7bc856e41275517a30cd5793c4b1afb94a467502374545467" + "sha256": "92b867725f0617f273d8f52d1d9174ace6a6712f4d9fb11da988e455e764a221" } ], "proves": [ @@ -354,6 +1098,8 @@ "title": "Python requirements graph", "area": "dependency-graph", "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "python", "inputs": [ { "kind": "git", @@ -380,7 +1126,7 @@ "TestScan$/scan-python-pip$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-python-pip.golden.json", "sha256": "025d14ad1a3c93f3af22a4a83a358ee85b42d782f50798b285048e35e8632bf2" @@ -398,6 +1144,8 @@ "title": "Yarn lockfile graph", "area": "dependency-graph", "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node", "inputs": [ { "kind": "git", @@ -425,10 +1173,10 @@ "TestScan$/scan-yarn$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-yarn.golden.json", - "sha256": "707e1ab208812a53527cc1de888d3e76c410d9b963b49c14076f023ceb5f7b90" + "sha256": "1e1a5f1579089169b1cf3f55bb399ea83591bfc660dcac82d3355ab83c98c2af" } ], "proves": [ @@ -443,6 +1191,7 @@ "title": "Complex and invalid SPDX policy", "area": "policy", "evidence_level": "deterministic", + "check_id": "unit-portable", "inputs": [ { "kind": "fixture", @@ -463,7 +1212,7 @@ "-count=1" ] ], - "evidence": [ + "artifacts": [ { "path": "internal/auditors/license/spdx_policy_matrix_test.go", "sha256": "04d8e3528b33687a09d317b867f7c24068081c1ac60bb53cee1e0a03c0b143f2" @@ -479,8 +1228,9 @@ { "id": "performance-stability", "title": "Repeated cold and warm scan measurements", - "area": "operations", - "evidence_level": "deterministic", + "area": "performance", + "evidence_level": "release-artifact", + "check_id": "perf-samples", "inputs": [ { "kind": "fixture", @@ -497,12 +1247,6 @@ "benchmark-samples" ] ], - "evidence": [ - { - "path": "internal/tools/benchmarkrun/main.go", - "sha256": "d99c52b52065a95b0cda323f082bbdcb04276ab7cef96f21678550154f340bef" - } - ], "proves": [ "The runner records five isolated cold and five shared-cache warm samples with output hashes, timing, memory, and dispersion." ], @@ -515,6 +1259,7 @@ "title": "Risk that persists across a version change", "area": "diff", "evidence_level": "deterministic", + "check_id": "unit-portable", "inputs": [ { "kind": "fixture", @@ -535,7 +1280,7 @@ "-count=1" ] ], - "evidence": [ + "artifacts": [ { "path": "internal/engine/diff/diff_test.go", "sha256": "abe87f7b95e55e5cf70bd0684c36027757308f7f405a9db370a894238f16f195" @@ -551,13 +1296,13 @@ { "id": "portable-platforms", "title": "Repeated unit tests and release builds", - "area": "operations", - "evidence_level": "manual-assurance", + "area": "stability", + "evidence_level": "platform-matrix", + "check_id": "unit-portable", "inputs": [ { "kind": "workflow", - "location": ".github/workflows/portable-assurance.yml", - "sha256": "fadc2ff37a55c3b0dbaed91d3db67745b32b6ca517075e55734cea49a231b4d2" + "location": ".github/workflows/portable-assurance.yml" } ], "required_tools": [ @@ -573,12 +1318,6 @@ "v0.20.0" ] ], - "evidence": [ - { - "path": ".github/workflows/portable-assurance.yml", - "sha256": "fadc2ff37a55c3b0dbaed91d3db67745b32b6ca517075e55734cea49a231b4d2" - } - ], "proves": [ "The manual workflow repeats Go unit tests on Linux, macOS, and Windows and cross-builds every release target." ], @@ -591,6 +1330,8 @@ "title": "Go vulnerability reachability", "area": "reachability", "evidence_level": "live-service", + "check_id": "smoke", + "instance": "go-reachability", "inputs": [ { "kind": "git", @@ -618,7 +1359,7 @@ "TestScan$/scan-go-reachability$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-go-reachability.golden.json", "sha256": "2d1db6318d75c491b8866ff9643ae80635702591046e4be89c362ca1b828a4cd" @@ -636,6 +1377,8 @@ "title": "Java package reachability", "area": "reachability", "evidence_level": "live-service", + "check_id": "smoke", + "instance": "java-reachability", "inputs": [ { "kind": "git", @@ -664,10 +1407,10 @@ "TestScan$/scan-java-maven-reachability$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-java-maven-reachability.golden.json", - "sha256": "b3fa7485ebcee27a4836f7ad6b318ca53c238b24fa303059dd518007fc8ecb0f" + "sha256": "cae9114ef8280849afad9135b92b7a6a7e1cb3c0c25f9e049a60448b912f7874" } ], "proves": [ @@ -682,6 +1425,8 @@ "title": "JavaScript package reachability", "area": "reachability", "evidence_level": "live-service", + "check_id": "smoke", + "instance": "node-reachability", "inputs": [ { "kind": "git", @@ -709,10 +1454,10 @@ "TestScan$/scan-npm-reachability$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-npm-reachability.golden.json", - "sha256": "2bb379a7e0b301357b24a8a1a169b805461530dac392af1908525e3ce5366760" + "sha256": "3fb1b3b9a11e65fa4b76daa56c90098ce442831d79bb1fdaeba675f201fb78ec" } ], "proves": [ @@ -727,6 +1472,8 @@ "title": "Python package reachability", "area": "reachability", "evidence_level": "live-service", + "check_id": "smoke", + "instance": "python-reachability", "inputs": [ { "kind": "git", @@ -753,10 +1500,10 @@ "TestScan$/scan-python-pip-reachability$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-python-pip-reachability.golden.json", - "sha256": "4fbf082566a5562308edd71fd86f155e3eaf1fc7ed0f6dc96c0f2f7778ca3f90" + "sha256": "61d9b6d3e91e33755c19c7d2b35c0b2fb1c6a98692cdf57dc4630323ad315ca4" } ], "proves": [ @@ -766,16 +1513,56 @@ "This is package-tier evidence, reflective imports can reduce confidence, and live advisory results may change." ] }, + { + "id": "release-integrity", + "title": "Published release files are complete and signed", + "area": "release-integrity", + "evidence_level": "release-artifact", + "check_id": "release-checksums", + "inputs": [ + { + "kind": "release", + "location": "https://github.com/bomly-dev/bomly-cli/releases" + } + ], + "required_tools": [ + "cosign", + "slsa-verifier" + ], + "reproduce": [ + [ + "gh", + "release", + "download", + "", + "--repo", + "bomly-dev/bomly-cli" + ], + [ + "sha256sum", + "--check", + "SHA256SUMS", + "--ignore-missing" + ] + ], + "proves": [ + "Every file this release publishes was checked against its checksum, its signature, and its build provenance before the release was made public." + ], + "limitations": [ + "These checks describe the files as published. Anyone can re-run them at any time with the commands above." + ] + }, { "id": "remediation-read-only", "title": "Canonical read-only remediation guidance", "area": "remediation", "evidence_level": "deterministic", + "check_id": "unit-portable", "inputs": [ { "kind": "fixture", "location": "internal/remediation/derive_test.go", - "sha256": "4ab715423343aad70cff5ea4a406b642eb687f2552afb894a9edfba4244c7582" + "sha256": "8afb2ca42404363f3eaaf29eaa20e0c57ddb92aed6209b5720af2c8ba8681e97" } ], "required_tools": [ @@ -791,10 +1578,10 @@ "-count=1" ] ], - "evidence": [ + "artifacts": [ { "path": "internal/remediation/derive_test.go", - "sha256": "4ab715423343aad70cff5ea4a406b642eb687f2552afb894a9edfba4244c7582" + "sha256": "8afb2ca42404363f3eaaf29eaa20e0c57ddb92aed6209b5720af2c8ba8681e97" } ], "proves": [ @@ -809,6 +1596,8 @@ "title": "CycloneDX 1.6 ingestion", "area": "sbom", "evidence_level": "deterministic", + "check_id": "smoke", + "instance": "sbom", "inputs": [ { "kind": "fixture", @@ -834,7 +1623,7 @@ "TestScan$/scan-sbom-cyclonedx$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json", "sha256": "54f5e33a47d884e1705d22102dd4745b4474295d0cc56da827bc3f172792dabb" @@ -851,12 +1640,12 @@ "id": "sbom-interoperability", "title": "External SBOM validation", "area": "sbom", - "evidence_level": "manual-assurance", + "evidence_level": "release-artifact", + "check_id": "sbom-interoperability", "inputs": [ { "kind": "workflow", - "location": ".github/workflows/sbom-interoperability.yml", - "sha256": "3d379300ab8e7bc02f6859c088ba06153fcbd2dc3cf159a62d890d1abda24f1a" + "location": ".github/workflows/sbom-interoperability.yml" } ], "required_tools": [ @@ -872,12 +1661,6 @@ "v0.20.0" ] ], - "evidence": [ - { - "path": ".github/workflows/sbom-interoperability.yml", - "sha256": "3d379300ab8e7bc02f6859c088ba06153fcbd2dc3cf159a62d890d1abda24f1a" - } - ], "proves": [ "Checksum-pinned official SPDX and CycloneDX validators can check Bomly's generated canonical SBOMs." ], @@ -890,6 +1673,8 @@ "title": "SPDX 2.3 ingestion", "area": "sbom", "evidence_level": "deterministic", + "check_id": "smoke", + "instance": "sbom", "inputs": [ { "kind": "fixture", @@ -915,7 +1700,7 @@ "TestScan$/scan-sbom-spdx$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/testdata/golden/scan-sbom-spdx.golden.json", "sha256": "d49e4fc3f9c61fdb4c533116d9a5988e1841c7ccacc3675b93fb069d5ec14a00" @@ -933,6 +1718,8 @@ "title": "Registry-to-Git source-change review", "area": "policy", "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "node-detail-policy", "inputs": [ { "kind": "git", @@ -966,10 +1753,10 @@ "TestDependencyDetailRiskPolicy$" ] ], - "evidence": [ + "artifacts": [ { "path": "test/smoke/audit_test.go", - "sha256": "363c2ab0ecd4bec931c18abb32706c528e22f0dd3331db2691822851d5eb5ae2" + "sha256": "96fcd3b6a411a9651edce806fdc0e60f2e71208c33f4d91a4640177ab2b828fe" } ], "proves": [ @@ -979,16 +1766,43 @@ "A source change is a review signal, not proof of malicious intent, and unknown source evidence cannot be classified." ] }, + { + "id": "supported-install-paths", + "title": "The documented install paths work", + "area": "installation", + "evidence_level": "release-artifact", + "check_id": "install-script", + "inputs": [ + { + "kind": "release", + "location": "https://bomly.dev/install.sh" + } + ], + "reproduce": [ + [ + "sh", + "-c", + "curl -fsSL https://bomly.dev/install.sh | sh" + ] + ], + "proves": [ + "The install script published for this release downloads, verifies, and installs a working binary on Linux, macOS, and Windows." + ], + "limitations": [ + "Package manager installs (Homebrew, Scoop, WinGet, and the Linux packages) are published by the same release but are not exercised by this check." + ] + }, { "id": "vulnerability-policy", "title": "Vulnerability policy constraints", "area": "policy", "evidence_level": "deterministic", + "check_id": "unit-portable", "inputs": [ { "kind": "fixture", "location": "internal/auditors/vulnerability/policy_matrix_test.go", - "sha256": "79b98c8c367eebe8933328980611b6d8497c369f6be5b55e57ef25405697c7a8" + "sha256": "8cd341ca5eea7df5ce50a1bdbe076637154e8f4ba760b64ba4ea366435a9a460" } ], "required_tools": [ @@ -1004,10 +1818,10 @@ "-count=1" ] ], - "evidence": [ + "artifacts": [ { "path": "internal/auditors/vulnerability/policy_matrix_test.go", - "sha256": "79b98c8c367eebe8933328980611b6d8497c369f6be5b55e57ef25405697c7a8" + "sha256": "8cd341ca5eea7df5ce50a1bdbe076637154e8f4ba760b64ba4ea366435a9a460" } ], "proves": [ diff --git a/docs/evidence/DEPENDENCY_GRAPHS.md b/docs/evidence/DEPENDENCY_GRAPHS.md deleted file mode 100644 index 7c28eab5..00000000 --- a/docs/evidence/DEPENDENCY_GRAPHS.md +++ /dev/null @@ -1,86 +0,0 @@ -# Dependency graph evidence - -Bomly's findings depend on the graph produced before enrichment or audit. -These public cases check the graph first: package identity, version, -relationship, scope, source, and the manifest that owns each occurrence. - -The cases use complete public example repositories at recorded Git commits. -Their normalized JSON results are checked into `test/smoke/testdata/golden`. -The evidence catalog records a SHA-256 checksum for every result, so an -expected change must also update the public evidence record. - -## What is covered - -| Case | Input evidence | Detector path | What the case checks | -| --- | --- | --- | --- | -| `graph-npm` | `package-lock.json` | npm lockfile detector | Package inventory, versions, scopes, and placement | -| `graph-pnpm` | `pnpm-lock.yaml` | pnpm lockfile detector | Package inventory and placement | -| `graph-yarn` | `yarn.lock` | Yarn lockfile detector | Package inventory and placement | -| `graph-bun` | `bun.lock` | Native Bun detector | Package inventory and placement | -| `graph-go` | `go.mod` and Go tool output | Go detector | Build-tool-backed module graph | -| `graph-python` | Pinned requirements lock | pip detector | Resolved Python package graph | -| `graph-maven` | `pom.xml` and Maven output | Maven detector | Build-tool-backed JVM graph | - -This is placement evidence, not just a list of package names. The Node cases -retain duplicate versions as separate package identities and keep occurrences -attached to the dependency paths represented by each lockfile. - -## Reproduce a case - -From a Bomly CLI checkout: - -```sh -make evidence CASE=graph-npm -``` - -The command verifies the catalog and prints the exact focused smoke command: - -```sh -go test -tags smoke ./test/smoke/ -v -count=1 -timeout 15m \ - -run 'TestScan$/scan-npm$' -``` - -The smoke test builds Bomly, clones the recorded public input, selects the -named detector, normalizes only documented volatile fields, and compares the -result with the checked-in JSON. - -## Example review workflow - -A maintainer updates a lockfile and wants to confirm that Bomly still sees the -same dependency structure: - -1. Run the matching graph case before changing the detector. -2. Make the detector change. -3. Run the case again. -4. Review every golden change as a package, relationship, scope, source, or - manifest-placement change. -5. Update the catalog checksum only when the new graph is intentional. - -This makes a changed count a starting point for review, not the conclusion. -The actual package and occurrence records explain what changed. - -## Degraded and unknown evidence - -Bomly can fall back when a preferred detector cannot run. A fallback is not -silently treated as equivalent: the result records which detector failed, -which detector ran, and whether coverage was reduced. Unresolved package -placement remains `unknown`; a synthetic link to a manifest helps navigation -but does not become a direct or transitive parent claim. - -The graph cases above pin their detector selector. If that detector is not -ready or fails, the smoke test fails instead of accepting a differently shaped -fallback result. - -## Limits - -- A case proves the checked repository and lockfile shape, not every format - version ever produced by that package manager. -- Build-tool-backed cases depend on compatible local tools and may need - registry access to resolve artifacts. -- A fallback inventory can contain useful packages while carrying less - relationship or source evidence than a native graph. -- Project roots, workspaces, local paths, Git dependencies, and arbitrary URL - dependencies stay in the graph but are not queried as published registry - packages. -- Graph evidence does not prove advisory accuracy. Matching is a later step - with separate evidence and limits. diff --git a/docs/evidence/POLICY_AND_GUIDANCE.md b/docs/evidence/POLICY_AND_GUIDANCE.md deleted file mode 100644 index c9859666..00000000 --- a/docs/evidence/POLICY_AND_GUIDANCE.md +++ /dev/null @@ -1,117 +0,0 @@ -# Policy and vulnerability-guidance evidence - -Bomly separates observed evidence from policy decisions: - -- enrichment attaches vulnerability and license information to packages; -- analysis can add reachability evidence; -- audit evaluates the available evidence against the selected policy; -- read-only remediation guidance is derived from vulnerability and dependency - evidence during enrichment. - -That separation matters when reviewing results. A vulnerability can be -present but warning-only, suppressed by a baseline, or failing policy. -Reachability is another property of the same vulnerability; it does not -replace severity or policy status. - -## Policy cases - -| Case | What it checks | Expected outcome | -| --- | --- | --- | -| `vulnerability-policy` | Severity, reachability, known exploitation, advisory allowlists, and repeated constraints | Only findings that satisfy the selected constraints fail | -| `license-policy` | SPDX `AND`, `OR`, nesting, exceptions, custom references, and invalid expressions | Valid expressions keep their Boolean meaning; invalid expressions remain visible and fail | -| `baseline-policy` | Create and automatically use a project finding baseline | The finding remains present with suppressed policy status | -| `source-change-policy` | Same-version move from a registry package to a Git source | Diff requests review; `--fail-on source-change` can fail the audited change | -| `persisted-risk` | Same finding remains across a package version change | The finding is reported as persisted, not hidden as one resolved and one new finding | - -Run any case through the public catalog: - -```sh -make evidence CASE=license-policy -make evidence CASE=source-change-policy -make evidence CASE=persisted-risk -``` - -Each command prints the focused test command, the recorded inputs, and the -case limitation. - -## Example policy workflow - -A team wants high-severity vulnerabilities and risky source changes to block -a pull request while keeping accepted findings visible: - -```sh -bomly diff \ - --base main \ - --head HEAD \ - --enrich \ - --audit \ - --fail-on high \ - --fail-on source-change -``` - -If a package changed from a known registry source to Git or an arbitrary URL, -Bomly explains that registry-based vulnerability matching may no longer cover -it. That is a review signal, not a claim that the change is malicious. Source -formats that cannot prove origin remain unknown. - -A finding baseline does not delete matching findings. It changes their policy -status to `suppressed`, so JSON, SARIF, Guard, MCP, and human output can still -show the accepted risk. - -## Reachability cases - -The catalog includes `reachability-go`, `reachability-node`, -`reachability-python`, and `reachability-java`. - -- The Go case records the analyzer and the tier actually returned for each - vulnerability. -- The JavaScript, Python, and Java cases prove package-tier reachable and - package-tier unreachable branches. -- Every case keeps the analyzer name, tier, status, and plain-language reason - with the vulnerability. - -These cases use current advisory services, so the checked golden is a dated -observation. Re-running later may legitimately discover added, changed, or -withdrawn advisories. - -`unreachable` never means `safe`. Static analysis can miss dynamic loading, -reflection, generated code, tests, build tags, or code paths outside the -analyzer's model. Use reachability to prioritize review, not to erase the -underlying vulnerability. - -## Read-only remediation guidance - -The `remediation-read-only` case exercises the central remediation component. -For each vulnerable package, it derives: - -- whether the available evidence describes a complete fix, partial fix, no - upstream fix, or an unknown fix state; -- a recommended version only when all vulnerability evidence supports one; -- an occurrence-specific suggested action; -- package-manager advice supplied by the detector when available. - -The guidance is derived during enrichment and applies only to vulnerabilities. -License and package-policy findings stay audit results because their meaning -depends on project policy. - -Suggestions do not edit manifests, run package managers, or claim that a -change is guaranteed to work. Unknown parents and non-registry occurrences -receive manual-review guidance instead of a guessed parent upgrade. - -Reproduce the deterministic derivation matrix: - -```sh -make evidence CASE=remediation-read-only -``` - -## Limits - -- `--audit` still requires `--enrich` by default. This avoids an accidental - audit with fewer findings because matching was forgotten. -- Policy cannot make missing evidence complete. Unknown source, license, - reachability, or fix data remains unknown. -- Live advisory services can change independently. Bomly does not ship an - immutable offline advisory snapshot. -- Source changes, reachability, and remediation are different signals. None - of them alone proves that a dependency is safe or malicious. -- Bomly provides read-only remediation suggestions; it does not apply fixes. diff --git a/docs/evidence/TARGETS_AND_OPERATIONS.md b/docs/evidence/TARGETS_AND_OPERATIONS.md deleted file mode 100644 index d71a3d8f..00000000 --- a/docs/evidence/TARGETS_AND_OPERATIONS.md +++ /dev/null @@ -1,126 +0,0 @@ -# Targets and operational assurance - -Bomly accepts local projects, Git repositories, container images, and existing -SBOMs. The public evidence uses the same engine paths as the CLI and states -where the input itself is not immutable. - -## Target cases - -| Target | Public case | What is checked | -| --- | --- | --- | -| Local project | `baseline-policy` | A public Git fixture is materialized locally, then scanned and audited through `--path` | -| Git repository | `graph-npm`, `graph-go`, and the other graph cases | The CLI clones a recorded commit and runs the selected detector | -| Container image | `container-inventory` | Built-in inventory reads packages from the checked Alpine image | -| SPDX SBOM | `sbom-spdx-ingest` | The checked SPDX 2.3 graph is ingested through the SBOM detector | -| CycloneDX SBOM | `sbom-cyclonedx-ingest` | The checked CycloneDX 1.6 graph is ingested through the SBOM detector | - -Inspect or reproduce one: - -```sh -make evidence CASE=container-inventory -make evidence CASE=sbom-spdx-ingest -``` - -The container smoke case currently uses `alpine:3.20`. The tag can move, so -the checked-in golden is explicitly a snapshot rather than an immutable image -claim. The Git cases separately record the full commit behind their readable -tag or ref. - -## Example SBOM workflow - -A release engineer receives a supplier SBOM and wants to apply the same policy -used for source scans: - -```sh -bomly scan \ - --sbom \ - --path supplier.spdx.json \ - --enrich \ - --audit \ - --fail-on high -``` - -The deterministic ingestion cases check Bomly's dependency graph. The -`SBOM interoperability assurance` workflow -(`.github/workflows/sbom-interoperability.yml`) in this repository adds an -external check. It runs weekly and can be started on demand: - -```sh -gh workflow run sbom-interoperability.yml -``` - -It builds the released CLI surface, uses the binary itself to generate SPDX -2.3 and CycloneDX 1.7 files from a pinned fixture SBOM, verifies the -downloaded validator checksums, runs the official validators -(`spdx/tools-java` and `cyclonedx-cli`), and uploads the generated files, -their checksums, and the validation logs as a public run artifact. - -Validator versions and download checksums stay in the workflow file, so -changing either requires an intentional, reviewable evidence update. See -[`test/assurance/SBOM_INTEROPERABILITY.md`](../../test/assurance/SBOM_INTEROPERABILITY.md) -for the workflow summary and failure-investigation steps. - -## Supported-system checks - -Unit-test and build assurance — the unit suite and release-target builds — -runs in this repository's public CI on every change. - -The publicly verifiable evidence for this repository is: - -- the `SBOM interoperability assurance` workflow described above, which - drives the built binary and official validators in public CI; -- the smoke suite, which runs the built binary end to end against pinned - public repositories and checked-in golden outputs; -- signed releases with SLSA build provenance, which tie each published - binary to the public revision it was built from. - -## Repeatable performance measurements - -Run: - -```sh -make benchmark-samples -``` - -The `performance-stability` case uses the checked SPDX fixture and the -lightweight Bomly binary. It records five isolated cold-cache scans and five -shared-cache warm scans under `.benchmark-runs/performance`. See -[`test/assurance/BENCHMARK_RUNS.md`](../../test/assurance/BENCHMARK_RUNS.md) -for the full description. - -The resulting `bomly.benchmark-run/v1` report includes: - -- repository and executable revisions and hashes; -- host and Go runtime details; -- exact command, working directory, cache mode, and network state; -- exit status, output size and hashes, timing, and peak memory for every run; -- median, variation, and an approximate 95% confidence interval. - -The stable gates are successful exit status, normalized output consistency, -and an optional explicit output-size cap. Wall time and memory remain -machine-specific measurements for review rather than universal limits. - -## Example release-confidence workflow - -Before a broad release: - -1. Run `make test` and the relevant pinned smoke slices. -2. Run `make benchmark-samples` and compare the report with the previous run - from a comparable host. -3. If SBOM output changed, start the interoperability workflow and inspect its - generated artifact hashes and validator results. -4. Record the repository commit with every retained workflow or benchmark - report. - -## Limits - -- Remote services and package registries can be temporarily unavailable. -- Build-tool-backed graph resolution can vary with tool versions; the catalog - states required tools and checked source revisions. -- Local repository scans can contain any number of individually bounded - files. Per-file parser limits do not create a total project-size limit. -- Portable Git options bound time and checkout validation but cannot reliably - cap transfer bytes or `.git` object storage before checkout completes. -- Workflow artifacts have retention periods. The workflow file, pinned tools, - reproduction command, and result link remain public after an artifact - expires, but the raw artifact may need to be regenerated. diff --git a/docs/manifest.json b/docs/manifest.json index 26a019d9..1f956b57 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -30,7 +30,7 @@ { "slug": "getting-started", "title": "Getting started", - "description": "First scan, enrich, audit, diff — all in five minutes.", + "description": "First scan, enrich, audit, diff \u2014 all in five minutes.", "group": "start" }, { @@ -60,7 +60,7 @@ { "slug": "output-formats", "title": "Output formats", - "description": "Text, JSON, SARIF, and SBOM artifacts — when to use each and how to combine them.", + "description": "Text, JSON, SARIF, and SBOM artifacts \u2014 when to use each and how to combine them.", "group": "start" }, { @@ -196,11 +196,10 @@ "group": "reference" }, { - "slug": "evidence", - "title": "Reproducible evidence", - "description": "Public inputs, commands, results, and limitations behind important behavior claims.", - "group": "reference", - "hasChildren": true + "slug": "assurance", + "title": "Release assurance", + "description": "The checks every release goes through, what they prove, and where to read the results.", + "group": "reference" }, { "slug": "glossary", diff --git a/internal/assurance/aggregate.go b/internal/assurance/aggregate.go new file mode 100644 index 00000000..5229eada --- /dev/null +++ b/internal/assurance/aggregate.go @@ -0,0 +1,402 @@ +package assurance + +import ( + "sort" + "strconv" + "strings" + "time" +) + +// BuildOptions configures how a report is assembled from check results. +type BuildOptions struct { + // Release identifies the release being reported on. + Release Release + // Stages limits the report to these stages; empty means every stage. + Stages []Stage + // StageRuns maps a stage to the workflow run that produced its results. + StageRuns map[Stage]string + // Previous is the previous release's report, used for trends. + Previous *Report + // IncludeEvidence attaches the public evidence claims to the report. + IncludeEvidence bool + // GeneratedBy names the tool version that produced the report. + GeneratedBy string + // Now is the report timestamp; the zero value means time.Now(). + Now time.Time +} + +// BuildReport merges check results into the catalog and produces the report +// that the public assurance page renders. Declared checks without a result are +// reported as missing; reported checks the catalog does not declare are listed +// separately so nothing is silently dropped. +func BuildReport(catalog Catalog, results []CheckResult, opts BuildOptions) Report { + now := opts.Now + if now.IsZero() { + now = time.Now() + } + stages := opts.Stages + if len(stages) == 0 { + stages = Stages() + } + selected := make(map[Stage]struct{}, len(stages)) + for _, stage := range stages { + selected[stage] = struct{}{} + } + + byCheck := make(map[string][]CheckResult, len(results)) + for _, result := range results { + byCheck[result.ID] = append(byCheck[result.ID], result) + } + + report := Report{ + SchemaVersion: ReportSchema, + GeneratedAt: now.UTC().Format(time.RFC3339), + Release: opts.Release, + Environment: Environment{GeneratedBy: opts.GeneratedBy, Runners: collectRunners(results)}, + } + + statusByCheck := make(map[string]Status, len(catalog.Checks)) + for _, check := range catalog.Checks { + if _, wanted := selected[check.Stage]; !wanted { + continue + } + reported := buildCheck(check, byCheck[check.ID]) + statusByCheck[check.ID] = reported.Status + report.Checks = append(report.Checks, reported) + } + sort.SliceStable(report.Checks, func(i, j int) bool { return report.Checks[i].ID < report.Checks[j].ID }) + + declared := make(map[string]struct{}, len(catalog.Checks)) + for _, check := range catalog.Checks { + declared[check.ID] = struct{}{} + } + for _, result := range results { + if _, known := declared[result.ID]; known { + continue + } + report.Unknown = append(report.Unknown, UnknownResult{ + ID: result.ID, Instance: result.Instance, Stage: result.Stage, Status: result.Status, + }) + } + sort.Slice(report.Unknown, func(i, j int) bool { + if report.Unknown[i].ID != report.Unknown[j].ID { + return report.Unknown[i].ID < report.Unknown[j].ID + } + return report.Unknown[i].Instance < report.Unknown[j].Instance + }) + + for _, stage := range Stages() { + if _, wanted := selected[stage]; !wanted { + continue + } + stageReport := StageReport{ID: stage, Title: stage.Title(), RunURL: opts.StageRuns[stage]} + var stageChecks []ReportCheck + for _, check := range report.Checks { + if check.Stage != stage { + continue + } + stageChecks = append(stageChecks, check) + stageReport.CheckIDs = append(stageReport.CheckIDs, check.ID) + } + stageReport.Verdict = summarize(stageChecks) + stageReport.Status = stageReport.Verdict.Overall + report.Stages = append(report.Stages, stageReport) + } + report.Verdict = summarize(report.Checks) + + for _, area := range catalog.Areas { + var areaChecks []ReportCheck + for _, check := range report.Checks { + if check.Area == area.ID { + areaChecks = append(areaChecks, check) + } + } + if len(areaChecks) == 0 { + continue + } + verdict := summarize(areaChecks) + report.Areas = append(report.Areas, AreaReport{ + ID: area.ID, Title: area.Title, Description: area.Description, + Status: verdict.Overall, Verdict: verdict, + }) + } + + if opts.IncludeEvidence { + report.Evidence = buildEvidence(catalog, report, selected) + } + report.Coverage = buildCoverage(catalog, report, selected) + if opts.Previous != nil { + report.Trends = buildTrends(*opts.Previous, report) + } + return report +} + +func buildCheck(check Check, results []CheckResult) ReportCheck { + reported := ReportCheck{ + ID: check.ID, Title: check.Title, Area: check.Area, Stage: check.Stage, + Level: check.Level, Description: check.Description, Source: check.Source, + Reproduce: check.Reproduce, Proves: check.Proves, Limitations: check.Limitations, + } + sort.Slice(results, func(i, j int) bool { return results[i].Key() < results[j].Key() }) + + seen := make(map[string]struct{}, len(results)) + status := Status("") + for _, result := range results { + name := result.Instance + if name == "" { + name = "default" + } + seen[name] = struct{}{} + reported.Instances = append(reported.Instances, InstanceReport{ + Name: name, Status: result.Status, Summary: result.Summary, + DurationMS: result.DurationMS, RunURL: result.RunURL, Runner: result.Runner, + Metrics: result.Metrics, Details: result.Details, + Artifacts: result.Artifacts, Links: result.Links, + }) + reported.DurationMS += result.DurationMS + if status == "" { + status = result.Status + } else { + status = Worse(status, result.Status) + } + } + for _, expected := range check.ExpectedInstances { + if _, present := seen[expected.Name]; !present { + reported.MissingInstances = append(reported.MissingInstances, expected.Name) + } + } + switch { + case len(results) == 0: + reported.Status = StatusMissing + reported.Summary = "No result was reported for this check." + default: + reported.Status = status + if len(reported.MissingInstances) > 0 { + reported.Status = Worse(reported.Status, StatusMissing) + } + reported.Summary = checkSummary(reported) + } + reported.Metrics = mergeMetrics(reported.Instances) + return reported +} + +func checkSummary(check ReportCheck) string { + if len(check.Instances) == 1 && len(check.MissingInstances) == 0 { + return check.Instances[0].Summary + } + counts := map[Status]int{} + for _, instance := range check.Instances { + counts[instance.Status]++ + } + parts := []string{plural(counts[StatusPass], "instance", "instances") + " passed"} + for _, status := range []Status{StatusFail, StatusDegraded, StatusSkip} { + if counts[status] > 0 { + parts = append(parts, plural(counts[status], "instance", "instances")+" "+string(status)) + } + } + if len(check.MissingInstances) > 0 { + parts = append(parts, plural(len(check.MissingInstances), "instance", "instances")+" reported nothing") + } + return strings.Join(parts, ", ") + "." +} + +func plural(count int, singular, pluralWord string) string { + word := pluralWord + if count == 1 { + word = singular + } + return strconv.Itoa(count) + " " + word +} + +// additiveMetrics are the metric names that stay meaningful when summed across +// the instances of one check. Everything else (medians, percentiles, sizes) +// stays visible per instance only, because summing it would be misleading. +var additiveMetrics = map[string]bool{ + "assets": true, + "builds_completed": true, + "builds_planned": true, + "cases": true, + "checks": true, + "completed_runs": true, + "packages": true, + "planned_runs": true, + "targets": true, + "tests_failed": true, + "tests_passed": true, + "tests_skipped": true, + "tests_total": true, +} + +func mergeMetrics(instances []InstanceReport) map[string]float64 { + if len(instances) == 0 { + return nil + } + if len(instances) == 1 { + if len(instances[0].Metrics) == 0 { + return nil + } + merged := make(map[string]float64, len(instances[0].Metrics)) + for key, value := range instances[0].Metrics { + merged[key] = value + } + return merged + } + merged := map[string]float64{} + for _, instance := range instances { + for key, value := range instance.Metrics { + if !additiveMetrics[key] { + continue + } + merged[key] += value + } + } + if len(merged) == 0 { + return nil + } + return merged +} + +func summarize(checks []ReportCheck) Verdict { + verdict := Verdict{Overall: StatusPass, Checks: len(checks)} + gateStatus := StatusPass + advisoryStatus := StatusPass + for _, check := range checks { + switch check.Status { + case StatusPass: + verdict.Passed++ + case StatusFail: + verdict.Failed++ + case StatusDegraded: + verdict.Degraded++ + case StatusSkip: + verdict.Skipped++ + case StatusMissing: + verdict.Missing++ + } + if check.Status == StatusMissing { + verdict.MissingChecks = append(verdict.MissingChecks, check.ID) + } + if check.Level == LevelGate { + gateStatus = Worse(gateStatus, check.Status) + if check.Status == StatusFail || check.Status == StatusDegraded { + verdict.GatesFailed = append(verdict.GatesFailed, check.ID) + } + continue + } + advisoryStatus = Worse(advisoryStatus, check.Status) + if check.Status == StatusFail || check.Status == StatusDegraded { + verdict.AdvisoriesFailed = append(verdict.AdvisoriesFailed, check.ID) + } + } + verdict.Overall = gateStatus + if verdict.Overall == StatusPass && advisoryStatus != StatusPass { + verdict.Overall = StatusDegraded + } + return verdict +} + +func buildEvidence(catalog Catalog, report Report, selected map[Stage]struct{}) []ReportEvidence { + var entries []ReportEvidence + for _, evidence := range catalog.Evidence { + check, exists := catalog.Check(evidence.CheckID) + if !exists { + continue + } + if _, wanted := selected[check.Stage]; !wanted { + continue + } + status := StatusMissing + if reported, found := report.Check(evidence.CheckID); found { + status = reported.Status + if evidence.Instance != "" { + status = StatusMissing + for _, instance := range reported.Instances { + if instance.Name == evidence.Instance { + status = instance.Status + break + } + } + } + } + entries = append(entries, ReportEvidence{ + ID: evidence.ID, Title: evidence.Title, Area: evidence.Area, + EvidenceLevel: evidence.EvidenceLevel, CheckID: evidence.CheckID, + Instance: evidence.Instance, Status: status, Inputs: evidence.Inputs, + RequiredTools: evidence.RequiredTools, Reproduce: evidence.Reproduce, + Artifacts: evidence.Artifacts, Proves: evidence.Proves, Limitations: evidence.Limitations, + }) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].ID < entries[j].ID }) + return entries +} + +func buildCoverage(catalog Catalog, report Report, selected map[Stage]struct{}) Coverage { + ecosystems := map[string]struct{}{} + var rows []CoverageRow + for _, check := range catalog.Checks { + if _, wanted := selected[check.Stage]; !wanted { + continue + } + reported, found := report.Check(check.ID) + if !found { + continue + } + cells := map[string]Status{} + for _, expected := range check.ExpectedInstances { + if len(expected.Ecosystems) == 0 { + continue + } + status := StatusMissing + for _, instance := range reported.Instances { + if instance.Name == expected.Name { + status = instance.Status + break + } + } + for _, ecosystem := range expected.Ecosystems { + ecosystems[ecosystem] = struct{}{} + if existing, present := cells[ecosystem]; present { + cells[ecosystem] = Worse(existing, status) + continue + } + cells[ecosystem] = status + } + } + if len(cells) == 0 { + continue + } + rows = append(rows, CoverageRow{CheckID: check.ID, Title: check.Title, Cells: cells}) + } + names := make([]string, 0, len(ecosystems)) + for ecosystem := range ecosystems { + names = append(names, ecosystem) + } + sort.Strings(names) + sort.Slice(rows, func(i, j int) bool { return rows[i].CheckID < rows[j].CheckID }) + return Coverage{Ecosystems: names, Rows: rows} +} + +func collectRunners(results []CheckResult) []Runner { + seen := map[Runner]struct{}{} + var runners []Runner + for _, result := range results { + if result.Runner == (Runner{}) { + continue + } + if _, exists := seen[result.Runner]; exists { + continue + } + seen[result.Runner] = struct{}{} + runners = append(runners, result.Runner) + } + sort.Slice(runners, func(i, j int) bool { + if runners[i].OS != runners[j].OS { + return runners[i].OS < runners[j].OS + } + if runners[i].Arch != runners[j].Arch { + return runners[i].Arch < runners[j].Arch + } + return runners[i].GoVersion < runners[j].GoVersion + }) + return runners +} diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go new file mode 100644 index 00000000..ebc0d721 --- /dev/null +++ b/internal/assurance/assurance_test.go @@ -0,0 +1,333 @@ +package assurance + +import ( + "encoding/json" + "flag" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +var updateGoldens = flag.Bool("update", false, "rewrite golden files") + +func loadFixture(t *testing.T, name string) []CheckResult { + t.Helper() + results, err := LoadResults(filepath.Join("testdata", "fixtures", name, "results")) + if err != nil { + t.Fatalf("load %s fixture: %v", name, err) + } + return results +} + +func fixtureCatalog(t *testing.T) Catalog { + t.Helper() + catalog, err := LoadCatalog(filepath.Join("testdata", "catalog.json")) + if err != nil { + t.Fatalf("load fixture catalog: %v", err) + } + return catalog +} + +func buildFixtureReport(t *testing.T, fixture string, previous *Report) Report { + t.Helper() + return BuildReport(fixtureCatalog(t), loadFixture(t, fixture), BuildOptions{ + Release: Release{Tag: "v9.9.9", Version: "9.9.9", Commit: "0f2103c7e671653e519cf5edb0d3e86020202ecf"}, + Previous: previous, + IncludeEvidence: true, + GeneratedBy: "assurance-test", + Now: time.Date(2026, 8, 19, 12, 0, 0, 0, time.UTC), + }) +} + +func TestStatusSeverityOrdering(t *testing.T) { + ordered := []Status{StatusPass, StatusSkip, StatusMissing, StatusDegraded, StatusFail} + for index := 1; index < len(ordered); index++ { + if ordered[index].Severity() <= ordered[index-1].Severity() { + t.Fatalf("%s must be more severe than %s", ordered[index], ordered[index-1]) + } + if got := Worse(ordered[index-1], ordered[index]); got != ordered[index] { + t.Fatalf("Worse(%s, %s) = %s", ordered[index-1], ordered[index], got) + } + } +} + +func TestParseCheckResultRejectsInvalidDocuments(t *testing.T) { + valid := CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Instance: "go", + Stage: StagePrerequisites, Level: LevelGate, Status: StatusPass, + Summary: "18 of 18 tests passed.", + } + encoded, err := valid.Encode() + if err != nil { + t.Fatalf("encode: %v", err) + } + if _, err := ParseCheckResult(encoded); err != nil { + t.Fatalf("valid result rejected: %v", err) + } + + cases := map[string]func(*CheckResult){ + "bad schema": func(r *CheckResult) { r.SchemaVersion = "other/v1" }, + "bad id": func(r *CheckResult) { r.ID = "Smoke Tests" }, + "bad instance": func(r *CheckResult) { r.Instance = "go/../etc" }, + "bad stage": func(r *CheckResult) { r.Stage = "whenever" }, + "bad level": func(r *CheckResult) { r.Level = "critical" }, + "bad status": func(r *CheckResult) { r.Status = "green" }, + "missing is not reportable": func(r *CheckResult) { r.Status = StatusMissing }, + "no summary": func(r *CheckResult) { r.Summary = " " }, + "bad detail status": func(r *CheckResult) { r.Details = []Detail{{Name: "x", Status: "green"}} }, + "detail without name": func(r *CheckResult) { r.Details = []Detail{{Name: "", Status: StatusPass}} }, + "link without url": func(r *CheckResult) { r.Links = []Link{{Label: "run"}} }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + broken := valid + mutate(&broken) + data, err := json.Marshal(broken) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if _, err := ParseCheckResult(data); err == nil { + t.Fatal("expected the document to be rejected") + } + }) + } +} + +func TestParseCheckResultRejectsUnknownFields(t *testing.T) { + data := []byte(`{"schema_version":"bomly.assurance-check/v1","id":"smoke","stage":"prerequisites",` + + `"status":"pass","summary":"ok","surprise":true}`) + if _, err := ParseCheckResult(data); err == nil { + t.Fatal("expected unknown fields to be rejected") + } +} + +func TestLoadResultsIgnoresForeignJSON(t *testing.T) { + dir := t.TempDir() + nested := filepath.Join(dir, "artifact") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "run-manifest.json"), + []byte(`{"schema_version":"bomly.benchmark-run/v1"}`), 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } + result := CheckResult{ + SchemaVersion: CheckSchema, ID: "fuzz", Stage: StagePrerequisites, + Level: LevelAdvisory, Status: StatusPass, Summary: "28 targets ran.", + } + encoded, err := result.Encode() + if err != nil { + t.Fatalf("encode: %v", err) + } + if err := os.WriteFile(filepath.Join(nested, result.FileName()), encoded, 0o644); err != nil { + t.Fatalf("write result: %v", err) + } + loaded, err := LoadResults(dir) + if err != nil { + t.Fatalf("load results: %v", err) + } + if len(loaded) != 1 || loaded[0].ID != "fuzz" { + t.Fatalf("expected only the check result, got %+v", loaded) + } +} + +func TestBuildReportMergesInstancesAndFlagsGaps(t *testing.T) { + report := buildFixtureReport(t, "mixed-failure", nil) + + smoke, found := report.Check("smoke") + if !found { + t.Fatal("smoke check missing from the report") + } + if smoke.Status != StatusMissing { + t.Fatalf("smoke status = %s, want missing because one slice reported nothing", smoke.Status) + } + if len(smoke.MissingInstances) != 1 || smoke.MissingInstances[0] != "node" { + t.Fatalf("missing instances = %v, want [node]", smoke.MissingInstances) + } + if report.Verdict.Overall != StatusMissing && report.Verdict.Overall != StatusFail { + t.Fatalf("overall verdict = %s, want a blocking verdict", report.Verdict.Overall) + } + if !report.Verdict.Blocking() { + t.Fatal("a failed gate check must block") + } + if len(report.Unknown) != 1 || report.Unknown[0].ID != "mystery-check" { + t.Fatalf("unknown results = %+v, want the undeclared check", report.Unknown) + } + if len(report.Stages) != 3 { + t.Fatalf("stages = %d, want 3", len(report.Stages)) + } + for _, stage := range report.Stages { + if stage.Title != stage.ID.Title() { + t.Fatalf("stage %s title = %q", stage.ID, stage.Title) + } + } +} + +func TestBuildReportPassesWhenEveryCheckReports(t *testing.T) { + report := buildFixtureReport(t, "all-pass", nil) + if report.Verdict.Overall != StatusPass { + t.Fatalf("overall verdict = %s, want pass", report.Verdict.Overall) + } + if report.Verdict.Blocking() { + t.Fatal("a passing report must not block") + } + smoke, _ := report.Check("smoke") + if len(smoke.Instances) != 2 { + t.Fatalf("smoke instances = %d, want 2", len(smoke.Instances)) + } + if smoke.Metrics["tests_total"] != 42 { + t.Fatalf("merged tests_total = %v, want 42", smoke.Metrics["tests_total"]) + } + if len(report.Coverage.Ecosystems) != 2 { + t.Fatalf("coverage ecosystems = %v", report.Coverage.Ecosystems) + } + if len(report.Evidence) != 1 || report.Evidence[0].Status != StatusPass { + t.Fatalf("evidence = %+v, want the go graph claim passing", report.Evidence) + } +} + +func TestBuildReportAdvisoryFailureDegradesButDoesNotBlock(t *testing.T) { + results := loadFixture(t, "all-pass") + for index := range results { + if results[index].ID == "perf-samples" { + results[index].Status = StatusFail + } + } + report := BuildReport(fixtureCatalog(t), results, BuildOptions{ + Release: Release{Tag: "v9.9.9"}, Now: time.Unix(0, 0).UTC(), + }) + if report.Verdict.Overall != StatusDegraded { + t.Fatalf("overall verdict = %s, want degraded", report.Verdict.Overall) + } + if report.Verdict.Blocking() { + t.Fatal("an advisory failure must not block a release") + } +} + +func TestTrendsCompareMetricsAndStatuses(t *testing.T) { + previous := buildFixtureReport(t, "all-pass", nil) + current := buildFixtureReport(t, "mixed-failure", &previous) + if current.Trends == nil { + t.Fatal("expected trends against the previous release") + } + if current.Trends.PreviousTag != "v9.9.9" { + t.Fatalf("previous tag = %q", current.Trends.PreviousTag) + } + var found bool + for _, metric := range current.Trends.Metrics { + if metric.CheckID == "perf-samples" && metric.Metric == "cold_median_ms" { + found = true + if metric.Delta != 293 { + t.Fatalf("cold_median_ms delta = %v, want 293", metric.Delta) + } + if metric.Better != betterLower { + t.Fatalf("cold_median_ms better = %q, want lower", metric.Better) + } + } + } + if !found { + t.Fatal("expected a cold_median_ms trend") + } + if len(current.Trends.Changed) == 0 { + t.Fatal("expected changed checks between the two fixtures") + } +} + +func TestReportGoldens(t *testing.T) { + previous := buildFixtureReport(t, "all-pass", nil) + for _, testCase := range []struct { + name string + previous *Report + }{ + {name: "all-pass"}, + {name: "mixed-failure", previous: &previous}, + } { + t.Run(testCase.name, func(t *testing.T) { + report := buildFixtureReport(t, testCase.name, testCase.previous) + encoded, err := report.Encode() + if err != nil { + t.Fatalf("encode: %v", err) + } + compareGolden(t, testCase.name+".report.json", encoded) + markdown := RenderMarkdown(report, MarkdownOptions{IncludeChecks: true, IncludeTrends: true}) + compareGolden(t, testCase.name+".summary.md", []byte(markdown)) + + // The report must survive a round trip through its own parser. + if _, err := ParseReport(encoded); err != nil { + t.Fatalf("parse encoded report: %v", err) + } + }) + } +} + +func compareGolden(t *testing.T, name string, actual []byte) { + t.Helper() + path := filepath.Join("testdata", "golden", name) + if *updateGoldens { + if err := os.WriteFile(path, actual, 0o644); err != nil { + t.Fatalf("update golden: %v", err) + } + return + } + expected, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read golden %s: %v (run go test ./internal/assurance -update)", name, err) + } + if string(expected) != string(actual) { + t.Fatalf("golden %s differs; run go test ./internal/assurance -update to refresh", name) + } +} + +func TestParseGoTestEventsSummarisesRuns(t *testing.T) { + stream := strings.Join([]string{ + `{"Action":"run","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan"}`, + `{"Action":"output","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan","Output":"=== RUN\n"}`, + `{"Action":"pass","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan/scan-go","Elapsed":41.5}`, + `{"Action":"fail","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan/scan-npm","Elapsed":2}`, + `{"Action":"skip","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan/scan-bun","Elapsed":0}`, + `{"Action":"fail","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Test":"TestScan","Elapsed":44}`, + `{"Action":"fail","Package":"github.com/bomly-dev/bomly-cli/test/smoke","Elapsed":44}`, + "go: downloading something", + }, "\n") + summary, err := ParseGoTestEvents(strings.NewReader(stream), nil) + if err != nil { + t.Fatalf("parse: %v", err) + } + if summary.Total != 4 || summary.Passed != 1 || summary.Failed != 2 || summary.Skipped != 1 { + t.Fatalf("summary = %+v", summary) + } + if len(summary.Anomalies) != 1 { + t.Fatalf("anomalies = %v, want the non-JSON line", summary.Anomalies) + } + result := summary.ToCheckResult(CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Instance: "go", + Stage: StagePrerequisites, Level: LevelGate, + }, 1) + if err := result.Validate(); err != nil { + t.Fatalf("converted result invalid: %v", err) + } + if result.Status != StatusFail { + t.Fatalf("status = %s, want fail", result.Status) + } + if result.Metrics["tests_failed"] != 2 { + t.Fatalf("tests_failed = %v", result.Metrics["tests_failed"]) + } +} + +func TestParseGoTestEventsBuildFailure(t *testing.T) { + summary, err := ParseGoTestEvents(strings.NewReader("# github.com/example\nsyntax error\n"), nil) + if err != nil { + t.Fatalf("parse: %v", err) + } + result := summary.ToCheckResult(CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Stage: StagePrerequisites, Level: LevelGate, + }, 2) + if result.Status != StatusFail { + t.Fatalf("status = %s, want fail when no test ran and the command failed", result.Status) + } + if !strings.Contains(result.Summary, "exited with code 2") { + t.Fatalf("summary = %q", result.Summary) + } +} diff --git a/internal/assurance/catalog.go b/internal/assurance/catalog.go new file mode 100644 index 00000000..3a89da99 --- /dev/null +++ b/internal/assurance/catalog.go @@ -0,0 +1,495 @@ +package assurance + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// CatalogSchema is the schema identifier of the assurance catalog. +const CatalogSchema = "bomly.assurance-catalog/v1" + +// MaxCatalogBytes bounds the catalog document. +const MaxCatalogBytes = 4 << 20 + +// DefaultCatalogPath is the repository-relative catalog location. +const DefaultCatalogPath = "docs/assurance/catalog.json" + +var ( + revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + containerDigestPattern = regexp.MustCompile(`@sha256:[0-9a-f]{64}$`) +) + +// EvidenceLevel describes how strong a public evidence claim is. +type EvidenceLevel string + +// Evidence strength levels, from strongest to weakest guarantee. +const ( + // EvidenceDeterministic is reproducible in process with no external input. + EvidenceDeterministic EvidenceLevel = "deterministic" + // EvidencePinnedInput depends on a pinned repository revision or fixture. + EvidencePinnedInput EvidenceLevel = "pinned-input" + // EvidenceSnapshot depends on an upstream tag that can move. + EvidenceSnapshot EvidenceLevel = "snapshot" + // EvidenceLiveService depends on data from a live service. + EvidenceLiveService EvidenceLevel = "live-service" + // EvidenceReleaseArtifact is measured against published release artifacts. + EvidenceReleaseArtifact EvidenceLevel = "release-artifact" + // EvidencePlatformMatrix comes from repeated runs across platforms. + EvidencePlatformMatrix EvidenceLevel = "platform-matrix" +) + +// Valid reports whether the evidence level is known. +func (e EvidenceLevel) Valid() bool { + switch e { + case EvidenceDeterministic, EvidencePinnedInput, EvidenceSnapshot, + EvidenceLiveService, EvidenceReleaseArtifact, EvidencePlatformMatrix: + return true + default: + return false + } +} + +// Catalog declares every check the framework expects and every public evidence +// claim those checks back. It is the single source the report is built against. +type Catalog struct { + SchemaVersion string `json:"schema_version"` + Areas []Area `json:"areas"` + Checks []Check `json:"checks"` + Evidence []Evidence `json:"evidence"` +} + +// Area groups related checks and evidence for presentation. +type Area struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` +} + +// Source records which workflow and job produce a check's results. +type Source struct { + Workflow string `json:"workflow"` + Job string `json:"job"` +} + +// ExpectedInstance declares one matrix leg a check must report. +type ExpectedInstance struct { + Name string `json:"name"` + Ecosystems []string `json:"ecosystems,omitempty"` + Platform string `json:"platform,omitempty"` +} + +// Check declares one quality check: which stage runs it, whether it gates the +// release, which instances must report, and what it does and does not prove. +type Check struct { + ID string `json:"id"` + Title string `json:"title"` + Area string `json:"area"` + Stage Stage `json:"stage"` + Level Level `json:"level"` + Description string `json:"description"` + Source Source `json:"source"` + ExpectedInstances []ExpectedInstance `json:"expected_instances,omitempty"` + Reproduce [][]string `json:"reproduce,omitempty"` + Proves []string `json:"proves"` + Limitations []string `json:"limitations"` +} + +// Input names a pinned repository, fixture, container image, or service an +// evidence claim depends on. +type Input struct { + Kind string `json:"kind"` + Location string `json:"location"` + Ref string `json:"ref,omitempty"` + Revision string `json:"revision,omitempty"` + SHA256 string `json:"sha256,omitempty"` +} + +// EvidenceArtifact is a repository file that records an evidence claim's result. +type EvidenceArtifact struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` +} + +// Evidence is one public claim about Bomly's behavior, backed by a check whose +// per-release status shows whether the claim still holds. +type Evidence struct { + ID string `json:"id"` + Title string `json:"title"` + Area string `json:"area"` + EvidenceLevel EvidenceLevel `json:"evidence_level"` + CheckID string `json:"check_id"` + Instance string `json:"instance,omitempty"` + Inputs []Input `json:"inputs"` + RequiredTools []string `json:"required_tools,omitempty"` + Reproduce [][]string `json:"reproduce"` + Artifacts []EvidenceArtifact `json:"artifacts,omitempty"` + Proves []string `json:"proves"` + Limitations []string `json:"limitations"` +} + +// ParseCatalog decodes and structurally validates a catalog document. It does +// not touch the filesystem; use VerifyArtifacts for the repository-side checks. +func ParseCatalog(data []byte) (Catalog, error) { + if len(data) > MaxCatalogBytes { + return Catalog{}, fmt.Errorf("catalog is %d bytes, limit is %d", len(data), MaxCatalogBytes) + } + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + var catalog Catalog + if err := decoder.Decode(&catalog); err != nil { + return Catalog{}, fmt.Errorf("decode assurance catalog: %w", err) + } + if err := ensureEOF(decoder, "assurance catalog"); err != nil { + return Catalog{}, err + } + if err := catalog.Validate(); err != nil { + return Catalog{}, err + } + return catalog, nil +} + +// LoadCatalog reads and validates the catalog at path. +func LoadCatalog(path string) (Catalog, error) { + data, err := readBounded(path, MaxCatalogBytes) + if err != nil { + return Catalog{}, err + } + return ParseCatalog(data) +} + +// Validate reports whether the catalog is internally consistent. +func (c Catalog) Validate() error { + if c.SchemaVersion != CatalogSchema { + return fmt.Errorf("unsupported assurance catalog schema %q", c.SchemaVersion) + } + if len(c.Areas) == 0 { + return errCatalog("catalog declares no areas") + } + areas := make(map[string]struct{}, len(c.Areas)) + for index, area := range c.Areas { + if !idPattern.MatchString(area.ID) { + return fmt.Errorf("area %d has invalid id %q", index+1, area.ID) + } + if _, exists := areas[area.ID]; exists { + return fmt.Errorf("duplicate area %q", area.ID) + } + if strings.TrimSpace(area.Title) == "" || strings.TrimSpace(area.Description) == "" { + return fmt.Errorf("area %q requires a title and description", area.ID) + } + areas[area.ID] = struct{}{} + } + if len(c.Checks) == 0 { + return errCatalog("catalog declares no checks") + } + checks := make(map[string]Check, len(c.Checks)) + previous := "" + for index, check := range c.Checks { + if !idPattern.MatchString(check.ID) { + return fmt.Errorf("check %d has invalid id %q", index+1, check.ID) + } + if _, exists := checks[check.ID]; exists { + return fmt.Errorf("duplicate check %q", check.ID) + } + if previous != "" && check.ID < previous { + return fmt.Errorf("checks are not sorted by id: %q follows %q", check.ID, previous) + } + previous = check.ID + if err := validateCheck(check, areas); err != nil { + return fmt.Errorf("check %q: %w", check.ID, err) + } + checks[check.ID] = check + } + if len(c.Evidence) == 0 { + return errCatalog("catalog declares no evidence") + } + seenEvidence := make(map[string]struct{}, len(c.Evidence)) + previous = "" + for index, evidence := range c.Evidence { + if !idPattern.MatchString(evidence.ID) { + return fmt.Errorf("evidence %d has invalid id %q", index+1, evidence.ID) + } + if _, exists := seenEvidence[evidence.ID]; exists { + return fmt.Errorf("duplicate evidence %q", evidence.ID) + } + if previous != "" && evidence.ID < previous { + return fmt.Errorf("evidence is not sorted by id: %q follows %q", evidence.ID, previous) + } + previous = evidence.ID + seenEvidence[evidence.ID] = struct{}{} + if err := validateEvidence(evidence, areas, checks); err != nil { + return fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + } + return nil +} + +func validateCheck(check Check, areas map[string]struct{}) error { + if strings.TrimSpace(check.Title) == "" || strings.TrimSpace(check.Description) == "" { + return errCatalog("title and description are required") + } + if _, exists := areas[check.Area]; !exists { + return fmt.Errorf("unknown area %q", check.Area) + } + if !check.Stage.Valid() { + return fmt.Errorf("unsupported stage %q", check.Stage) + } + if !check.Level.Valid() { + return fmt.Errorf("unsupported level %q", check.Level) + } + if strings.TrimSpace(check.Source.Workflow) == "" || strings.TrimSpace(check.Source.Job) == "" { + return errCatalog("source workflow and job are required") + } + seen := make(map[string]struct{}, len(check.ExpectedInstances)) + for _, instance := range check.ExpectedInstances { + if !instancePattern.MatchString(instance.Name) { + return fmt.Errorf("invalid expected instance %q", instance.Name) + } + if _, exists := seen[instance.Name]; exists { + return fmt.Errorf("duplicate expected instance %q", instance.Name) + } + seen[instance.Name] = struct{}{} + } + if err := validateCommands(check.Reproduce); err != nil { + return err + } + return validateClaims(check.Proves, check.Limitations) +} + +func validateEvidence(evidence Evidence, areas map[string]struct{}, checks map[string]Check) error { + if strings.TrimSpace(evidence.Title) == "" { + return errCatalog("title is required") + } + if _, exists := areas[evidence.Area]; !exists { + return fmt.Errorf("unknown area %q", evidence.Area) + } + if !evidence.EvidenceLevel.Valid() { + return fmt.Errorf("unsupported evidence level %q", evidence.EvidenceLevel) + } + check, exists := checks[evidence.CheckID] + if !exists { + return fmt.Errorf("unknown backing check %q", evidence.CheckID) + } + if evidence.Instance != "" { + if !instancePattern.MatchString(evidence.Instance) { + return fmt.Errorf("invalid instance %q", evidence.Instance) + } + if !hasInstance(check, evidence.Instance) { + return fmt.Errorf("check %q does not declare instance %q", evidence.CheckID, evidence.Instance) + } + } + if len(evidence.Inputs) == 0 { + return errCatalog("at least one input is required") + } + for _, input := range evidence.Inputs { + if err := validateInput(evidence.EvidenceLevel, input); err != nil { + return err + } + } + if len(evidence.Reproduce) == 0 { + return errCatalog("at least one reproduction command is required") + } + if err := validateCommands(evidence.Reproduce); err != nil { + return err + } + if len(evidence.Artifacts) == 0 && requiresArtifact(evidence) { + return errCatalog("repository-backed evidence requires at least one hashed artifact") + } + for _, artifact := range evidence.Artifacts { + if !hashPattern.MatchString(artifact.SHA256) { + return fmt.Errorf("artifact %q has an invalid SHA-256 hash", artifact.Path) + } + if err := validateRepoPath(artifact.Path); err != nil { + return err + } + } + return validateClaims(evidence.Proves, evidence.Limitations) +} + +// requiresArtifact reports whether an evidence claim must point at a hashed +// repository file. Claims proven by committed fixtures and goldens must; claims +// proven by running a workflow against a release are recorded by the per-release +// check result instead. +func requiresArtifact(evidence Evidence) bool { + switch evidence.EvidenceLevel { + case EvidenceDeterministic, EvidencePinnedInput, EvidenceSnapshot, EvidenceLiveService: + return true + default: + return false + } +} + +func hasInstance(check Check, name string) bool { + for _, instance := range check.ExpectedInstances { + if instance.Name == name { + return true + } + } + return false +} + +func validateInput(level EvidenceLevel, input Input) error { + if strings.TrimSpace(input.Location) == "" { + return errCatalog("input location is required") + } + switch input.Kind { + case "git": + if !revisionPattern.MatchString(input.Revision) { + return errCatalog("git input requires a full lowercase commit revision") + } + case "fixture": + if !hashPattern.MatchString(input.SHA256) { + return errCatalog("fixture input requires a SHA-256 hash") + } + return validateRepoPath(input.Location) + case "container": + if input.Ref == "" { + return errCatalog("container input requires an image reference") + } + if level == EvidencePinnedInput && !containerDigestPattern.MatchString(input.Ref) { + return errCatalog("pinned container input requires an immutable sha256 digest") + } + case "release", "service", "workflow": + // A published release asset, an external service, or a CI workflow: + // identified by location. The per-release check result is the record + // that it ran, so there is no repository hash to pin. + default: + return fmt.Errorf("unsupported input kind %q", input.Kind) + } + return nil +} + +func validateCommands(commands [][]string) error { + for index, command := range commands { + if len(command) == 0 { + return fmt.Errorf("command %d is empty", index+1) + } + for _, argument := range command { + if argument == "" { + return fmt.Errorf("command %d contains an empty argument", index+1) + } + } + } + return nil +} + +func validateClaims(proves, limitations []string) error { + if len(proves) == 0 || len(limitations) == 0 { + return errCatalog("proves and limitations must both be explicit") + } + for _, entry := range append(append([]string{}, proves...), limitations...) { + if strings.TrimSpace(entry) == "" { + return errCatalog("proves and limitations cannot contain blank entries") + } + } + return nil +} + +func validateRepoPath(path string) error { + clean := filepath.Clean(filepath.FromSlash(path)) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return fmt.Errorf("path %q must stay inside the repository", path) + } + return nil +} + +// VerifyArtifacts confirms every catalog artifact and fixture input still +// matches its recorded hash inside root. +func (c Catalog) VerifyArtifacts(root string) error { + for _, evidence := range c.Evidence { + for _, artifact := range evidence.Artifacts { + if err := verifyHashedFile(root, artifact.Path, artifact.SHA256); err != nil { + return fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + } + for _, input := range evidence.Inputs { + if input.Kind != "fixture" { + continue + } + if err := verifyHashedFile(root, input.Location, input.SHA256); err != nil { + return fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + } + } + return nil +} + +func verifyHashedFile(root, path, want string) error { + if err := validateRepoPath(path); err != nil { + return err + } + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return fmt.Errorf("resolve repository root: %w", err) + } + resolved, err := filepath.EvalSymlinks(filepath.Join(root, filepath.FromSlash(path))) + if err != nil { + return fmt.Errorf("resolve %q: %w", path, err) + } + relative, err := filepath.Rel(resolvedRoot, resolved) + if err != nil { + return fmt.Errorf("resolve %q relative to repository: %w", path, err) + } + if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("path %q resolves outside the repository", path) + } + info, err := os.Stat(resolved) + if err != nil { + return fmt.Errorf("inspect %q: %w", path, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%q is not a regular file", path) + } + data, err := os.ReadFile(resolved) + if err != nil { + return fmt.Errorf("read %q: %w", path, err) + } + sum := sha256.Sum256(data) + if actual := hex.EncodeToString(sum[:]); actual != want { + return fmt.Errorf("%q hash is %s, want %s", path, actual, want) + } + return nil +} + +// ChecksForStage returns the catalog checks that belong to one stage. +func (c Catalog) ChecksForStage(stage Stage) []Check { + var selected []Check + for _, check := range c.Checks { + if check.Stage == stage { + selected = append(selected, check) + } + } + return selected +} + +// Check looks up one catalog check by ID. +func (c Catalog) Check(id string) (Check, bool) { + for _, check := range c.Checks { + if check.ID == id { + return check, true + } + } + return Check{}, false +} + +// AreaTitle returns the display title of an area, falling back to its ID. +func (c Catalog) AreaTitle(id string) string { + for _, area := range c.Areas { + if area.ID == id { + return area.Title + } + } + return id +} + +type catalogError string + +func (e catalogError) Error() string { return string(e) } + +func errCatalog(message string) error { return catalogError(message) } diff --git a/internal/assurance/catalog_test.go b/internal/assurance/catalog_test.go new file mode 100644 index 00000000..0b2987cc --- /dev/null +++ b/internal/assurance/catalog_test.go @@ -0,0 +1,187 @@ +package assurance + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "gopkg.in/yaml.v3" +) + +// repositoryRoot is the module root relative to this package. +const repositoryRoot = "../.." + +func repositoryCatalog(t *testing.T) Catalog { + t.Helper() + catalog, err := LoadCatalog(filepath.Join(repositoryRoot, filepath.FromSlash(DefaultCatalogPath))) + if err != nil { + t.Fatalf("load repository catalog: %v", err) + } + return catalog +} + +func TestRepositoryCatalogIsValid(t *testing.T) { + catalog := repositoryCatalog(t) + if err := catalog.VerifyArtifacts(repositoryRoot); err != nil { + t.Fatalf("catalog artifacts drifted: %v", err) + } + for _, stage := range Stages() { + if len(catalog.ChecksForStage(stage)) == 0 { + t.Fatalf("no checks declared for the %s stage", stage) + } + } + gates := 0 + for _, check := range catalog.Checks { + if check.Level == LevelGate { + gates++ + } + } + if gates == 0 { + t.Fatal("the catalog declares no gate checks, so nothing could block a release") + } +} + +// TestCatalogSmokeInstancesMatchWorkflowMatrix keeps the declared smoke slices +// and the workflow matrix in step: a slice added to one and not the other would +// otherwise silently drop out of the report. +func TestCatalogSmokeInstancesMatchWorkflowMatrix(t *testing.T) { + catalog := repositoryCatalog(t) + check, found := catalog.Check("smoke") + if !found { + t.Fatal("the catalog does not declare the smoke check") + } + declared := map[string]struct{}{} + for _, instance := range check.ExpectedInstances { + declared[instance.Name] = struct{}{} + } + + data, err := os.ReadFile(filepath.Join(repositoryRoot, ".github", "workflows", "smoke.yml")) + if err != nil { + t.Fatalf("read smoke workflow: %v", err) + } + var workflow struct { + Jobs map[string]struct { + Strategy struct { + Matrix struct { + Slice []struct { + Name string `yaml:"name"` + } `yaml:"slice"` + } `yaml:"matrix"` + } `yaml:"strategy"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(data, &workflow); err != nil { + t.Fatalf("decode smoke workflow: %v", err) + } + slices := workflow.Jobs["smoke"].Strategy.Matrix.Slice + if len(slices) == 0 { + t.Fatal("the smoke workflow declares no slices") + } + seen := map[string]struct{}{} + for _, slice := range slices { + seen[slice.Name] = struct{}{} + if _, ok := declared[slice.Name]; !ok { + t.Errorf("smoke slice %q runs in CI but is not declared in %s", slice.Name, DefaultCatalogPath) + } + } + for name := range declared { + if _, ok := seen[name]; !ok { + t.Errorf("smoke slice %q is declared in the catalog but not in the workflow matrix", name) + } + } +} + +func TestCatalogRejectsInvalidDocuments(t *testing.T) { + base := func() Catalog { + return Catalog{ + SchemaVersion: CatalogSchema, + Areas: []Area{{ID: "end-to-end", Title: "End to end", Description: "Real runs."}}, + Checks: []Check{{ + ID: "smoke", Title: "Smoke", Area: "end-to-end", Stage: StagePrerequisites, + Level: LevelGate, Description: "Runs real scans.", + Source: Source{Workflow: "smoke.yml", Job: "smoke"}, + Proves: []string{"It scans."}, Limitations: []string{"One project per ecosystem."}, + }}, + Evidence: []Evidence{{ + ID: "graph-go", Title: "Go graph", Area: "end-to-end", + EvidenceLevel: EvidenceReleaseArtifact, CheckID: "smoke", + Inputs: []Input{{Kind: "release", Location: "https://example.test/release"}}, + Reproduce: [][]string{{"make", "smoke"}}, + Proves: []string{"It resolves."}, + Limitations: []string{"One toolchain."}, + }}, + } + } + if _, err := ParseCatalog(mustJSON(t, base())); err != nil { + t.Fatalf("valid catalog rejected: %v", err) + } + + cases := map[string]func(*Catalog){ + "bad schema": func(c *Catalog) { c.SchemaVersion = "other/v1" }, + "no areas": func(c *Catalog) { c.Areas = nil }, + "unknown check area": func(c *Catalog) { c.Checks[0].Area = "nowhere" }, + "bad stage": func(c *Catalog) { c.Checks[0].Stage = "later" }, + "bad level": func(c *Catalog) { c.Checks[0].Level = "blocking" }, + "no source": func(c *Catalog) { c.Checks[0].Source = Source{} }, + "blank claim": func(c *Catalog) { c.Checks[0].Proves = []string{" "} }, + "no limitations": func(c *Catalog) { c.Checks[0].Limitations = nil }, + "unknown backing": func(c *Catalog) { c.Evidence[0].CheckID = "nothing" }, + "undeclared instance": func(c *Catalog) { c.Evidence[0].Instance = "go" }, + "bad evidence level": func(c *Catalog) { c.Evidence[0].EvidenceLevel = "vibes" }, + "no inputs": func(c *Catalog) { c.Evidence[0].Inputs = nil }, + "git without revision": func(c *Catalog) { + c.Evidence[0].Inputs = []Input{{Kind: "git", Location: "https://example.test/repo"}} + }, + "fixture without hash": func(c *Catalog) { + c.Evidence[0].Inputs = []Input{{Kind: "fixture", Location: "go.mod"}} + }, + "escaping fixture path": func(c *Catalog) { + c.Evidence[0].Inputs = []Input{{ + Kind: "fixture", Location: "../../../etc/passwd", + SHA256: "0000000000000000000000000000000000000000000000000000000000000000", + }} + }, + "deterministic evidence needs an artifact": func(c *Catalog) { + c.Evidence[0].EvidenceLevel = EvidenceDeterministic + }, + "unsorted checks": func(c *Catalog) { + c.Checks = append(c.Checks, c.Checks[0]) + c.Checks[1].ID = "aaa" + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + catalog := base() + mutate(&catalog) + if _, err := ParseCatalog(mustJSON(t, catalog)); err == nil { + t.Fatal("expected the catalog to be rejected") + } + }) + } +} + +func TestVerifyArtifactsDetectsDrift(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } + catalog := Catalog{Evidence: []Evidence{{ + ID: "example", + Artifacts: []EvidenceArtifact{{ + Path: "golden.json", SHA256: "1111111111111111111111111111111111111111111111111111111111111111", + }}, + }}} + if err := catalog.VerifyArtifacts(root); err == nil { + t.Fatal("expected a hash mismatch to be reported") + } +} + +func mustJSON(t *testing.T, value any) []byte { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return data +} diff --git a/internal/assurance/cmd/commands.go b/internal/assurance/cmd/commands.go new file mode 100644 index 00000000..bed30091 --- /dev/null +++ b/internal/assurance/cmd/commands.go @@ -0,0 +1,546 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "time" + + "github.com/bomly-dev/bomly-cli/internal/assurance" +) + +func goos() string { return runtime.GOOS } +func goarch() string { return runtime.GOARCH } +func goVersion() string { return runtime.Version() } + +// ------------------------------------------------------------------ emit --- + +func runEmit(args []string) error { + flags := flag.NewFlagSet("emit", flag.ExitOnError) + id := flags.String("id", "", "catalog check id") + instance := flags.String("instance", "", "matrix instance name, such as an ecosystem or platform") + stage := flags.String("stage", "", "override the stage declared in the catalog") + level := flags.String("level", "", "override the level declared in the catalog") + status := flags.String("status", "", "explicit status: pass, fail, degraded, or skip") + exitCode := flags.Int("exit-code", 0, "command exit code that decides the status") + summary := flags.String("summary", "", "one-line plain-language summary") + durationMS := flags.Float64("duration-ms", 0, "check duration in milliseconds") + startedAt := flags.String("started-at", "", "RFC 3339 start time") + out := flags.String("out", "assurance-results", "directory to write the check result into") + catalogPath := flags.String("catalog", "", "assurance catalog path") + detailsFile := flags.String("details-jsonl", "", "file of JSON detail objects, one per line") + stepSummary := flags.Bool("step-summary", false, "append a markdown block to the workflow step summary") + var metrics, details, artifacts, links stringList + flags.Var(&metrics, "metric", "numeric metric as name=value (repeatable)") + flags.Var(&details, "detail", "sub-result as name=status[:note] (repeatable)") + flags.Var(&artifacts, "artifact", "produced file as name=path (repeatable)") + flags.Var(&links, "link", "related link as label=url (repeatable)") + if err := flags.Parse(args); err != nil { + return err + } + if *id == "" || *summary == "" { + return fmt.Errorf("emit requires --id and --summary") + } + + result := baseResult(*id, *instance) + if *startedAt != "" { + result.StartedAt = *startedAt + } + result.FinishedAt = time.Now().UTC().Format(time.RFC3339) + result.DurationMS = *durationMS + if result.DurationMS == 0 { + result.DurationMS = elapsedMS(result.StartedAt, result.FinishedAt) + } + result.Summary = *summary + result.Status = statusFromExit(*exitCode) + if *status != "" { + result.Status = assurance.Status(*status) + } + catalogCtx := loadContext(*catalogPath) + if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { + return err + } + for _, entry := range metrics { + name, raw, err := splitPair(entry, "metric") + if err != nil { + return err + } + value, convErr := strconv.ParseFloat(raw, 64) + if convErr != nil { + return fmt.Errorf("metric %q must be numeric: %w", entry, convErr) + } + if result.Metrics == nil { + result.Metrics = map[string]float64{} + } + result.Metrics[name] = value + } + for _, entry := range details { + detail, err := parseDetail(entry) + if err != nil { + return err + } + result.Details = append(result.Details, detail) + } + if *detailsFile != "" { + loaded, err := readDetailLines(*detailsFile) + if err != nil { + return err + } + result.Details = append(result.Details, loaded...) + } + for _, entry := range artifacts { + name, path, err := splitPair(entry, "artifact") + if err != nil { + return err + } + artifact := assurance.Artifact{Name: name} + if info, statErr := os.Stat(path); statErr == nil && info.Mode().IsRegular() { + artifact.Bytes = info.Size() + } + result.Artifacts = append(result.Artifacts, artifact) + } + for _, entry := range links { + label, url, err := splitPair(entry, "link") + if err != nil { + return err + } + result.Links = append(result.Links, assurance.Link{Label: label, URL: url}) + } + return writeResult(*out, result, *stepSummary) +} + +func parseDetail(entry string) (assurance.Detail, error) { + name, rest, err := splitPair(entry, "detail") + if err != nil { + return assurance.Detail{}, err + } + status, note, _ := strings.Cut(rest, ":") + detail := assurance.Detail{ + Name: name, + Status: assurance.Status(strings.TrimSpace(status)), + Note: strings.TrimSpace(note), + } + if !detail.Status.Valid() { + return assurance.Detail{}, fmt.Errorf("detail %q has unsupported status %q", entry, status) + } + return detail, nil +} + +// readDetailLines reads sub-results from a JSONL file so shell steps can record +// many items without building long command lines. +func readDetailLines(path string) ([]assurance.Detail, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open details file: %w", err) + } + defer file.Close() + var details []assurance.Detail + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + var raw struct { + Name string `json:"name"` + Status string `json:"status"` + ExitCode *int `json:"exit_code"` + Note string `json:"note"` + DurationMS float64 `json:"duration_ms"` + DurationS float64 `json:"duration_s"` + } + if err := json.Unmarshal([]byte(line), &raw); err != nil { + return nil, fmt.Errorf("decode details file line: %w", err) + } + detail := assurance.Detail{Name: raw.Name, Note: raw.Note, DurationMS: raw.DurationMS} + if detail.DurationMS == 0 && raw.DurationS > 0 { + detail.DurationMS = raw.DurationS * 1000 + } + switch { + case raw.Status != "": + detail.Status = assurance.Status(raw.Status) + case raw.ExitCode != nil: + detail.Status = statusFromExit(*raw.ExitCode) + default: + detail.Status = assurance.StatusPass + } + if detail.Name == "" || !detail.Status.Valid() { + return nil, fmt.Errorf("details file line %q needs a name and a valid status", line) + } + details = append(details, detail) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read details file: %w", err) + } + return details, nil +} + +func elapsedMS(started, finished string) float64 { + start, err := time.Parse(time.RFC3339, started) + if err != nil { + return 0 + } + end, err := time.Parse(time.RFC3339, finished) + if err != nil { + return 0 + } + return float64(end.Sub(start).Milliseconds()) +} + +// ---------------------------------------------------------------- gotest --- + +func runGoTest(args []string) error { + flags := flag.NewFlagSet("gotest", flag.ExitOnError) + id := flags.String("id", "", "catalog check id") + instance := flags.String("instance", "", "matrix instance name") + stage := flags.String("stage", "", "override the stage declared in the catalog") + level := flags.String("level", "", "override the level declared in the catalog") + input := flags.String("input", "", `file holding "go test -json" output`) + exitCode := flags.Int("exit-code", 0, "exit code of the test command") + echo := flags.Bool("echo", false, "replay test output to stdout so CI logs stay readable") + out := flags.String("out", "assurance-results", "directory to write the check result into") + catalogPath := flags.String("catalog", "", "assurance catalog path") + stepSummary := flags.Bool("step-summary", false, "append a markdown block to the workflow step summary") + if err := flags.Parse(args); err != nil { + return err + } + if *id == "" || *input == "" { + return fmt.Errorf("gotest requires --id and --input") + } + file, err := os.Open(*input) + if err != nil { + return fmt.Errorf("open test output: %w", err) + } + defer file.Close() + var echoWriter *os.File + if *echo { + echoWriter = os.Stdout + } + summary, err := assurance.ParseGoTestEvents(file, writerOrNil(echoWriter)) + if err != nil { + return err + } + result := baseResult(*id, *instance) + result.FinishedAt = time.Now().UTC().Format(time.RFC3339) + catalogCtx := loadContext(*catalogPath) + if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { + return err + } + return writeResult(*out, summary.ToCheckResult(result, *exitCode), *stepSummary) +} + +func writerOrNil(file *os.File) *os.File { + if file == nil { + return nil + } + return file +} + +// --------------------------------------------------------------- convert --- + +func runConvert(args []string) error { + if len(args) == 0 { + return fmt.Errorf("convert requires a manifest kind: benchmark-run or sbom-assurance") + } + kind := args[0] + flags := flag.NewFlagSet("convert "+kind, flag.ExitOnError) + id := flags.String("id", "", "catalog check id") + instance := flags.String("instance", "", "matrix instance name") + stage := flags.String("stage", "", "override the stage declared in the catalog") + level := flags.String("level", "", "override the level declared in the catalog") + input := flags.String("input", "", "manifest path") + out := flags.String("out", "assurance-results", "directory to write the check result into") + catalogPath := flags.String("catalog", "", "assurance catalog path") + stepSummary := flags.Bool("step-summary", false, "append a markdown block to the workflow step summary") + if err := flags.Parse(args[1:]); err != nil { + return err + } + if *id == "" || *input == "" { + return fmt.Errorf("convert requires --id and --input") + } + data, err := os.ReadFile(*input) + if err != nil { + return fmt.Errorf("read manifest: %w", err) + } + result := baseResult(*id, *instance) + result.FinishedAt = time.Now().UTC().Format(time.RFC3339) + catalogCtx := loadContext(*catalogPath) + if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { + return err + } + var converted assurance.CheckResult + switch kind { + case "benchmark-run": + converted, err = assurance.ConvertBenchmarkRun(data, result) + case "sbom-assurance": + converted, err = assurance.ConvertSBOMAssurance(data, result) + default: + return fmt.Errorf("unsupported manifest kind %q", kind) + } + if err != nil { + return err + } + return writeResult(*out, converted, *stepSummary) +} + +// -------------------------------------------------------- verify-release --- + +func runVerifyRelease(args []string) error { + flags := flag.NewFlagSet("verify-release", flag.ExitOnError) + dir := flags.String("dir", "", "directory holding downloaded release assets") + version := flags.String("version", "", "release version without the leading v") + scope := flags.String("scope", "full", "full checks every expected asset; native checks only this platform's") + out := flags.String("out", "assurance-results", "directory to write check results into") + workDir := flags.String("work", "", "extraction directory (a temporary directory by default)") + catalogPath := flags.String("catalog", "", "assurance catalog path") + stepSummary := flags.Bool("step-summary", false, "append markdown blocks to the workflow step summary") + if err := flags.Parse(args); err != nil { + return err + } + if *dir == "" || *version == "" { + return fmt.Errorf("verify-release requires --dir and --version") + } + trimmed := strings.TrimPrefix(*version, "v") + catalogCtx := loadContext(*catalogPath) + work := *workDir + if work == "" { + created, err := os.MkdirTemp("", "assurance-release-") + if err != nil { + return fmt.Errorf("create extraction directory: %w", err) + } + defer os.RemoveAll(created) + work = created + } + + failures := 0 + emit := func(result assurance.CheckResult) error { + if result.Status == assurance.StatusFail { + failures++ + } + return writeResult(*out, result, *stepSummary) + } + + if *scope == "full" { + presence, err := assurance.InspectAssets(*dir, trimmed) + if err != nil { + return err + } + result := baseResult("release-assets", "") + result.FinishedAt = time.Now().UTC().Format(time.RFC3339) + if err := applyCatalog(&result, catalogCtx.catalog, "", ""); err != nil { + return err + } + result.Status = assurance.StatusPass + result.Metrics = map[string]float64{ + "assets": float64(len(presence.Present)), + "assets_missing": float64(len(presence.Missing)), + } + for _, name := range presence.Present { + result.Details = append(result.Details, assurance.Detail{Name: name, Status: assurance.StatusPass}) + } + for _, name := range presence.Missing { + result.Status = assurance.StatusFail + result.Details = append(result.Details, assurance.Detail{ + Name: name, Status: assurance.StatusFail, Note: "not attached to the release", + }) + } + result.Summary = fmt.Sprintf("%d of %d expected release assets are attached.", + len(presence.Present), len(presence.Present)+len(presence.Missing)) + if len(presence.Extra) > 0 { + result.Summary += fmt.Sprintf(" %d unexpected file(s): %s.", + len(presence.Extra), strings.Join(presence.Extra, ", ")) + } + if err := emit(result); err != nil { + return err + } + } + + sumsData, err := os.ReadFile(filepath.Join(*dir, "SHA256SUMS")) + if err != nil { + return fmt.Errorf("read SHA256SUMS: %w", err) + } + entries, err := assurance.ParseSHA256SUMS(sumsData) + if err != nil { + return err + } + outcome, err := assurance.VerifyChecksums(*dir, entries) + if err != nil { + return err + } + checksums := baseResult("release-checksums", "") + checksums.FinishedAt = time.Now().UTC().Format(time.RFC3339) + if err := applyCatalog(&checksums, catalogCtx.catalog, "", ""); err != nil { + return err + } + checksums.Status = assurance.StatusPass + checksums.Metrics = map[string]float64{ + "verified": float64(len(outcome.Verified)), + "mismatched": float64(len(outcome.Mismatched)), + "listed": float64(len(entries)), + } + for _, name := range outcome.Verified { + checksums.Details = append(checksums.Details, assurance.Detail{Name: name, Status: assurance.StatusPass}) + } + for _, name := range outcome.Mismatched { + checksums.Status = assurance.StatusFail + checksums.Details = append(checksums.Details, assurance.Detail{ + Name: name, Status: assurance.StatusFail, Note: "hash does not match SHA256SUMS", + }) + } + for _, name := range outcome.Unlisted { + checksums.Status = assurance.StatusFail + checksums.Details = append(checksums.Details, assurance.Detail{ + Name: name, Status: assurance.StatusFail, Note: "downloaded but absent from SHA256SUMS", + }) + } + if *scope == "full" && len(outcome.NotDownloaded) > 0 { + checksums.Status = assurance.StatusFail + for _, name := range outcome.NotDownloaded { + checksums.Details = append(checksums.Details, assurance.Detail{ + Name: name, Status: assurance.StatusFail, Note: "listed in SHA256SUMS but not attached", + }) + } + } + checksums.Summary = fmt.Sprintf("%d downloaded assets match SHA256SUMS, which lists %d files.", + len(outcome.Verified), len(entries)) + if len(outcome.Mismatched) > 0 { + checksums.Summary = fmt.Sprintf("%d assets do not match SHA256SUMS: %s.", + len(outcome.Mismatched), strings.Join(outcome.Mismatched, ", ")) + } + if err := emit(checksums); err != nil { + return err + } + + probes, err := assurance.ProbeNativeBinaries(context.Background(), *dir, trimmed, work) + if err != nil { + return err + } + binaries := baseResult("release-binaries", runtime.GOOS+"-"+runtime.GOARCH) + binaries.FinishedAt = time.Now().UTC().Format(time.RFC3339) + if err := applyCatalog(&binaries, catalogCtx.catalog, "", ""); err != nil { + return err + } + binaries.Status = assurance.StatusPass + passed := 0 + for _, probe := range probes { + note := probe.Note + if probe.Status == assurance.StatusPass { + note = probe.Output + passed++ + } else { + binaries.Status = assurance.StatusFail + } + binaries.Details = append(binaries.Details, assurance.Detail{ + Name: probe.Archive, Status: probe.Status, Note: firstLine(note), + }) + } + binaries.Metrics = map[string]float64{"binaries": float64(len(probes)), "binaries_passed": float64(passed)} + binaries.Summary = fmt.Sprintf("%d of %d %s/%s binaries report version %s.", + passed, len(probes), runtime.GOOS, runtime.GOARCH, trimmed) + if err := emit(binaries); err != nil { + return err + } + + if failures > 0 { + return blockingError{message: fmt.Sprintf("%d release verification check(s) failed", failures), code: 1} + } + return nil +} + +func firstLine(value string) string { + if index := strings.IndexAny(value, "\r\n"); index >= 0 { + return strings.TrimSpace(value[:index]) + } + return strings.TrimSpace(value) +} + +// --------------------------------------------------------------- verdict --- + +func runVerdict(args []string) error { + flags := flag.NewFlagSet("verdict", flag.ExitOnError) + resultsDir := flags.String("results", "assurance-results", "directory of collected check results") + stage := flags.String("stage", "", "stage to judge") + catalogPath := flags.String("catalog", "", "assurance catalog path") + tag := flags.String("tag", "", "release tag, when one exists") + stepSummary := flags.Bool("step-summary", false, "append the verdict to the workflow step summary") + jsonOut := flags.String("json", "", "also write the stage report as JSON to this path") + if err := flags.Parse(args); err != nil { + return err + } + if *stage == "" { + return fmt.Errorf("verdict requires --stage") + } + selected := assurance.Stage(*stage) + if !selected.Valid() { + return fmt.Errorf("unsupported stage %q", *stage) + } + catalog, _, err := mustLoadCatalog(*catalogPath) + if err != nil { + return err + } + results, err := assurance.LoadResults(*resultsDir) + if err != nil { + return err + } + report := assurance.BuildReport(catalog, filterStage(results, selected), assurance.BuildOptions{ + Release: assurance.Release{Tag: *tag, Version: strings.TrimPrefix(*tag, "v")}, + Stages: []assurance.Stage{selected}, + StageRuns: map[assurance.Stage]string{selected: runURL()}, + GeneratedBy: "assurance", + }) + markdown := assurance.RenderMarkdown(report, assurance.MarkdownOptions{ + Heading: selected.Title(), + IncludeChecks: true, + }) + fmt.Print(markdown) + if *stepSummary { + if err := appendStepSummary(markdown); err != nil { + return err + } + } + if *jsonOut != "" { + data, encodeErr := report.Encode() + if encodeErr != nil { + return encodeErr + } + if err := os.WriteFile(*jsonOut, data, 0o644); err != nil { + return fmt.Errorf("write stage report: %w", err) + } + } + if report.Verdict.Blocking() { + return blockingError{ + message: fmt.Sprintf("%s did not pass: %s", selected.Title(), strings.Join(blockers(report.Verdict), ", ")), + code: 1, + } + } + return nil +} + +func blockers(verdict assurance.Verdict) []string { + var names []string + names = append(names, verdict.GatesFailed...) + for _, id := range verdict.MissingChecks { + names = append(names, id+" (no result)") + } + sort.Strings(names) + return names +} + +func filterStage(results []assurance.CheckResult, stage assurance.Stage) []assurance.CheckResult { + var selected []assurance.CheckResult + for _, result := range results { + if result.Stage == stage { + selected = append(selected, result) + } + } + return selected +} diff --git a/internal/assurance/cmd/main.go b/internal/assurance/cmd/main.go new file mode 100644 index 00000000..d833a1bc --- /dev/null +++ b/internal/assurance/cmd/main.go @@ -0,0 +1,265 @@ +// Command assurance drives Bomly's release assurance framework. Quality checks +// call it to emit their results, and the release pipeline calls it to judge a +// stage and to build the per-release report the public assurance page renders. +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/bomly-dev/bomly-cli/internal/assurance" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + var err error + switch os.Args[1] { + case "emit": + err = runEmit(os.Args[2:]) + case "gotest": + err = runGoTest(os.Args[2:]) + case "convert": + err = runConvert(os.Args[2:]) + case "verify-release": + err = runVerifyRelease(os.Args[2:]) + case "verdict": + err = runVerdict(os.Args[2:]) + case "report": + err = runReport(os.Args[2:]) + case "catalog-validate": + err = runCatalogValidate(os.Args[2:]) + case "-h", "--help", "help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "assurance: unknown command %q\n\n", os.Args[1]) + usage() + os.Exit(2) + } + if err != nil { + var blocking blockingError + if errors.As(err, &blocking) { + fmt.Fprintln(os.Stderr, "assurance:", blocking.Error()) + os.Exit(blocking.code) + } + fmt.Fprintln(os.Stderr, "assurance:", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `assurance drives Bomly's release assurance framework. + +Commands: + emit write a check result from flags + gotest turn a "go test -json" stream into a check result + convert turn a tool manifest into a check result + verify-release verify downloaded release assets and emit check results + verdict judge one stage from collected check results + report build the per-release assurance report and index + catalog-validate validate the assurance catalog and print its contents + +Run "assurance -h" for the flags of one command. +`) +} + +type blockingError struct { + message string + code int +} + +func (e blockingError) Error() string { return e.message } + +// ---------------------------------------------------------------- shared --- + +type resultContext struct { + catalog *assurance.Catalog + catalogPath string + root string +} + +func loadContext(catalogPath string) resultContext { + ctx := resultContext{catalogPath: catalogPath} + root, err := repositoryRoot() + if err != nil { + return ctx + } + ctx.root = root + path := catalogPath + if path == "" { + path = filepath.Join(root, filepath.FromSlash(assurance.DefaultCatalogPath)) + } else if !filepath.IsAbs(path) { + path = filepath.Join(root, filepath.FromSlash(path)) + } + catalog, err := assurance.LoadCatalog(path) + if err != nil { + return ctx + } + ctx.catalog = &catalog + ctx.catalogPath = path + return ctx +} + +func mustLoadCatalog(catalogPath string) (assurance.Catalog, string, error) { + ctx := loadContext(catalogPath) + if ctx.catalog == nil { + path := catalogPath + if path == "" { + path = assurance.DefaultCatalogPath + } + resolved := path + if ctx.root != "" && !filepath.IsAbs(path) { + resolved = filepath.Join(ctx.root, filepath.FromSlash(path)) + } + catalog, err := assurance.LoadCatalog(resolved) + if err != nil { + return assurance.Catalog{}, "", err + } + return catalog, resolved, nil + } + return *ctx.catalog, ctx.catalogPath, nil +} + +func repositoryRoot() (string, error) { + current, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("resolve working directory: %w", err) + } + for { + if info, statErr := os.Stat(filepath.Join(current, "go.mod")); statErr == nil && !info.IsDir() { + return current, nil + } + parent := filepath.Dir(current) + if parent == current { + return "", errors.New("find repository root: go.mod not found") + } + current = parent + } +} + +// baseResult fills the fields every check result shares from the environment. +func baseResult(id, instance string) assurance.CheckResult { + result := assurance.CheckResult{ + SchemaVersion: assurance.CheckSchema, + ID: id, + Instance: instance, + Ref: os.Getenv("GITHUB_REF"), + Commit: os.Getenv("GITHUB_SHA"), + Tag: os.Getenv("BOMLY_ASSURANCE_TAG"), + Job: os.Getenv("GITHUB_JOB"), + Runner: assurance.Runner{OS: goos(), Arch: goarch(), GoVersion: goVersion()}, + StartedAt: time.Now().UTC().Format(time.RFC3339), + } + result.Version = strings.TrimPrefix(result.Tag, "v") + result.RunURL = runURL() + return result +} + +func runURL() string { + server := os.Getenv("GITHUB_SERVER_URL") + repository := os.Getenv("GITHUB_REPOSITORY") + runID := os.Getenv("GITHUB_RUN_ID") + if server == "" || repository == "" || runID == "" { + return "" + } + url := fmt.Sprintf("%s/%s/actions/runs/%s", server, repository, runID) + if attempt := os.Getenv("GITHUB_RUN_ATTEMPT"); attempt != "" { + url += "/attempts/" + attempt + } + return url +} + +// applyCatalog fills the stage and level a check is declared with. +func applyCatalog(result *assurance.CheckResult, catalog *assurance.Catalog, stage, level string) error { + if catalog != nil { + if check, found := catalog.Check(result.ID); found { + if result.Stage == "" { + result.Stage = check.Stage + } + if result.Level == "" { + result.Level = check.Level + } + } + } + if stage != "" { + result.Stage = assurance.Stage(stage) + } + if level != "" { + result.Level = assurance.Level(level) + } + if result.Stage == "" { + return fmt.Errorf("check %q is not in the catalog, so --stage is required", result.ID) + } + return nil +} + +func writeResult(outDir string, result assurance.CheckResult, stepSummary bool) error { + if err := result.Validate(); err != nil { + return err + } + if err := os.MkdirAll(outDir, 0o755); err != nil { + return fmt.Errorf("create result directory: %w", err) + } + data, err := result.Encode() + if err != nil { + return err + } + path := filepath.Join(outDir, result.FileName()) + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + fmt.Printf("%s %s: %s\n", assurance.StatusIcon(result.Status), result.Key(), result.Summary) + if stepSummary { + if err := appendStepSummary(assurance.RenderResultMarkdown(result)); err != nil { + return err + } + } + return nil +} + +func appendStepSummary(markdown string) error { + path := os.Getenv("GITHUB_STEP_SUMMARY") + if path == "" { + return nil + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("open step summary: %w", err) + } + defer file.Close() + if _, err := file.WriteString(markdown); err != nil { + return fmt.Errorf("write step summary: %w", err) + } + return nil +} + +type stringList []string + +func (s *stringList) String() string { return strings.Join(*s, ",") } + +func (s *stringList) Set(value string) error { + *s = append(*s, value) + return nil +} + +func splitPair(value, what string) (string, string, error) { + key, rest, found := strings.Cut(value, "=") + if !found || strings.TrimSpace(key) == "" { + return "", "", fmt.Errorf("%s %q must be written as name=value", what, value) + } + return strings.TrimSpace(key), strings.TrimSpace(rest), nil +} + +func statusFromExit(code int) assurance.Status { + if code == 0 { + return assurance.StatusPass + } + return assurance.StatusFail +} diff --git a/internal/assurance/cmd/report.go b/internal/assurance/cmd/report.go new file mode 100644 index 00000000..4e01fa18 --- /dev/null +++ b/internal/assurance/cmd/report.go @@ -0,0 +1,364 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/bomly-dev/bomly-cli/internal/assurance" +) + +// defaultReportDir is the repository directory holding published reports. +const defaultReportDir = "docs/assurance" + +func runReport(args []string) error { + flags := flag.NewFlagSet("report", flag.ExitOnError) + resultsDir := flags.String("results", "assurance-results", "directory of collected check results") + catalogPath := flags.String("catalog", "", "assurance catalog path") + tag := flags.String("tag", "", "release tag, such as v0.24.0") + commit := flags.String("commit", "", "release commit") + releaseURL := flags.String("url", "", "release page URL") + publishedAt := flags.String("published-at", "", "RFC 3339 publication time") + previous := flags.String("previous", "", "previous release report for trends (auto by default)") + outDir := flags.String("out", "", "directory to write reports and the index into") + summaryOut := flags.String("summary-out", "", "write the markdown summary to this path") + stepSummary := flags.Bool("step-summary", false, "append the markdown summary to the workflow step summary") + stages := flags.String("stages", "", "comma-separated stages to include (every stage by default)") + prerequisitesRun := flags.String("prerequisites-run", "", "workflow run URL for the prerequisites stage") + preReleaseRun := flags.String("pre-release-run", "", "workflow run URL for the pre-release stage") + assessmentRun := flags.String("assessment-run", "", "workflow run URL for the post-release stage") + failOnBlocking := flags.Bool("fail-on-blocking", false, "exit non-zero when a gate check did not pass") + allowUnknown := flags.Bool("allow-unknown", false, "report results the catalog does not declare instead of failing") + if err := flags.Parse(args); err != nil { + return err + } + if *tag == "" { + return fmt.Errorf("report requires --tag") + } + catalog, catalogFile, err := mustLoadCatalog(*catalogPath) + if err != nil { + return err + } + results, err := assurance.LoadResults(*resultsDir) + if err != nil { + return err + } + targetDir := *outDir + if targetDir == "" { + root := filepath.Dir(filepath.Dir(catalogFile)) + targetDir = filepath.Join(root, filepath.FromSlash(defaultReportDir)) + if filepath.Base(filepath.Dir(catalogFile)) == "assurance" { + targetDir = filepath.Dir(catalogFile) + } + } + indexPath := filepath.Join(targetDir, "index.json") + + index := assurance.Index{SchemaVersion: assurance.IndexSchema} + if data, readErr := os.ReadFile(indexPath); readErr == nil { + loaded, parseErr := assurance.ParseIndex(data) + if parseErr != nil { + return parseErr + } + index = loaded + } + + previousReport, previousTag, err := resolvePrevious(*previous, *tag, targetDir, index) + if err != nil { + return err + } + + options := assurance.BuildOptions{ + Release: assurance.Release{ + Tag: *tag, Version: strings.TrimPrefix(*tag, "v"), + Commit: *commit, URL: *releaseURL, PublishedAt: *publishedAt, + }, + StageRuns: map[assurance.Stage]string{ + assurance.StagePrerequisites: *prerequisitesRun, + assurance.StagePreRelease: *preReleaseRun, + assurance.StagePostRelease: *assessmentRun, + }, + Previous: previousReport, + IncludeEvidence: true, + GeneratedBy: "assurance", + } + if *stages != "" { + for _, name := range strings.Split(*stages, ",") { + stage := assurance.Stage(strings.TrimSpace(name)) + if !stage.Valid() { + return fmt.Errorf("unsupported stage %q", name) + } + options.Stages = append(options.Stages, stage) + } + } + report := assurance.BuildReport(catalog, results, options) + + reportPath := filepath.Join(targetDir, "reports", *tag+".json") + if err := os.MkdirAll(filepath.Dir(reportPath), 0o755); err != nil { + return fmt.Errorf("create report directory: %w", err) + } + data, err := report.Encode() + if err != nil { + return err + } + if err := os.WriteFile(reportPath, data, 0o644); err != nil { + return fmt.Errorf("write assurance report: %w", err) + } + + index.SchemaVersion = assurance.IndexSchema + index.GeneratedAt = report.GeneratedAt + index.Releases = upsertIndexEntry(index.Releases, assurance.IndexEntry{ + Tag: report.Release.Tag, Version: report.Release.Version, + PublishedAt: report.Release.PublishedAt, GeneratedAt: report.GeneratedAt, + Verdict: report.Verdict.Overall, Gates: len(report.Verdict.GatesFailed), + Path: "reports/" + report.Release.Tag + ".json", + }) + if len(index.Releases) > 0 { + index.Latest = index.Releases[0].Tag + } + indexData, err := index.Encode() + if err != nil { + return err + } + if err := os.WriteFile(indexPath, indexData, 0o644); err != nil { + return fmt.Errorf("write assurance index: %w", err) + } + + markdown := assurance.RenderMarkdown(report, assurance.MarkdownOptions{ + IncludeChecks: true, IncludeTrends: true, + }) + fmt.Print(markdown) + if *summaryOut != "" { + if err := os.WriteFile(*summaryOut, []byte(markdown), 0o644); err != nil { + return fmt.Errorf("write markdown summary: %w", err) + } + } + if *stepSummary { + if err := appendStepSummary(markdown); err != nil { + return err + } + } + fmt.Fprintf(os.Stderr, "wrote %s and %s\n", reportPath, indexPath) + if previousTag != "" { + fmt.Fprintf(os.Stderr, "compared against %s\n", previousTag) + } + if len(report.Unknown) > 0 && !*allowUnknown { + return blockingError{ + message: fmt.Sprintf("%d reported check(s) are not declared in the catalog", len(report.Unknown)), + code: 3, + } + } + if *failOnBlocking && report.Verdict.Blocking() { + return blockingError{ + message: "the release did not pass every gate check: " + strings.Join(blockers(report.Verdict), ", "), + code: 1, + } + } + return nil +} + +func resolvePrevious(flagValue, tag, targetDir string, index assurance.Index) (*assurance.Report, string, error) { + switch flagValue { + case "none": + return nil, "", nil + case "": + previousTag := previousRelease(tag, index) + if previousTag == "" { + return nil, "", nil + } + path := filepath.Join(targetDir, "reports", previousTag+".json") + report, err := assurance.LoadReport(path) + if err != nil { + // A missing previous report is not an error: the first release has + // none, and an older one may predate the framework. + return nil, "", nil + } + return &report, previousTag, nil + default: + report, err := assurance.LoadReport(flagValue) + if err != nil { + return nil, "", err + } + return &report, report.Release.Tag, nil + } +} + +// previousRelease returns the highest released tag below the current one. +func previousRelease(tag string, index assurance.Index) string { + current, err := semver.NewVersion(strings.TrimPrefix(tag, "v")) + if err != nil { + return "" + } + best := "" + var bestVersion *semver.Version + for _, entry := range index.Releases { + candidate, parseErr := semver.NewVersion(strings.TrimPrefix(entry.Tag, "v")) + if parseErr != nil || !candidate.LessThan(current) { + continue + } + if bestVersion == nil || candidate.GreaterThan(bestVersion) { + bestVersion = candidate + best = entry.Tag + } + } + return best +} + +func upsertIndexEntry(entries []assurance.IndexEntry, entry assurance.IndexEntry) []assurance.IndexEntry { + replaced := false + for index, existing := range entries { + if existing.Tag == entry.Tag { + entries[index] = entry + replaced = true + break + } + } + if !replaced { + entries = append(entries, entry) + } + sort.SliceStable(entries, func(i, j int) bool { + left, leftErr := semver.NewVersion(strings.TrimPrefix(entries[i].Tag, "v")) + right, rightErr := semver.NewVersion(strings.TrimPrefix(entries[j].Tag, "v")) + if leftErr != nil || rightErr != nil { + return entries[i].Tag > entries[j].Tag + } + return left.GreaterThan(right) + }) + return entries +} + +// -------------------------------------------------------- catalog-validate --- + +func runCatalogValidate(args []string) error { + flags := flag.NewFlagSet("catalog-validate", flag.ExitOnError) + catalogPath := flags.String("catalog", "", "assurance catalog path") + checkID := flags.String("check", "", "print one check") + evidenceID := flags.String("evidence", "", "print one evidence claim") + skipArtifacts := flags.Bool("skip-artifacts", false, "skip repository artifact hash verification") + if err := flags.Parse(args); err != nil { + return err + } + catalog, catalogFile, err := mustLoadCatalog(*catalogPath) + if err != nil { + return err + } + if !*skipArtifacts { + root, rootErr := repositoryRoot() + if rootErr != nil { + return rootErr + } + if err := catalog.VerifyArtifacts(root); err != nil { + return err + } + } + fmt.Printf("Validated %s: %d areas, %d checks, %d evidence claims.\n", + relativeToWorkingDir(catalogFile), len(catalog.Areas), len(catalog.Checks), len(catalog.Evidence)) + + if *checkID != "" || *evidenceID != "" { + if *checkID != "" { + check, found := catalog.Check(*checkID) + if !found { + return fmt.Errorf("unknown check %q", *checkID) + } + printCheck(catalog, check) + } + if *evidenceID != "" { + for _, evidence := range catalog.Evidence { + if evidence.ID == *evidenceID { + printEvidence(evidence) + return nil + } + } + return fmt.Errorf("unknown evidence claim %q", *evidenceID) + } + return nil + } + for _, stage := range assurance.Stages() { + checks := catalog.ChecksForStage(stage) + if len(checks) == 0 { + continue + } + fmt.Printf("\n%s (%d checks)\n", stage.Title(), len(checks)) + for _, check := range checks { + instances := "" + if len(check.ExpectedInstances) > 0 { + instances = fmt.Sprintf(", %d instances", len(check.ExpectedInstances)) + } + fmt.Printf(" %-24s %-9s %s%s\n", check.ID, check.Level, check.Title, instances) + } + } + fmt.Printf("\nEvidence (%d claims)\n", len(catalog.Evidence)) + for _, evidence := range catalog.Evidence { + fmt.Printf(" %-26s %-16s backed by %s\n", evidence.ID, evidence.EvidenceLevel, evidence.CheckID) + } + return nil +} + +func printCheck(catalog assurance.Catalog, check assurance.Check) { + fmt.Printf("\n%s — %s\n", check.ID, check.Title) + fmt.Printf(" Area: %s (%s); stage: %s; level: %s\n", + check.Area, catalog.AreaTitle(check.Area), check.Stage, check.Level) + fmt.Printf(" Source: %s job %s\n", check.Source.Workflow, check.Source.Job) + for _, instance := range check.ExpectedInstances { + fmt.Printf(" Instance: %s\n", instance.Name) + } + for _, command := range check.Reproduce { + fmt.Printf(" Reproduce: %s\n", shellCommand(command)) + } + for _, claim := range check.Proves { + fmt.Printf(" Proves: %s\n", claim) + } + for _, limitation := range check.Limitations { + fmt.Printf(" Limitation: %s\n", limitation) + } +} + +func printEvidence(evidence assurance.Evidence) { + fmt.Printf("\n%s — %s\n", evidence.ID, evidence.Title) + fmt.Printf(" Area: %s; evidence: %s; backed by check %s\n", + evidence.Area, evidence.EvidenceLevel, evidence.CheckID) + for _, input := range evidence.Inputs { + fmt.Printf(" Input: %s %s\n", input.Kind, input.Location) + } + for _, command := range evidence.Reproduce { + fmt.Printf(" Reproduce: %s\n", shellCommand(command)) + } + for _, claim := range evidence.Proves { + fmt.Printf(" Proves: %s\n", claim) + } + for _, limitation := range evidence.Limitations { + fmt.Printf(" Limitation: %s\n", limitation) + } +} + +func relativeToWorkingDir(path string) string { + working, err := os.Getwd() + if err != nil { + return path + } + relative, err := filepath.Rel(working, path) + if err != nil { + return path + } + return relative +} + +func shellCommand(command []string) string { + quoted := make([]string, len(command)) + for index, argument := range command { + if argument != "" && strings.IndexFunc(argument, func(r rune) bool { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + return false + } + return !strings.ContainsRune("@%_+=:,./-", r) + }) == -1 { + quoted[index] = argument + continue + } + quoted[index] = "'" + strings.ReplaceAll(argument, "'", "'\"'\"'") + "'" + } + return strings.Join(quoted, " ") +} diff --git a/internal/assurance/contract.go b/internal/assurance/contract.go new file mode 100644 index 00000000..c96e813b --- /dev/null +++ b/internal/assurance/contract.go @@ -0,0 +1,348 @@ +// Package assurance implements Bomly's release assurance framework: the shared +// check-result contract every quality check emits, the declarative catalog that +// declares which checks must exist, and the report the public assurance page +// renders for each release. +package assurance + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// CheckSchema is the schema identifier every check result carries. +const CheckSchema = "bomly.assurance-check/v1" + +// MaxResultBytes bounds a single check-result document. +const MaxResultBytes = 8 << 20 + +// Stage names the release phase a check belongs to. +type Stage string + +// The three release assurance stages. +const ( + // StagePrerequisites runs on the source tree before a release tag exists. + StagePrerequisites Stage = "prerequisites" + // StagePreRelease runs inside the release pipeline against the draft release. + StagePreRelease Stage = "pre-release" + // StagePostRelease runs after publication against the shipped artifacts. + StagePostRelease Stage = "post-release" +) + +// Stages lists every release assurance stage in execution order. +func Stages() []Stage { + return []Stage{StagePrerequisites, StagePreRelease, StagePostRelease} +} + +// Valid reports whether the stage is one of the three known stages. +func (s Stage) Valid() bool { + switch s { + case StagePrerequisites, StagePreRelease, StagePostRelease: + return true + default: + return false + } +} + +// Title returns the human-readable stage name used in reports. +func (s Stage) Title() string { + switch s { + case StagePrerequisites: + return "Release prerequisites" + case StagePreRelease: + return "Final pre-release checks" + case StagePostRelease: + return "Post-release assessment" + default: + return string(s) + } +} + +// Level distinguishes checks that block a release from advisory observations. +type Level string + +// Check enforcement levels. +const ( + // LevelGate blocks the stage when it does not pass. + LevelGate Level = "gate" + // LevelAdvisory is reported but never blocks a release. + LevelAdvisory Level = "advisory" +) + +// Valid reports whether the level is known. +func (l Level) Valid() bool { return l == LevelGate || l == LevelAdvisory } + +// Status is the outcome of a check or check instance. +type Status string + +// Check outcomes, ordered from best to worst by Severity. +const ( + // StatusPass means the check completed and every expectation held. + StatusPass Status = "pass" + // StatusSkip means the check did not run for a declared reason. + StatusSkip Status = "skip" + // StatusMissing means no result was reported for a declared check. + StatusMissing Status = "missing" + // StatusDegraded means the check completed with reduced confidence. + StatusDegraded Status = "degraded" + // StatusFail means the check did not hold. + StatusFail Status = "fail" +) + +// Valid reports whether the status is one a check result may carry. +func (s Status) Valid() bool { + switch s { + case StatusPass, StatusSkip, StatusMissing, StatusDegraded, StatusFail: + return true + default: + return false + } +} + +// Severity ranks statuses so the worst instance decides a merged check. +func (s Status) Severity() int { + switch s { + case StatusPass: + return 0 + case StatusSkip: + return 1 + case StatusMissing: + return 2 + case StatusDegraded: + return 3 + case StatusFail: + return 4 + default: + return 5 + } +} + +// Worse returns whichever status is more severe. +func Worse(a, b Status) Status { + if b.Severity() > a.Severity() { + return b + } + return a +} + +// Runner records the machine a check instance executed on. +type Runner struct { + OS string `json:"os,omitempty"` + Arch string `json:"arch,omitempty"` + GoVersion string `json:"go_version,omitempty"` +} + +// Detail is one named sub-result inside a check instance, such as a single +// smoke test, fuzz target, or cross-build target. +type Detail struct { + Name string `json:"name"` + Status Status `json:"status"` + Note string `json:"note,omitempty"` + DurationMS float64 `json:"duration_ms,omitempty"` +} + +// Artifact records a file a check produced or verified. +type Artifact struct { + Name string `json:"name"` + SHA256 string `json:"sha256,omitempty"` + Bytes int64 `json:"bytes,omitempty"` +} + +// Link is a labelled URL shown alongside a check. +type Link struct { + Label string `json:"label"` + URL string `json:"url"` +} + +// CheckResult is what a single check instance writes when it finishes. One +// file per instance; instances sharing an ID merge into one reported check. +type CheckResult struct { + SchemaVersion string `json:"schema_version"` + ID string `json:"id"` + Instance string `json:"instance,omitempty"` + Stage Stage `json:"stage"` + Level Level `json:"level,omitempty"` + Status Status `json:"status"` + StartedAt string `json:"started_at,omitempty"` + FinishedAt string `json:"finished_at,omitempty"` + DurationMS float64 `json:"duration_ms,omitempty"` + Ref string `json:"ref,omitempty"` + Tag string `json:"tag,omitempty"` + Commit string `json:"commit,omitempty"` + Version string `json:"version,omitempty"` + RunURL string `json:"run_url,omitempty"` + Job string `json:"job,omitempty"` + Runner Runner `json:"runner,omitempty"` + Summary string `json:"summary"` + Metrics map[string]float64 `json:"metrics,omitempty"` + Details []Detail `json:"details,omitempty"` + Artifacts []Artifact `json:"artifacts,omitempty"` + Links []Link `json:"links,omitempty"` +} + +var idPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) + +// instancePattern allows the platform and slice names checks report, which +// include dots (ubuntu-24.04) and underscores (linux_amd64). +var instancePattern = regexp.MustCompile(`^[a-zA-Z0-9]+(?:[._-][a-zA-Z0-9]+)*$`) + +// ParseCheckResult decodes and validates one check-result document. +func ParseCheckResult(data []byte) (CheckResult, error) { + if len(data) > MaxResultBytes { + return CheckResult{}, fmt.Errorf("check result is %d bytes, limit is %d", len(data), MaxResultBytes) + } + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + var result CheckResult + if err := decoder.Decode(&result); err != nil { + return CheckResult{}, fmt.Errorf("decode check result: %w", err) + } + if err := ensureEOF(decoder, "check result"); err != nil { + return CheckResult{}, err + } + if err := result.Validate(); err != nil { + return CheckResult{}, err + } + return result, nil +} + +// Validate reports whether the check result satisfies the contract. +func (r CheckResult) Validate() error { + if r.SchemaVersion != CheckSchema { + return fmt.Errorf("unsupported check-result schema %q", r.SchemaVersion) + } + if !idPattern.MatchString(r.ID) { + return fmt.Errorf("invalid check id %q", r.ID) + } + if r.Instance != "" && !instancePattern.MatchString(r.Instance) { + return fmt.Errorf("check %q has invalid instance %q", r.ID, r.Instance) + } + if !r.Stage.Valid() { + return fmt.Errorf("check %q has unsupported stage %q", r.ID, r.Stage) + } + if r.Level != "" && !r.Level.Valid() { + return fmt.Errorf("check %q has unsupported level %q", r.ID, r.Level) + } + if !r.Status.Valid() || r.Status == StatusMissing { + return fmt.Errorf("check %q has unsupported status %q", r.ID, r.Status) + } + if strings.TrimSpace(r.Summary) == "" { + return fmt.Errorf("check %q requires a summary", r.ID) + } + for index, detail := range r.Details { + if strings.TrimSpace(detail.Name) == "" { + return fmt.Errorf("check %q detail %d requires a name", r.ID, index+1) + } + if !detail.Status.Valid() { + return fmt.Errorf("check %q detail %q has unsupported status %q", r.ID, detail.Name, detail.Status) + } + } + for index, artifact := range r.Artifacts { + if strings.TrimSpace(artifact.Name) == "" { + return fmt.Errorf("check %q artifact %d requires a name", r.ID, index+1) + } + } + for index, link := range r.Links { + if strings.TrimSpace(link.Label) == "" || strings.TrimSpace(link.URL) == "" { + return fmt.Errorf("check %q link %d requires a label and url", r.ID, index+1) + } + } + return nil +} + +// Key identifies a check instance uniquely within one run. +func (r CheckResult) Key() string { + if r.Instance == "" { + return r.ID + } + return r.ID + "." + r.Instance +} + +// FileName is the on-disk name a check result is written under. +func (r CheckResult) FileName() string { return r.Key() + ".json" } + +// Encode renders the check result as indented JSON with a trailing newline. +func (r CheckResult) Encode() ([]byte, error) { + data, err := json.MarshalIndent(r, "", " ") + if err != nil { + return nil, fmt.Errorf("encode check result: %w", err) + } + return append(data, '\n'), nil +} + +// LoadResults reads every check-result document under dir, recursively, so a +// directory of merged CI artifacts can be consumed directly. Files that are not +// check results are ignored; malformed check results are an error. +func LoadResults(dir string) ([]CheckResult, error) { + var results []CheckResult + walkErr := filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + return nil + } + data, readErr := readBounded(path, MaxResultBytes) + if readErr != nil { + return readErr + } + if !looksLikeCheckResult(data) { + return nil + } + result, parseErr := ParseCheckResult(data) + if parseErr != nil { + return fmt.Errorf("%s: %w", path, parseErr) + } + results = append(results, result) + return nil + }) + if walkErr != nil { + return nil, fmt.Errorf("load check results from %s: %w", dir, walkErr) + } + sort.Slice(results, func(i, j int) bool { return results[i].Key() < results[j].Key() }) + return results, nil +} + +func looksLikeCheckResult(data []byte) bool { + var probe struct { + SchemaVersion string `json:"schema_version"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return false + } + return probe.SchemaVersion == CheckSchema +} + +func readBounded(path string, limit int64) ([]byte, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open %s: %w", path, err) + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, limit+1)) + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("read %s: file exceeds %d bytes", path, limit) + } + return data, nil +} + +func ensureEOF(decoder *json.Decoder, what string) error { + var extra any + err := decoder.Decode(&extra) + if errors.Is(err, io.EOF) { + return nil + } + if err == nil { + return fmt.Errorf("decode %s: multiple JSON values", what) + } + return fmt.Errorf("decode %s trailing data: %w", what, err) +} diff --git a/internal/assurance/convert.go b/internal/assurance/convert.go new file mode 100644 index 00000000..b26696ab --- /dev/null +++ b/internal/assurance/convert.go @@ -0,0 +1,220 @@ +package assurance + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" +) + +// BenchmarkRunSchema is the schema of the performance sample manifest produced +// by internal/assurance/perfrun. +const BenchmarkRunSchema = "bomly.benchmark-run/v1" + +// SBOMAssuranceSchema is the schema of the SBOM interoperability run manifest +// produced by internal/assurance/sbominterop. +const SBOMAssuranceSchema = "bomly.sbom-assurance-run/v1" + +// maxManifestBytes bounds a converted manifest document. +const maxManifestBytes = 32 << 20 + +type benchmarkManifest struct { + SchemaVersion string `json:"schema_version"` + Case struct { + Name string `json:"name"` + SamplesPerMode int `json:"samples_per_mode"` + NetworkState string `json:"network_state"` + } `json:"case"` + Summaries []struct { + Mode string `json:"mode"` + Samples int `json:"samples"` + MedianMS float64 `json:"median_ms"` + MeanMS float64 `json:"mean_ms"` + ConfidenceInterval95 [2]float64 `json:"confidence_interval_95_ms"` + PeakMemoryBytes uint64 `json:"peak_memory_bytes"` + } `json:"summaries"` + Gates struct { + Passed bool `json:"passed"` + AllExitCodesZero bool `json:"all_exit_codes_zero"` + NormalizedOutputStable bool `json:"normalized_output_stable"` + OutputCapPassed bool `json:"output_cap_passed"` + FailureReason string `json:"failure_reason"` + } `json:"gates"` +} + +// ConvertBenchmarkRun turns a performance sample manifest into a check result. +func ConvertBenchmarkRun(data []byte, base CheckResult) (CheckResult, error) { + var manifest benchmarkManifest + if err := decodeManifest(data, &manifest, "performance manifest"); err != nil { + return CheckResult{}, err + } + if manifest.SchemaVersion != BenchmarkRunSchema { + return CheckResult{}, fmt.Errorf("unsupported performance manifest schema %q", manifest.SchemaVersion) + } + result := base + result.SchemaVersion = CheckSchema + result.Status = StatusPass + if !manifest.Gates.Passed { + result.Status = StatusFail + } + result.Metrics = map[string]float64{"samples_per_mode": float64(manifest.Case.SamplesPerMode)} + var peak float64 + for _, summary := range manifest.Summaries { + mode := strings.ToLower(summary.Mode) + result.Metrics[mode+"_median_ms"] = round(summary.MedianMS) + result.Metrics[mode+"_mean_ms"] = round(summary.MeanMS) + result.Metrics[mode+"_ci95_upper_ms"] = round(summary.ConfidenceInterval95[1]) + if float64(summary.PeakMemoryBytes) > peak { + peak = float64(summary.PeakMemoryBytes) + } + result.Details = append(result.Details, Detail{ + Name: mode + " cache", + Status: StatusPass, + Note: fmt.Sprintf("%d samples, median %.0f ms, mean %.0f ms", + summary.Samples, summary.MedianMS, summary.MeanMS), + DurationMS: round(summary.MedianMS), + }) + } + if peak > 0 { + result.Metrics["peak_memory_bytes"] = peak + } + switch { + case manifest.Gates.FailureReason != "": + result.Summary = fmt.Sprintf("Performance sampling for %s failed: %s.", + manifest.Case.Name, manifest.Gates.FailureReason) + case len(manifest.Summaries) == 0: + result.Status = StatusFail + result.Summary = "The performance run recorded no samples." + default: + result.Summary = fmt.Sprintf("%s completed %d samples per cache mode with identical normalized output (%s).", + manifest.Case.Name, manifest.Case.SamplesPerMode, describeModes(result.Metrics)) + } + return result, nil +} + +func describeModes(metrics map[string]float64) string { + var parts []string + for _, mode := range []string{"cold", "warm"} { + if value, ok := metrics[mode+"_median_ms"]; ok { + parts = append(parts, fmt.Sprintf("%s median %.0f ms", mode, value)) + } + } + if len(parts) == 0 { + return "no timings recorded" + } + return strings.Join(parts, ", ") +} + +type sbomAssuranceManifest struct { + SchemaVersion string `json:"schema_version"` + Validators []struct { + Name string `json:"name"` + Version string `json:"version"` + SHA256 string `json:"sha256"` + } `json:"validators"` + Artifacts []struct { + Format string `json:"format"` + Path string `json:"path"` + SHA256 string `json:"sha256"` + Bytes int64 `json:"bytes"` + } `json:"artifacts"` + Commands []struct { + Executable string `json:"executable"` + Args []string `json:"args"` + ExitCode int `json:"exit_code"` + DurationMS int64 `json:"duration_ms"` + } `json:"commands"` + Failure string `json:"failure,omitempty"` +} + +// ConvertSBOMAssurance turns an SBOM interoperability run manifest into a +// check result. +func ConvertSBOMAssurance(data []byte, base CheckResult) (CheckResult, error) { + var manifest sbomAssuranceManifest + if err := decodeManifest(data, &manifest, "SBOM assurance manifest"); err != nil { + return CheckResult{}, err + } + if manifest.SchemaVersion != SBOMAssuranceSchema { + return CheckResult{}, fmt.Errorf("unsupported SBOM assurance manifest schema %q", manifest.SchemaVersion) + } + result := base + result.SchemaVersion = CheckSchema + result.Status = StatusPass + failures := 0 + for _, command := range manifest.Commands { + status := StatusPass + if command.ExitCode != 0 { + status = StatusFail + failures++ + } + result.Details = append(result.Details, Detail{ + Name: strings.TrimSpace(filepath.Base(command.Executable) + " " + firstArgument(command.Args)), + Status: status, + Note: fmt.Sprintf("exit code %d", command.ExitCode), + DurationMS: float64(command.DurationMS), + }) + } + for _, validator := range manifest.Validators { + result.Details = append(result.Details, Detail{ + Name: validator.Name + " " + validator.Version, + Status: StatusPass, + Note: "checksum " + shortHash(validator.SHA256), + }) + } + for _, artifact := range manifest.Artifacts { + result.Artifacts = append(result.Artifacts, Artifact{ + Name: artifact.Format, SHA256: artifact.SHA256, Bytes: artifact.Bytes, + }) + } + result.Metrics = map[string]float64{ + "validators": float64(len(manifest.Validators)), + "documents": float64(len(manifest.Artifacts)), + } + switch { + case manifest.Failure != "": + result.Status = StatusFail + result.Summary = "SBOM interoperability run failed: " + manifest.Failure + "." + case failures > 0: + result.Status = StatusFail + result.Summary = fmt.Sprintf("%d of %d validator commands failed.", failures, len(manifest.Commands)) + case len(manifest.Artifacts) == 0: + result.Status = StatusFail + result.Summary = "The run produced no SBOM documents to validate." + default: + names := make([]string, 0, len(manifest.Validators)) + for _, validator := range manifest.Validators { + names = append(names, validator.Name+" "+validator.Version) + } + result.Summary = fmt.Sprintf("%d generated SBOM documents passed %s.", + len(manifest.Artifacts), strings.Join(names, " and ")) + } + return result, nil +} + +func firstArgument(args []string) string { + if len(args) == 0 { + return "" + } + return args[0] +} + +func shortHash(hash string) string { + if len(hash) <= 12 { + return hash + } + return hash[:12] +} + +func round(value float64) float64 { + return float64(int64(value*100+0.5)) / 100 +} + +func decodeManifest(data []byte, target any, what string) error { + if len(data) > maxManifestBytes { + return fmt.Errorf("%s is %d bytes, limit is %d", what, len(data), maxManifestBytes) + } + if err := json.Unmarshal(data, target); err != nil { + return fmt.Errorf("decode %s: %w", what, err) + } + return nil +} diff --git a/internal/assurance/convert_test.go b/internal/assurance/convert_test.go new file mode 100644 index 00000000..1900b8e2 --- /dev/null +++ b/internal/assurance/convert_test.go @@ -0,0 +1,191 @@ +package assurance + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func baseFor(id string, stage Stage, level Level) CheckResult { + return CheckResult{SchemaVersion: CheckSchema, ID: id, Stage: stage, Level: level} +} + +func TestConvertBenchmarkRun(t *testing.T) { + manifest := map[string]any{ + "schema_version": BenchmarkRunSchema, + "case": map[string]any{"name": "canonical-sbom-scan", "samples_per_mode": 5}, + "summaries": []map[string]any{ + {"mode": "cold", "samples": 5, "median_ms": 412.4, "mean_ms": 430.0, + "confidence_interval_95_ms": []float64{400, 460}, "peak_memory_bytes": 91234304}, + {"mode": "warm", "samples": 5, "median_ms": 288.1, "mean_ms": 291.0, + "confidence_interval_95_ms": []float64{280, 300}, "peak_memory_bytes": 80234304}, + }, + "gates": map[string]any{"passed": true}, + } + data, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal: %v", err) + } + result, err := ConvertBenchmarkRun(data, baseFor("perf-samples", StagePostRelease, LevelAdvisory)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if err := result.Validate(); err != nil { + t.Fatalf("converted result invalid: %v", err) + } + if result.Status != StatusPass { + t.Fatalf("status = %s, want pass", result.Status) + } + if result.Metrics["cold_median_ms"] != 412.4 || result.Metrics["warm_median_ms"] != 288.1 { + t.Fatalf("metrics = %v", result.Metrics) + } + if result.Metrics["peak_memory_bytes"] != 91234304 { + t.Fatalf("peak memory = %v", result.Metrics["peak_memory_bytes"]) + } + if len(result.Details) != 2 { + t.Fatalf("details = %+v", result.Details) + } +} + +func TestConvertBenchmarkRunFailedGate(t *testing.T) { + data := []byte(`{"schema_version":"bomly.benchmark-run/v1","case":{"name":"c","samples_per_mode":5}, + "summaries":[],"gates":{"passed":false,"failure_reason":"normalized output changed between samples"}}`) + result, err := ConvertBenchmarkRun(data, baseFor("perf-samples", StagePostRelease, LevelAdvisory)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if result.Status != StatusFail { + t.Fatalf("status = %s, want fail", result.Status) + } + if !strings.Contains(result.Summary, "normalized output changed") { + t.Fatalf("summary = %q", result.Summary) + } +} + +func TestConvertSBOMAssurance(t *testing.T) { + data := []byte(`{"schema_version":"bomly.sbom-assurance-run/v1", + "validators":[{"name":"spdx-tools-java","version":"2.0.7","sha256":"2dc63c3399c5178058b1be8a3de6f13b9f24981cd86c4292ef98f4a7e90de36d"}, + {"name":"cyclonedx-cli","version":"0.32.0","sha256":"454879e6a4a405c8a13bff49b8982adcb0596f3019b26b0811c66e4d7f0783e1"}], + "artifacts":[{"format":"spdx-2.3-json","path":"a.json","sha256":"aa","bytes":120}, + {"format":"cyclonedx-1.7-json","path":"b.json","sha256":"bb","bytes":140}], + "commands":[{"executable":"/usr/bin/java","args":["-jar","tools.jar","Verify"],"exit_code":0,"duration_ms":900}]}`) + result, err := ConvertSBOMAssurance(data, baseFor("sbom-interoperability", StagePostRelease, LevelGate)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if err := result.Validate(); err != nil { + t.Fatalf("converted result invalid: %v", err) + } + if result.Status != StatusPass { + t.Fatalf("status = %s, want pass", result.Status) + } + if len(result.Artifacts) != 2 { + t.Fatalf("artifacts = %+v", result.Artifacts) + } + if !strings.Contains(result.Summary, "spdx-tools-java 2.0.7") { + t.Fatalf("summary = %q", result.Summary) + } +} + +func TestConvertSBOMAssuranceRecordsFailure(t *testing.T) { + data := []byte(`{"schema_version":"bomly.sbom-assurance-run/v1","validators":[],"artifacts":[], + "commands":[{"executable":"cyclonedx-cli","args":["validate"],"exit_code":1,"duration_ms":10}], + "failure":"validator cyclonedx-cli failed with exit 1"}`) + result, err := ConvertSBOMAssurance(data, baseFor("sbom-interoperability", StagePostRelease, LevelGate)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if result.Status != StatusFail || !strings.Contains(result.Summary, "validator cyclonedx-cli failed") { + t.Fatalf("result = %+v", result) + } +} + +func TestConvertRejectsWrongSchema(t *testing.T) { + if _, err := ConvertBenchmarkRun([]byte(`{"schema_version":"other/v1"}`), CheckResult{}); err == nil { + t.Fatal("expected the wrong schema to be rejected") + } + if _, err := ConvertSBOMAssurance([]byte(`{"schema_version":"other/v1"}`), CheckResult{}); err == nil { + t.Fatal("expected the wrong schema to be rejected") + } +} + +func TestExpectedAssetsCoverEveryPlatform(t *testing.T) { + assets := ExpectedAssets("0.23.0") + if len(assets) != 23 { + t.Fatalf("expected 23 release assets, got %d: %v", len(assets), assets) + } + for _, required := range []string{ + "SHA256SUMS", "SHA256SUMS.sigstore.json", "multiple.intoto.jsonl", + "bomly_0.23.0_linux_amd64.tar.gz", "bomly-lite_0.23.0_windows_arm64.zip", + "bomly_0.23.0_linux_arm64.pkg.tar.zst", + } { + if !contains(assets, required) { + t.Fatalf("expected asset list to contain %s", required) + } + } +} + +func contains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func TestParseSHA256SUMS(t *testing.T) { + entries, err := ParseSHA256SUMS([]byte( + "aa2c19f8a17ad4c65c6b6a41c9de9b8bbd9d3d3b1f1d6b0b5b0a67f6d4f1a2b3 bomly_1.0.0_linux_amd64.tar.gz\n" + + "bb2c19f8a17ad4c65c6b6a41c9de9b8bbd9d3d3b1f1d6b0b5b0a67f6d4f1a2b3 *bomly_1.0.0_windows_amd64.zip\n")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(entries) != 2 { + t.Fatalf("entries = %v", entries) + } + for _, broken := range []string{ + "not-a-hash file.tar.gz\n", + "aa2c19f8a17ad4c65c6b6a41c9de9b8bbd9d3d3b1f1d6b0b5b0a67f6d4f1a2b3 ../escape.tar.gz\n", + "onlyonefield\n", + "", + } { + if _, err := ParseSHA256SUMS([]byte(broken)); err == nil { + t.Fatalf("expected %q to be rejected", broken) + } + } +} + +func TestVerifyChecksumsDetectsProblems(t *testing.T) { + dir := t.TempDir() + payload := []byte("binary\n") + if err := os.WriteFile(filepath.Join(dir, "bomly_1.0.0_linux_amd64.tar.gz"), payload, 0o644); err != nil { + t.Fatalf("write asset: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "stray.txt"), payload, 0o644); err != nil { + t.Fatalf("write stray: %v", err) + } + entries := ChecksumEntries{ + "bomly_1.0.0_linux_amd64.tar.gz": "1111111111111111111111111111111111111111111111111111111111111111", + "bomly_1.0.0_darwin_arm64.tar.gz": "2222222222222222222222222222222222222222222222222222222222222222", + } + outcome, err := VerifyChecksums(dir, entries) + if err != nil { + t.Fatalf("verify: %v", err) + } + if len(outcome.Mismatched) != 1 || len(outcome.Unlisted) != 1 || len(outcome.NotDownloaded) != 1 { + t.Fatalf("outcome = %+v", outcome) + } +} + +func TestArchiveNameUsesZipOnWindows(t *testing.T) { + if got := ArchiveName("bomly", "1.2.3", "windows", "arm64"); got != "bomly_1.2.3_windows_arm64.zip" { + t.Fatalf("archive name = %s", got) + } + if got := ArchiveName("bomly-lite", "1.2.3", runtime.GOOS, "amd64"); !strings.Contains(got, "1.2.3") { + t.Fatalf("archive name = %s", got) + } +} diff --git a/internal/assurance/gotest.go b/internal/assurance/gotest.go new file mode 100644 index 00000000..95773ced --- /dev/null +++ b/internal/assurance/gotest.go @@ -0,0 +1,227 @@ +package assurance + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "sort" + "strings" +) + +// MaxGoTestDetails caps how many individual test outcomes a check result +// records, so one slice with thousands of subtests cannot bloat the report. +const MaxGoTestDetails = 200 + +// maxGoTestLine bounds one line of `go test -json` output. +const maxGoTestLine = 1 << 20 + +// GoTestSummary is the outcome of one `go test -json` stream. +type GoTestSummary struct { + Packages int + Total int + Passed int + Failed int + Skipped int + ElapsedSec float64 + Details []Detail + Truncated int + // FailedTests names the tests that failed, in report order. + FailedTests []string + // Anomalies records non-JSON lines, which usually mean a build failure. + Anomalies []string +} + +type goTestEvent struct { + Action string `json:"Action"` + Package string `json:"Package"` + Test string `json:"Test"` + Elapsed float64 `json:"Elapsed"` + Output string `json:"Output"` +} + +// ParseGoTestEvents reads a `go test -json` stream and summarises it. Lines that +// are not JSON events (build errors, toolchain notices) are recorded as +// anomalies instead of failing the parse. When echo is non-nil, test output is +// replayed to it so CI logs stay readable. +func ParseGoTestEvents(reader io.Reader, echo io.Writer) (GoTestSummary, error) { + summary := GoTestSummary{} + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 64*1024), maxGoTestLine) + + type outcome struct { + status Status + elapsed float64 + } + tests := map[string]outcome{} + packages := map[string]struct{}{} + + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + var event goTestEvent + if err := json.Unmarshal(line, &event); err != nil || event.Action == "" { + text := strings.TrimSpace(string(line)) + if text != "" && len(summary.Anomalies) < 20 { + summary.Anomalies = append(summary.Anomalies, truncate(text, 300)) + } + if echo != nil { + fmt.Fprintln(echo, text) + } + continue + } + if event.Package != "" { + packages[event.Package] = struct{}{} + } + if event.Action == "output" { + if echo != nil { + fmt.Fprint(echo, event.Output) + } + continue + } + if event.Test == "" { + if event.Action == "fail" && event.Elapsed > summary.ElapsedSec { + summary.ElapsedSec = event.Elapsed + } + if event.Action == "pass" && event.Elapsed > summary.ElapsedSec { + summary.ElapsedSec = event.Elapsed + } + continue + } + name := event.Test + if event.Package != "" { + name = shortPackage(event.Package) + "." + event.Test + } + switch event.Action { + case "pass": + tests[name] = outcome{status: StatusPass, elapsed: event.Elapsed} + case "fail": + tests[name] = outcome{status: StatusFail, elapsed: event.Elapsed} + case "skip": + tests[name] = outcome{status: StatusSkip, elapsed: event.Elapsed} + } + } + if err := scanner.Err(); err != nil { + return summary, fmt.Errorf("read go test output: %w", err) + } + + summary.Packages = len(packages) + names := make([]string, 0, len(tests)) + for name := range tests { + names = append(names, name) + } + sort.Strings(names) + + var reportable []string + for _, name := range names { + result := tests[name] + summary.Total++ + switch result.status { + case StatusPass: + summary.Passed++ + case StatusFail: + summary.Failed++ + summary.FailedTests = append(summary.FailedTests, name) + case StatusSkip: + summary.Skipped++ + } + if result.status == StatusFail || testDepth(name) <= 2 { + reportable = append(reportable, name) + } + } + for _, name := range reportable { + if len(summary.Details) >= MaxGoTestDetails { + summary.Truncated = len(reportable) - len(summary.Details) + break + } + result := tests[name] + summary.Details = append(summary.Details, Detail{ + Name: name, Status: result.status, DurationMS: result.elapsed * 1000, + }) + } + return summary, nil +} + +// testDepth counts the subtest levels in a `Package.TestName/sub/case` label. +func testDepth(name string) int { + if index := strings.Index(name, "."); index >= 0 { + name = name[index+1:] + } + return strings.Count(name, "/") + 1 +} + +func shortPackage(pkg string) string { + parts := strings.Split(pkg, "/") + if len(parts) == 0 { + return pkg + } + return parts[len(parts)-1] +} + +func truncate(value string, limit int) string { + if len(value) <= limit { + return value + } + return value[:limit] + "…" +} + +// ToCheckResult turns a `go test -json` summary into a check result. exitCode is +// the test command's exit status, which catches build failures that produce no +// test events at all. +func (s GoTestSummary) ToCheckResult(base CheckResult, exitCode int) CheckResult { + result := base + result.SchemaVersion = CheckSchema + result.Status = StatusPass + switch { + case s.Failed > 0 || exitCode != 0: + result.Status = StatusFail + case s.Total == 0: + result.Status = StatusSkip + } + result.Metrics = map[string]float64{ + "tests_total": float64(s.Total), + "tests_passed": float64(s.Passed), + "tests_failed": float64(s.Failed), + "tests_skipped": float64(s.Skipped), + "packages": float64(s.Packages), + } + result.Details = s.Details + if result.DurationMS == 0 && s.ElapsedSec > 0 { + result.DurationMS = s.ElapsedSec * 1000 + } + result.Summary = s.summaryLine(exitCode) + return result +} + +func (s GoTestSummary) summaryLine(exitCode int) string { + if s.Total == 0 { + if exitCode != 0 { + detail := "" + if len(s.Anomalies) > 0 { + detail = " " + s.Anomalies[0] + } + return fmt.Sprintf("No tests ran and the command exited with code %d.%s", exitCode, detail) + } + return "No tests ran." + } + parts := []string{fmt.Sprintf("%d of %d tests passed", s.Passed, s.Total)} + if s.Failed > 0 { + failed := s.FailedTests + if len(failed) > 3 { + failed = failed[:3] + } + parts = append(parts, fmt.Sprintf("%d failed (%s)", s.Failed, strings.Join(failed, ", "))) + } + if s.Skipped > 0 { + parts = append(parts, fmt.Sprintf("%d skipped", s.Skipped)) + } + if s.Failed == 0 && exitCode != 0 { + parts = append(parts, fmt.Sprintf("the command exited with code %d", exitCode)) + } + if s.Truncated > 0 { + parts = append(parts, fmt.Sprintf("%d further results are not listed", s.Truncated)) + } + return strings.Join(parts, ", ") + "." +} diff --git a/internal/assurance/parse_fuzz_test.go b/internal/assurance/parse_fuzz_test.go new file mode 100644 index 00000000..643a9405 --- /dev/null +++ b/internal/assurance/parse_fuzz_test.go @@ -0,0 +1,130 @@ +package assurance + +import ( + "bytes" + "testing" + + testutil "github.com/bomly-dev/bomly-sdk/testkit" +) + +func FuzzParseCheckResult(f *testing.F) { + for _, seed := range []string{ + "", + "{}", + `{"schema_version":"bomly.assurance-check/v1","id":"smoke","instance":"go","stage":"prerequisites",` + + `"level":"gate","status":"pass","summary":"18 of 18 tests passed.","metrics":{"tests_total":18},` + + `"details":[{"name":"smoke.TestScan","status":"pass","duration_ms":41000}]}`, + `{"schema_version":"bomly.assurance-check/v1","id":"SMOKE","stage":"whenever","status":"green"}`, + `{"schema_version":"bomly.assurance-check/v1","id":"smoke","stage":"prerequisites","status":"pass"`, + `{"schema_version":"bomly.assurance-check/v1","id":"smoke","stage":"prerequisites","status":"pass","summary":"ok"} {}`, + } { + f.Add([]byte(seed)) + } + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > testutil.MaxFuzzInputSize { + return + } + first, firstErr := ParseCheckResult(data) + second, secondErr := ParseCheckResult(data) + if (firstErr == nil) != (secondErr == nil) { + t.Fatalf("ParseCheckResult changed success state: first=%v second=%v", firstErr, secondErr) + } + if firstErr != nil { + if firstErr.Error() != secondErr.Error() { + t.Fatalf("ParseCheckResult changed error: first=%v second=%v", firstErr, secondErr) + } + return + } + if err := first.Validate(); err != nil { + t.Fatalf("accepted an invalid check result: %v", err) + } + encodedFirst, err := first.Encode() + if err != nil { + t.Fatalf("encode accepted result: %v", err) + } + encodedSecond, err := second.Encode() + if err != nil { + t.Fatalf("encode accepted result: %v", err) + } + if !bytes.Equal(encodedFirst, encodedSecond) { + t.Fatal("ParseCheckResult produced different documents for identical input") + } + if _, err := ParseCheckResult(encodedFirst); err != nil { + t.Fatalf("re-parsing an accepted result failed: %v", err) + } + }) +} + +func FuzzParseCatalog(f *testing.F) { + valid := `{"schema_version":"bomly.assurance-catalog/v1",` + + `"areas":[{"id":"end-to-end","title":"End to end","description":"Real runs."}],` + + `"checks":[{"id":"smoke","title":"Smoke","area":"end-to-end","stage":"prerequisites","level":"gate",` + + `"description":"Runs scans.","source":{"workflow":"smoke.yml","job":"smoke"},` + + `"proves":["It scans."],"limitations":["One project."]}],` + + `"evidence":[{"id":"graph-go","title":"Go graph","area":"end-to-end","evidence_level":"release-artifact",` + + `"check_id":"smoke","inputs":[{"kind":"release","location":"https://example.test"}],` + + `"reproduce":[["make","smoke"]],"proves":["It resolves."],"limitations":["One toolchain."]}]}` + for _, seed := range []string{"", "{}", valid, valid[:len(valid)/2], `{"schema_version":"bomly.assurance-catalog/v1","areas":[]}`} { + f.Add([]byte(seed)) + } + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > testutil.MaxFuzzInputSize { + return + } + first, firstErr := ParseCatalog(data) + _, secondErr := ParseCatalog(data) + if (firstErr == nil) != (secondErr == nil) { + t.Fatalf("ParseCatalog changed success state: first=%v second=%v", firstErr, secondErr) + } + if firstErr != nil { + if firstErr.Error() != secondErr.Error() { + t.Fatalf("ParseCatalog changed error: first=%v second=%v", firstErr, secondErr) + } + return + } + if err := first.Validate(); err != nil { + t.Fatalf("accepted an invalid catalog: %v", err) + } + // An accepted catalog must always produce a report, whatever it contains. + BuildReport(first, nil, BuildOptions{Release: Release{Tag: "v0.0.0"}, IncludeEvidence: true}) + }) +} + +func FuzzParseGoTestEvents(f *testing.F) { + for _, seed := range []string{ + "", + `{"Action":"pass","Package":"p","Test":"TestOne","Elapsed":1}`, + "{\"Action\":\"output\",\"Package\":\"p\",\"Test\":\"TestOne\",\"Output\":\"ok\\n\"}\n" + + "{\"Action\":\"fail\",\"Package\":\"p\",\"Test\":\"TestOne/sub\",\"Elapsed\":2}\n", + "# github.com/example\nsyntax error\n", + "{\"Action\":\"pass\"", + } { + f.Add([]byte(seed)) + } + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > testutil.MaxFuzzInputSize { + return + } + first, firstErr := ParseGoTestEvents(bytes.NewReader(data), nil) + second, secondErr := ParseGoTestEvents(bytes.NewReader(data), nil) + if (firstErr == nil) != (secondErr == nil) { + t.Fatalf("ParseGoTestEvents changed success state: first=%v second=%v", firstErr, secondErr) + } + if firstErr != nil { + return + } + if first.Total != second.Total || first.Failed != second.Failed { + t.Fatal("ParseGoTestEvents produced different counts for identical input") + } + base := CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Stage: StagePrerequisites, Level: LevelGate, + } + result := first.ToCheckResult(base, 0) + if err := result.Validate(); err != nil { + t.Fatalf("converted result is invalid: %v", err) + } + if len(result.Details) > MaxGoTestDetails { + t.Fatalf("detail count %d exceeds the cap", len(result.Details)) + } + }) +} diff --git a/internal/tools/benchmarkrun/main.go b/internal/assurance/perfrun/main.go similarity index 99% rename from internal/tools/benchmarkrun/main.go rename to internal/assurance/perfrun/main.go index ee77d463..4e48ec2b 100644 --- a/internal/tools/benchmarkrun/main.go +++ b/internal/assurance/perfrun/main.go @@ -1,4 +1,4 @@ -// Command benchmarkrun repeatedly executes one deterministic assurance case +// Command perfrun repeatedly executes one deterministic assurance case // and records cold/warm samples in a versioned machine-readable manifest. package main @@ -124,7 +124,7 @@ func main() { flag.Parse() command := flag.Args() if len(command) == 0 { - fmt.Fprintln(os.Stderr, "benchmarkrun requires a command after --") + fmt.Fprintln(os.Stderr, "perfrun requires a command after --") os.Exit(2) } if samples < 1 { diff --git a/internal/tools/benchmarkrun/main_test.go b/internal/assurance/perfrun/main_test.go similarity index 100% rename from internal/tools/benchmarkrun/main_test.go rename to internal/assurance/perfrun/main_test.go diff --git a/internal/assurance/releaseassets.go b/internal/assurance/releaseassets.go new file mode 100644 index 00000000..643aa548 --- /dev/null +++ b/internal/assurance/releaseassets.go @@ -0,0 +1,338 @@ +package assurance + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" +) + +// maxAssetBytes bounds one release asset read during verification. +const maxAssetBytes = 512 << 20 + +// maxExtractedBytes bounds one file extracted from a release archive. +const maxExtractedBytes = 512 << 20 + +// releasePlatforms are the operating system and architecture pairs every +// release ships binaries for. +var releasePlatforms = []struct{ OS, Arch string }{ + {"darwin", "amd64"}, {"darwin", "arm64"}, + {"linux", "amd64"}, {"linux", "arm64"}, + {"windows", "amd64"}, {"windows", "arm64"}, +} + +// linuxPackageFormats are the Linux package file extensions a release ships. +var linuxPackageFormats = []string{"apk", "deb", "pkg.tar.zst", "rpm"} + +// ArchiveName returns the release archive file name for one binary, version, +// and platform. version is the tag without its leading "v". +func ArchiveName(binary, version, goos, goarch string) string { + extension := "tar.gz" + if goos == "windows" { + extension = "zip" + } + return fmt.Sprintf("%s_%s_%s_%s.%s", binary, version, goos, goarch, extension) +} + +// ExpectedAssets lists every file a published release must carry, sorted. +// version is the release tag without its leading "v". +func ExpectedAssets(version string) []string { + assets := []string{"SHA256SUMS", "SHA256SUMS.sigstore.json", "multiple.intoto.jsonl"} + for _, platform := range releasePlatforms { + assets = append(assets, + ArchiveName("bomly", version, platform.OS, platform.Arch), + ArchiveName("bomly-lite", version, platform.OS, platform.Arch), + ) + } + for _, arch := range []string{"amd64", "arm64"} { + for _, format := range linuxPackageFormats { + assets = append(assets, fmt.Sprintf("bomly_%s_linux_%s.%s", version, arch, format)) + } + } + sort.Strings(assets) + return assets +} + +// ChecksumEntries maps asset file names to their recorded SHA-256 hashes. +type ChecksumEntries map[string]string + +// ParseSHA256SUMS reads a GoReleaser SHA256SUMS document. +func ParseSHA256SUMS(data []byte) (ChecksumEntries, error) { + entries := ChecksumEntries{} + for index, line := range strings.Split(string(data), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + fields := strings.Fields(trimmed) + if len(fields) != 2 { + return nil, fmt.Errorf("SHA256SUMS line %d is malformed", index+1) + } + hash := fields[0] + name := strings.TrimPrefix(fields[1], "*") + if !hashPattern.MatchString(hash) { + return nil, fmt.Errorf("SHA256SUMS line %d has an invalid hash", index+1) + } + if name == "" || strings.ContainsAny(name, "/\\") { + return nil, fmt.Errorf("SHA256SUMS line %d has an invalid file name", index+1) + } + entries[name] = hash + } + if len(entries) == 0 { + return nil, errCatalog("SHA256SUMS lists no files") + } + return entries, nil +} + +// AssetPresence reports which expected assets a directory holds. +type AssetPresence struct { + Present []string + Missing []string + Extra []string +} + +// InspectAssets compares the files in dir against the expected asset list. +func InspectAssets(dir, version string) (AssetPresence, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return AssetPresence{}, fmt.Errorf("read release asset directory: %w", err) + } + found := map[string]struct{}{} + for _, entry := range entries { + if entry.IsDir() { + continue + } + found[entry.Name()] = struct{}{} + } + presence := AssetPresence{} + expected := map[string]struct{}{} + for _, name := range ExpectedAssets(version) { + expected[name] = struct{}{} + if _, ok := found[name]; ok { + presence.Present = append(presence.Present, name) + continue + } + presence.Missing = append(presence.Missing, name) + } + for name := range found { + if _, ok := expected[name]; !ok { + presence.Extra = append(presence.Extra, name) + } + } + sort.Strings(presence.Extra) + return presence, nil +} + +// ChecksumOutcome is the result of hashing the assets present in a directory. +type ChecksumOutcome struct { + Verified []string + Mismatched []string + Unlisted []string + // NotDownloaded names files SHA256SUMS lists that are absent locally, which + // is expected when only the platform-native assets were fetched. + NotDownloaded []string +} + +// VerifyChecksums hashes every file in dir and compares it to SHA256SUMS. +func VerifyChecksums(dir string, entries ChecksumEntries) (ChecksumOutcome, error) { + outcome := ChecksumOutcome{} + present := map[string]struct{}{} + files, err := os.ReadDir(dir) + if err != nil { + return outcome, fmt.Errorf("read release asset directory: %w", err) + } + for _, file := range files { + if file.IsDir() { + continue + } + name := file.Name() + if name == "SHA256SUMS" || name == "SHA256SUMS.sigstore.json" || name == "multiple.intoto.jsonl" { + continue + } + present[name] = struct{}{} + want, listed := entries[name] + if !listed { + outcome.Unlisted = append(outcome.Unlisted, name) + continue + } + actual, hashErr := hashFile(filepath.Join(dir, name)) + if hashErr != nil { + return outcome, hashErr + } + if actual != want { + outcome.Mismatched = append(outcome.Mismatched, name) + continue + } + outcome.Verified = append(outcome.Verified, name) + } + for name := range entries { + if _, ok := present[name]; !ok { + outcome.NotDownloaded = append(outcome.NotDownloaded, name) + } + } + sort.Strings(outcome.Verified) + sort.Strings(outcome.Mismatched) + sort.Strings(outcome.Unlisted) + sort.Strings(outcome.NotDownloaded) + return outcome, nil +} + +func hashFile(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open %s: %w", path, err) + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, io.LimitReader(file, maxAssetBytes)); err != nil { + return "", fmt.Errorf("hash %s: %w", path, err) + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// BinaryProbe is the outcome of extracting one release archive and asking the +// binary inside it for its version. +type BinaryProbe struct { + Archive string + Binary string + Output string + Status Status + Note string +} + +// ProbeNativeBinaries extracts the archives built for the host platform and +// checks that each binary reports the released version. +func ProbeNativeBinaries(ctx context.Context, dir, version, workDir string) ([]BinaryProbe, error) { + var probes []BinaryProbe + for _, binary := range []string{"bomly", "bomly-lite"} { + archive := ArchiveName(binary, version, runtime.GOOS, runtime.GOARCH) + probe := BinaryProbe{Archive: archive, Binary: binary, Status: StatusFail} + target := filepath.Join(workDir, binary) + if err := os.MkdirAll(target, 0o755); err != nil { + return nil, fmt.Errorf("create extraction directory: %w", err) + } + executable, err := extractBinary(filepath.Join(dir, archive), target, binary) + if err != nil { + probe.Note = err.Error() + probes = append(probes, probe) + continue + } + output, err := runVersion(ctx, executable) + probe.Output = output + switch { + case err != nil: + probe.Note = err.Error() + case !strings.Contains(output, version): + probe.Note = fmt.Sprintf("reported %q, want version %s", output, version) + default: + probe.Status = StatusPass + } + probes = append(probes, probe) + } + return probes, nil +} + +func runVersion(ctx context.Context, executable string) (string, error) { + runCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + command := exec.CommandContext(runCtx, executable, "version") + output, err := command.CombinedOutput() + text := strings.TrimSpace(string(output)) + if err != nil { + return text, fmt.Errorf("run %s version: %w", filepath.Base(executable), err) + } + return text, nil +} + +// extractBinary pulls one named binary out of a release archive into target. +func extractBinary(archivePath, target, binary string) (string, error) { + name := binary + if runtime.GOOS == "windows" { + name += ".exe" + } + if strings.HasSuffix(archivePath, ".zip") { + return extractFromZip(archivePath, target, name) + } + return extractFromTarGz(archivePath, target, name) +} + +func extractFromZip(archivePath, target, name string) (string, error) { + reader, err := zip.OpenReader(archivePath) + if err != nil { + return "", fmt.Errorf("open %s: %w", filepath.Base(archivePath), err) + } + defer reader.Close() + for _, file := range reader.File { + if filepath.Base(file.Name) != name || file.FileInfo().IsDir() { + continue + } + source, openErr := file.Open() + if openErr != nil { + return "", fmt.Errorf("read %s from %s: %w", name, filepath.Base(archivePath), openErr) + } + defer source.Close() + return writeExecutable(filepath.Join(target, name), source) + } + return "", fmt.Errorf("%s does not contain %s", filepath.Base(archivePath), name) +} + +func extractFromTarGz(archivePath, target, name string) (string, error) { + file, err := os.Open(archivePath) + if err != nil { + return "", fmt.Errorf("open %s: %w", filepath.Base(archivePath), err) + } + defer file.Close() + gzipReader, err := gzip.NewReader(file) + if err != nil { + return "", fmt.Errorf("read %s: %w", filepath.Base(archivePath), err) + } + defer gzipReader.Close() + tarReader := tar.NewReader(gzipReader) + for { + header, readErr := tarReader.Next() + if readErr == io.EOF { + break + } + if readErr != nil { + return "", fmt.Errorf("read %s: %w", filepath.Base(archivePath), readErr) + } + if header.Typeflag != tar.TypeReg || filepath.Base(header.Name) != name { + continue + } + return writeExecutable(filepath.Join(target, name), tarReader) + } + return "", fmt.Errorf("%s does not contain %s", filepath.Base(archivePath), name) +} + +// writeExecutable copies a bounded amount of archive content to an executable +// file. Destinations are always names this package chose, never archive paths, +// so an archive cannot direct a write outside the extraction directory. +func writeExecutable(destination string, source io.Reader) (string, error) { + file, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o755) + if err != nil { + return "", fmt.Errorf("create %s: %w", filepath.Base(destination), err) + } + written, copyErr := io.Copy(file, io.LimitReader(source, maxExtractedBytes+1)) + closeErr := file.Close() + if copyErr != nil { + return "", fmt.Errorf("write %s: %w", filepath.Base(destination), copyErr) + } + if closeErr != nil { + return "", fmt.Errorf("close %s: %w", filepath.Base(destination), closeErr) + } + if written > maxExtractedBytes { + return "", fmt.Errorf("%s exceeds the %d byte extraction limit", filepath.Base(destination), maxExtractedBytes) + } + return destination, nil +} diff --git a/internal/assurance/render_markdown.go b/internal/assurance/render_markdown.go new file mode 100644 index 00000000..30c199cf --- /dev/null +++ b/internal/assurance/render_markdown.go @@ -0,0 +1,187 @@ +package assurance + +import ( + "fmt" + "strings" +) + +// StatusIcon returns the emoji shown next to a status in markdown summaries. +func StatusIcon(status Status) string { + switch status { + case StatusPass: + return "✅" + case StatusFail: + return "❌" + case StatusDegraded: + return "⚠️" + case StatusSkip: + return "⏭️" + case StatusMissing: + return "❓" + default: + return "•" + } +} + +// MarkdownOptions controls how a report renders as markdown. +type MarkdownOptions struct { + // Heading is the top-level title; a default is used when empty. + Heading string + // IncludeChecks lists every check in a per-stage table. + IncludeChecks bool + // IncludeTrends adds the comparison against the previous release. + IncludeTrends bool +} + +// RenderMarkdown renders a report as the markdown used for job summaries and +// the per-release tracking issue. +func RenderMarkdown(report Report, opts MarkdownOptions) string { + var out strings.Builder + heading := opts.Heading + if heading == "" { + heading = "Release assurance" + if report.Release.Tag != "" { + heading += " " + report.Release.Tag + } + } + fmt.Fprintf(&out, "## %s %s\n\n", StatusIcon(report.Verdict.Overall), heading) + fmt.Fprintf(&out, "%s\n\n", verdictSentence(report.Verdict)) + + if len(report.Stages) > 0 { + out.WriteString("| Stage | Result | Checks | Passed | Failed | Missing |\n") + out.WriteString("| --- | --- | --- | --- | --- | --- |\n") + for _, stage := range report.Stages { + fmt.Fprintf(&out, "| %s | %s %s | %d | %d | %d | %d |\n", + stage.Title, StatusIcon(stage.Status), stage.Status, + stage.Verdict.Checks, stage.Verdict.Passed, stage.Verdict.Failed, stage.Verdict.Missing) + } + out.WriteString("\n") + } + + if opts.IncludeChecks { + for _, stage := range report.Stages { + if len(stage.CheckIDs) == 0 { + continue + } + fmt.Fprintf(&out, "### %s\n\n", stage.Title) + out.WriteString("| Check | Level | Result | Summary |\n") + out.WriteString("| --- | --- | --- | --- |\n") + for _, id := range stage.CheckIDs { + check, found := report.Check(id) + if !found { + continue + } + fmt.Fprintf(&out, "| %s | %s | %s %s | %s |\n", + check.Title, check.Level, StatusIcon(check.Status), check.Status, + markdownCell(check.Summary)) + } + out.WriteString("\n") + } + } + + if attention := attentionLines(report); len(attention) > 0 { + out.WriteString("### Needs attention\n\n") + for _, line := range attention { + out.WriteString("- " + line + "\n") + } + out.WriteString("\n") + } + + if opts.IncludeTrends && report.Trends != nil { + out.WriteString(renderTrends(*report.Trends)) + } + return out.String() +} + +func verdictSentence(verdict Verdict) string { + switch verdict.Overall { + case StatusPass: + return fmt.Sprintf("All %d checks passed.", verdict.Checks) + case StatusDegraded: + return fmt.Sprintf("%d of %d checks passed; advisory checks reported problems that do not block the release.", + verdict.Passed, verdict.Checks) + case StatusMissing: + return fmt.Sprintf("%d of %d checks reported nothing, so the release cannot be judged complete.", + verdict.Missing, verdict.Checks) + default: + return fmt.Sprintf("%d of %d checks passed; %d failed and %d reported nothing.", + verdict.Passed, verdict.Checks, verdict.Failed, verdict.Missing) + } +} + +func attentionLines(report Report) []string { + var lines []string + for _, check := range report.Checks { + if check.Status == StatusPass || check.Status == StatusSkip { + continue + } + line := fmt.Sprintf("%s **%s** (%s, %s) — %s", + StatusIcon(check.Status), check.Title, check.Level, check.Stage, markdownCell(check.Summary)) + if len(check.MissingInstances) > 0 { + line += fmt.Sprintf(" Missing: %s.", strings.Join(check.MissingInstances, ", ")) + } + lines = append(lines, line) + } + for _, unknown := range report.Unknown { + lines = append(lines, fmt.Sprintf("❓ Result `%s` is not declared in the assurance catalog.", unknown.ID)) + } + return lines +} + +func renderTrends(trends Trends) string { + if len(trends.Changed) == 0 && len(trends.Metrics) == 0 { + return "" + } + var out strings.Builder + fmt.Fprintf(&out, "### Compared with %s\n\n", trends.PreviousTag) + for _, change := range trends.Changed { + fmt.Fprintf(&out, "- `%s`: %s → %s\n", change.CheckID, change.Previous, change.Current) + } + shown := 0 + for _, metric := range trends.Metrics { + if shown >= 8 { + break + } + if metric.Better == betterNeutral && metric.DeltaPct < 5 && metric.DeltaPct > -5 { + continue + } + fmt.Fprintf(&out, "- `%s` %s: %.2f → %.2f (%+.1f%%)\n", + metric.CheckID, metric.Metric, metric.Previous, metric.Current, metric.DeltaPct) + shown++ + } + out.WriteString("\n") + return out.String() +} + +// RenderResultMarkdown renders one check result for a workflow step summary. +func RenderResultMarkdown(result CheckResult) string { + var out strings.Builder + title := result.ID + if result.Instance != "" { + title += " (" + result.Instance + ")" + } + fmt.Fprintf(&out, "### %s %s\n\n%s\n\n", StatusIcon(result.Status), title, result.Summary) + if len(result.Details) > 0 { + out.WriteString("| Item | Result | Note |\n| --- | --- | --- |\n") + shown := 0 + for _, detail := range result.Details { + if shown >= 30 { + fmt.Fprintf(&out, "| … | | %d further items are recorded in the check result |\n", + len(result.Details)-shown) + break + } + fmt.Fprintf(&out, "| %s | %s %s | %s |\n", + markdownCell(detail.Name), StatusIcon(detail.Status), detail.Status, markdownCell(detail.Note)) + shown++ + } + out.WriteString("\n") + } + return out.String() +} + +func markdownCell(value string) string { + cleaned := strings.ReplaceAll(strings.TrimSpace(value), "|", "\\|") + cleaned = strings.ReplaceAll(cleaned, "\r", " ") + cleaned = strings.ReplaceAll(cleaned, "\n", " ") + return cleaned +} diff --git a/internal/assurance/report.go b/internal/assurance/report.go new file mode 100644 index 00000000..c78d02b4 --- /dev/null +++ b/internal/assurance/report.go @@ -0,0 +1,290 @@ +package assurance + +import ( + "encoding/json" + "fmt" + "strings" +) + +// ReportSchema is the schema identifier of a per-release assurance report. +const ReportSchema = "bomly.assurance-report/v1" + +// IndexSchema is the schema identifier of the release index. +const IndexSchema = "bomly.assurance-index/v1" + +// MaxReportBytes bounds a report document. +const MaxReportBytes = 16 << 20 + +// Release identifies the release a report describes. +type Release struct { + Tag string `json:"tag"` + Version string `json:"version"` + Commit string `json:"commit,omitempty"` + URL string `json:"url,omitempty"` + PublishedAt string `json:"published_at,omitempty"` +} + +// Verdict counts outcomes and names the checks that need attention. +type Verdict struct { + Overall Status `json:"overall"` + Checks int `json:"checks"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Degraded int `json:"degraded"` + Skipped int `json:"skipped"` + Missing int `json:"missing"` + GatesFailed []string `json:"gates_failed,omitempty"` + AdvisoriesFailed []string `json:"advisories_failed,omitempty"` + MissingChecks []string `json:"missing_checks,omitempty"` +} + +// Blocking reports whether the verdict must stop a release. +func (v Verdict) Blocking() bool { return len(v.GatesFailed) > 0 || len(v.MissingChecks) > 0 } + +// StageReport is one release stage and the checks it contributed. +type StageReport struct { + ID Stage `json:"id"` + Title string `json:"title"` + Status Status `json:"status"` + Verdict Verdict `json:"verdict"` + RunURL string `json:"run_url,omitempty"` + CheckIDs []string `json:"check_ids"` +} + +// AreaReport summarises one catalog area. +type AreaReport struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Status Status `json:"status"` + Verdict Verdict `json:"verdict"` +} + +// InstanceReport is one reported leg of a check. +type InstanceReport struct { + Name string `json:"name"` + Status Status `json:"status"` + Summary string `json:"summary,omitempty"` + DurationMS float64 `json:"duration_ms,omitempty"` + RunURL string `json:"run_url,omitempty"` + Runner Runner `json:"runner,omitempty"` + Metrics map[string]float64 `json:"metrics,omitempty"` + Details []Detail `json:"details,omitempty"` + Artifacts []Artifact `json:"artifacts,omitempty"` + Links []Link `json:"links,omitempty"` +} + +// ReportCheck is one catalog check merged with the results it received. +type ReportCheck struct { + ID string `json:"id"` + Title string `json:"title"` + Area string `json:"area"` + Stage Stage `json:"stage"` + Level Level `json:"level"` + Description string `json:"description"` + Status Status `json:"status"` + Summary string `json:"summary,omitempty"` + DurationMS float64 `json:"duration_ms,omitempty"` + Source Source `json:"source"` + Instances []InstanceReport `json:"instances,omitempty"` + MissingInstances []string `json:"missing_instances,omitempty"` + Metrics map[string]float64 `json:"metrics,omitempty"` + Reproduce [][]string `json:"reproduce,omitempty"` + Proves []string `json:"proves"` + Limitations []string `json:"limitations"` +} + +// ReportEvidence is one public claim with the status of the check backing it. +type ReportEvidence struct { + ID string `json:"id"` + Title string `json:"title"` + Area string `json:"area"` + EvidenceLevel EvidenceLevel `json:"evidence_level"` + CheckID string `json:"check_id"` + Instance string `json:"instance,omitempty"` + Status Status `json:"status"` + Inputs []Input `json:"inputs"` + RequiredTools []string `json:"required_tools,omitempty"` + Reproduce [][]string `json:"reproduce"` + Artifacts []EvidenceArtifact `json:"artifacts"` + Proves []string `json:"proves"` + Limitations []string `json:"limitations"` +} + +// CoverageRow is one check's status per ecosystem. +type CoverageRow struct { + CheckID string `json:"check_id"` + Title string `json:"title"` + Cells map[string]Status `json:"cells"` +} + +// Coverage maps ecosystems against the checks that exercise them. +type Coverage struct { + Ecosystems []string `json:"ecosystems"` + Rows []CoverageRow `json:"rows"` +} + +// MetricTrend compares one metric against the previous release's report. +type MetricTrend struct { + CheckID string `json:"check_id"` + Metric string `json:"metric"` + Previous float64 `json:"previous"` + Current float64 `json:"current"` + Delta float64 `json:"delta"` + DeltaPct float64 `json:"delta_pct,omitempty"` + Better string `json:"better"` +} + +// StatusChange records a check whose status moved between releases. +type StatusChange struct { + CheckID string `json:"check_id"` + Previous Status `json:"previous"` + Current Status `json:"current"` +} + +// Trends compares this report against the previous release's report. +type Trends struct { + PreviousTag string `json:"previous_tag"` + Metrics []MetricTrend `json:"metrics,omitempty"` + Changed []StatusChange `json:"changed,omitempty"` +} + +// UnknownResult records a reported check that the catalog does not declare. +type UnknownResult struct { + ID string `json:"id"` + Instance string `json:"instance,omitempty"` + Stage Stage `json:"stage"` + Status Status `json:"status"` +} + +// Environment records where the checks ran. +type Environment struct { + Runners []Runner `json:"runners,omitempty"` + GeneratedBy string `json:"generated_by,omitempty"` +} + +// Report is the per-release document the public assurance page renders. +type Report struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt string `json:"generated_at"` + Release Release `json:"release"` + Verdict Verdict `json:"verdict"` + Stages []StageReport `json:"stages"` + Areas []AreaReport `json:"areas"` + Checks []ReportCheck `json:"checks"` + Evidence []ReportEvidence `json:"evidence"` + Coverage Coverage `json:"coverage"` + Trends *Trends `json:"trends,omitempty"` + Unknown []UnknownResult `json:"unknown_results,omitempty"` + Environment Environment `json:"environment"` +} + +// Encode renders the report as indented JSON with a trailing newline. +func (r Report) Encode() ([]byte, error) { + data, err := json.MarshalIndent(r, "", " ") + if err != nil { + return nil, fmt.Errorf("encode assurance report: %w", err) + } + return append(data, '\n'), nil +} + +// Check looks up one reported check by ID. +func (r Report) Check(id string) (ReportCheck, bool) { + for _, check := range r.Checks { + if check.ID == id { + return check, true + } + } + return ReportCheck{}, false +} + +// ParseReport decodes and structurally validates a report document. +func ParseReport(data []byte) (Report, error) { + if len(data) > MaxReportBytes { + return Report{}, fmt.Errorf("assurance report is %d bytes, limit is %d", len(data), MaxReportBytes) + } + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + var report Report + if err := decoder.Decode(&report); err != nil { + return Report{}, fmt.Errorf("decode assurance report: %w", err) + } + if err := ensureEOF(decoder, "assurance report"); err != nil { + return Report{}, err + } + if report.SchemaVersion != ReportSchema { + return Report{}, fmt.Errorf("unsupported assurance report schema %q", report.SchemaVersion) + } + if strings.TrimSpace(report.Release.Tag) == "" { + return Report{}, errCatalog("assurance report is missing its release tag") + } + if !report.Verdict.Overall.Valid() { + return Report{}, fmt.Errorf("assurance report has unsupported verdict %q", report.Verdict.Overall) + } + for _, check := range report.Checks { + if !idPattern.MatchString(check.ID) { + return Report{}, fmt.Errorf("assurance report has invalid check id %q", check.ID) + } + if !check.Status.Valid() { + return Report{}, fmt.Errorf("check %q has unsupported status %q", check.ID, check.Status) + } + } + return report, nil +} + +// LoadReport reads and validates the report at path. +func LoadReport(path string) (Report, error) { + data, err := readBounded(path, MaxReportBytes) + if err != nil { + return Report{}, err + } + return ParseReport(data) +} + +// IndexEntry is one release listed in the assurance index. +type IndexEntry struct { + Tag string `json:"tag"` + Version string `json:"version"` + PublishedAt string `json:"published_at,omitempty"` + GeneratedAt string `json:"generated_at"` + Verdict Status `json:"verdict"` + Gates int `json:"gates_failed"` + Path string `json:"path"` +} + +// Index lists every release that has a published assurance report, newest first. +type Index struct { + SchemaVersion string `json:"schema_version"` + Latest string `json:"latest"` + GeneratedAt string `json:"generated_at"` + Releases []IndexEntry `json:"releases"` +} + +// Encode renders the index as indented JSON with a trailing newline. +func (i Index) Encode() ([]byte, error) { + data, err := json.MarshalIndent(i, "", " ") + if err != nil { + return nil, fmt.Errorf("encode assurance index: %w", err) + } + return append(data, '\n'), nil +} + +// ParseIndex decodes and validates an index document. +func ParseIndex(data []byte) (Index, error) { + if len(data) > MaxReportBytes { + return Index{}, fmt.Errorf("assurance index is %d bytes, limit is %d", len(data), MaxReportBytes) + } + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + var index Index + if err := decoder.Decode(&index); err != nil { + return Index{}, fmt.Errorf("decode assurance index: %w", err) + } + if err := ensureEOF(decoder, "assurance index"); err != nil { + return Index{}, err + } + if index.SchemaVersion != IndexSchema { + return Index{}, fmt.Errorf("unsupported assurance index schema %q", index.SchemaVersion) + } + return index, nil +} diff --git a/internal/tools/sbomassurance/main.go b/internal/assurance/sbominterop/main.go similarity index 97% rename from internal/tools/sbomassurance/main.go rename to internal/assurance/sbominterop/main.go index c0131d89..387410b1 100644 --- a/internal/tools/sbomassurance/main.go +++ b/internal/assurance/sbominterop/main.go @@ -1,6 +1,6 @@ -// Command sbomassurance generates canonical SBOMs and validates them with -// checksum-pinned upstream tools. It is intended only for the explicit -// interoperability assurance workflow. +// Command sbominterop generates canonical SBOMs and validates them with +// checksum-pinned upstream tools. It backs the SBOM interoperability check of +// Bomly's release assurance framework. package main import ( @@ -44,6 +44,7 @@ type runManifest struct { Validators []validatorInfo `json:"validators"` Artifacts []artifactInfo `json:"artifacts"` Commands []commandResult `json:"commands"` + Failure string `json:"failure,omitempty"` } type hostInfo struct { @@ -301,6 +302,7 @@ func describeArtifact(format, path string) (artifactInfo, error) { func writeFailure(outputDir string, manifest *runManifest, cause error) error { manifest.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano) + manifest.Failure = cause.Error() if err := writeManifest(outputDir, *manifest); err != nil { return fmt.Errorf("%v; additionally write run manifest: %w", cause, err) } diff --git a/internal/tools/sbomassurance/main_test.go b/internal/assurance/sbominterop/main_test.go similarity index 100% rename from internal/tools/sbomassurance/main_test.go rename to internal/assurance/sbominterop/main_test.go diff --git a/internal/assurance/testdata/catalog.json b/internal/assurance/testdata/catalog.json new file mode 100644 index 00000000..61ca8bfe --- /dev/null +++ b/internal/assurance/testdata/catalog.json @@ -0,0 +1,90 @@ +{ + "schema_version": "bomly.assurance-catalog/v1", + "areas": [ + { + "id": "end-to-end", + "title": "End-to-end behavior", + "description": "Running the real command line against real projects." + }, + { + "id": "release-integrity", + "title": "Release integrity", + "description": "Whether the published files are complete and unmodified." + }, + { + "id": "performance", + "title": "Speed and stability", + "description": "How long a repeated scan takes and whether it returns the same answer." + } + ], + "checks": [ + { + "id": "perf-samples", + "title": "Repeated scan speed and stability", + "area": "performance", + "stage": "post-release", + "level": "advisory", + "description": "Runs the same scan several times and records timing.", + "source": { "workflow": "assurance-assessment.yml", "job": "perf-samples" }, + "reproduce": [["make", "benchmark-samples"]], + "proves": ["Repeated runs produced identical normalized output."], + "limitations": ["Timings are observations from one machine, not limits."] + }, + { + "id": "release-checksums", + "title": "Release files match their checksums", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Hashes every release file and compares it with SHA256SUMS.", + "source": { "workflow": "release.yml", "job": "verify-draft" }, + "reproduce": [["sha256sum", "--check", "SHA256SUMS"]], + "proves": ["Every published file hashes to the recorded value."], + "limitations": ["Checksums prove the files were not altered after they were built."] + }, + { + "id": "smoke", + "title": "End-to-end scans of real projects", + "area": "end-to-end", + "stage": "prerequisites", + "level": "gate", + "description": "Runs scan against pinned public repositories.", + "source": { "workflow": "smoke.yml", "job": "smoke" }, + "expected_instances": [ + { "name": "go", "ecosystems": ["Go"] }, + { "name": "node", "ecosystems": ["JavaScript"] } + ], + "reproduce": [["make", "smoke"]], + "proves": ["Each ecosystem produces the dependency graph recorded in its golden file."], + "limitations": ["Golden files pin one revision of one example project per ecosystem."] + } + ], + "evidence": [ + { + "id": "graph-go", + "title": "Go module graph", + "area": "end-to-end", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "go", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": ["git", "go"], + "reproduce": [["make", "smoke"]], + "artifacts": [ + { + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", + "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + } + ], + "proves": ["The Go detector resolves a build-tool-backed module graph."], + "limitations": ["The result depends on a compatible Go toolchain."] + } + ] +} diff --git a/internal/assurance/testdata/fixtures/all-pass/results/perf-samples.json b/internal/assurance/testdata/fixtures/all-pass/results/perf-samples.json new file mode 100644 index 00000000..1e728acf --- /dev/null +++ b/internal/assurance/testdata/fixtures/all-pass/results/perf-samples.json @@ -0,0 +1,21 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "perf-samples", + "stage": "post-release", + "level": "advisory", + "status": "pass", + "duration_ms": 52000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/3", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 412 ms, warm median 288 ms).", + "metrics": { + "cold_median_ms": 412, + "warm_median_ms": 288, + "samples_per_mode": 5, + "peak_memory_bytes": 91234304 + } +} diff --git a/internal/assurance/testdata/fixtures/all-pass/results/release-checksums.json b/internal/assurance/testdata/fixtures/all-pass/results/release-checksums.json new file mode 100644 index 00000000..7249bc62 --- /dev/null +++ b/internal/assurance/testdata/fixtures/all-pass/results/release-checksums.json @@ -0,0 +1,20 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "release-checksums", + "stage": "pre-release", + "level": "gate", + "status": "pass", + "duration_ms": 4200, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/2", + "summary": "23 downloaded assets match SHA256SUMS, which lists 23 files.", + "metrics": { + "verified": 23, + "mismatched": 0, + "listed": 23 + } +} diff --git a/internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json b/internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json new file mode 100644 index 00000000..6d623e5a --- /dev/null +++ b/internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json @@ -0,0 +1,29 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "smoke", + "instance": "go", + "stage": "prerequisites", + "level": "gate", + "status": "pass", + "duration_ms": 61000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/1", + "summary": "18 of 18 tests passed.", + "metrics": { + "tests_total": 18, + "tests_passed": 18, + "tests_failed": 0, + "tests_skipped": 0 + }, + "details": [ + { + "name": "smoke.TestScan/scan-go", + "status": "pass", + "duration_ms": 41000 + } + ] +} diff --git a/internal/assurance/testdata/fixtures/all-pass/results/smoke.node.json b/internal/assurance/testdata/fixtures/all-pass/results/smoke.node.json new file mode 100644 index 00000000..0315970b --- /dev/null +++ b/internal/assurance/testdata/fixtures/all-pass/results/smoke.node.json @@ -0,0 +1,22 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "smoke", + "instance": "node", + "stage": "prerequisites", + "level": "gate", + "status": "pass", + "duration_ms": 95000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/1", + "summary": "24 of 24 tests passed.", + "metrics": { + "tests_total": 24, + "tests_passed": 24, + "tests_failed": 0, + "tests_skipped": 0 + } +} diff --git a/internal/assurance/testdata/fixtures/mixed-failure/results/mystery-check.json b/internal/assurance/testdata/fixtures/mixed-failure/results/mystery-check.json new file mode 100644 index 00000000..6874d2f0 --- /dev/null +++ b/internal/assurance/testdata/fixtures/mixed-failure/results/mystery-check.json @@ -0,0 +1,14 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "mystery-check", + "stage": "post-release", + "level": "gate", + "status": "pass", + "duration_ms": 10, + "summary": "A check that the catalog does not declare." +} diff --git a/internal/assurance/testdata/fixtures/mixed-failure/results/perf-samples.json b/internal/assurance/testdata/fixtures/mixed-failure/results/perf-samples.json new file mode 100644 index 00000000..80a2bb7c --- /dev/null +++ b/internal/assurance/testdata/fixtures/mixed-failure/results/perf-samples.json @@ -0,0 +1,21 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "perf-samples", + "stage": "post-release", + "level": "advisory", + "status": "degraded", + "duration_ms": 61000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/11", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms).", + "metrics": { + "cold_median_ms": 705, + "warm_median_ms": 402, + "samples_per_mode": 5, + "peak_memory_bytes": 120586240 + } +} diff --git a/internal/assurance/testdata/fixtures/mixed-failure/results/release-checksums.json b/internal/assurance/testdata/fixtures/mixed-failure/results/release-checksums.json new file mode 100644 index 00000000..609841bf --- /dev/null +++ b/internal/assurance/testdata/fixtures/mixed-failure/results/release-checksums.json @@ -0,0 +1,27 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "release-checksums", + "stage": "pre-release", + "level": "gate", + "status": "fail", + "duration_ms": 3900, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/10", + "summary": "1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz.", + "metrics": { + "verified": 22, + "mismatched": 1, + "listed": 23 + }, + "details": [ + { + "name": "bomly_9.9.9_linux_amd64.tar.gz", + "status": "fail", + "note": "hash does not match SHA256SUMS" + } + ] +} diff --git a/internal/assurance/testdata/fixtures/mixed-failure/results/smoke.go.json b/internal/assurance/testdata/fixtures/mixed-failure/results/smoke.go.json new file mode 100644 index 00000000..56259e85 --- /dev/null +++ b/internal/assurance/testdata/fixtures/mixed-failure/results/smoke.go.json @@ -0,0 +1,22 @@ +{ + "schema_version": "bomly.assurance-check/v1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "id": "smoke", + "instance": "go", + "stage": "prerequisites", + "level": "gate", + "status": "pass", + "duration_ms": 60000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/9", + "summary": "18 of 18 tests passed.", + "metrics": { + "tests_total": 18, + "tests_passed": 18, + "tests_failed": 0, + "tests_skipped": 0 + } +} diff --git a/internal/assurance/testdata/golden/all-pass.report.json b/internal/assurance/testdata/golden/all-pass.report.json new file mode 100644 index 00000000..38b4ba22 --- /dev/null +++ b/internal/assurance/testdata/golden/all-pass.report.json @@ -0,0 +1,370 @@ +{ + "schema_version": "bomly.assurance-report/v1", + "generated_at": "2026-08-19T12:00:00Z", + "release": { + "tag": "v9.9.9", + "version": "9.9.9", + "commit": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + }, + "verdict": { + "overall": "pass", + "checks": 3, + "passed": 3, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "stages": [ + { + "id": "prerequisites", + "title": "Release prerequisites", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "check_ids": [ + "smoke" + ] + }, + { + "id": "pre-release", + "title": "Final pre-release checks", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "check_ids": [ + "release-checksums" + ] + }, + { + "id": "post-release", + "title": "Post-release assessment", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + }, + "check_ids": [ + "perf-samples" + ] + } + ], + "areas": [ + { + "id": "end-to-end", + "title": "End-to-end behavior", + "description": "Running the real command line against real projects.", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + } + }, + { + "id": "release-integrity", + "title": "Release integrity", + "description": "Whether the published files are complete and unmodified.", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + } + }, + { + "id": "performance", + "title": "Speed and stability", + "description": "How long a repeated scan takes and whether it returns the same answer.", + "status": "pass", + "verdict": { + "overall": "pass", + "checks": 1, + "passed": 1, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 0 + } + } + ], + "checks": [ + { + "id": "perf-samples", + "title": "Repeated scan speed and stability", + "area": "performance", + "stage": "post-release", + "level": "advisory", + "description": "Runs the same scan several times and records timing.", + "status": "pass", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 412 ms, warm median 288 ms).", + "duration_ms": 52000, + "source": { + "workflow": "assurance-assessment.yml", + "job": "perf-samples" + }, + "instances": [ + { + "name": "default", + "status": "pass", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 412 ms, warm median 288 ms).", + "duration_ms": 52000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/3", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "cold_median_ms": 412, + "peak_memory_bytes": 91234304, + "samples_per_mode": 5, + "warm_median_ms": 288 + } + } + ], + "metrics": { + "cold_median_ms": 412, + "peak_memory_bytes": 91234304, + "samples_per_mode": 5, + "warm_median_ms": 288 + }, + "reproduce": [ + [ + "make", + "benchmark-samples" + ] + ], + "proves": [ + "Repeated runs produced identical normalized output." + ], + "limitations": [ + "Timings are observations from one machine, not limits." + ] + }, + { + "id": "release-checksums", + "title": "Release files match their checksums", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Hashes every release file and compares it with SHA256SUMS.", + "status": "pass", + "summary": "23 downloaded assets match SHA256SUMS, which lists 23 files.", + "duration_ms": 4200, + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "instances": [ + { + "name": "default", + "status": "pass", + "summary": "23 downloaded assets match SHA256SUMS, which lists 23 files.", + "duration_ms": 4200, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/2", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "listed": 23, + "mismatched": 0, + "verified": 23 + } + } + ], + "metrics": { + "listed": 23, + "mismatched": 0, + "verified": 23 + }, + "reproduce": [ + [ + "sha256sum", + "--check", + "SHA256SUMS" + ] + ], + "proves": [ + "Every published file hashes to the recorded value." + ], + "limitations": [ + "Checksums prove the files were not altered after they were built." + ] + }, + { + "id": "smoke", + "title": "End-to-end scans of real projects", + "area": "end-to-end", + "stage": "prerequisites", + "level": "gate", + "description": "Runs scan against pinned public repositories.", + "status": "pass", + "summary": "2 instances passed.", + "duration_ms": 156000, + "source": { + "workflow": "smoke.yml", + "job": "smoke" + }, + "instances": [ + { + "name": "go", + "status": "pass", + "summary": "18 of 18 tests passed.", + "duration_ms": 61000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "tests_failed": 0, + "tests_passed": 18, + "tests_skipped": 0, + "tests_total": 18 + }, + "details": [ + { + "name": "smoke.TestScan/scan-go", + "status": "pass", + "duration_ms": 41000 + } + ] + }, + { + "name": "node", + "status": "pass", + "summary": "24 of 24 tests passed.", + "duration_ms": 95000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/1", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "tests_failed": 0, + "tests_passed": 24, + "tests_skipped": 0, + "tests_total": 24 + } + } + ], + "metrics": { + "tests_failed": 0, + "tests_passed": 42, + "tests_skipped": 0, + "tests_total": 42 + }, + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "proves": [ + "Each ecosystem produces the dependency graph recorded in its golden file." + ], + "limitations": [ + "Golden files pin one revision of one example project per ecosystem." + ] + } + ], + "evidence": [ + { + "id": "graph-go", + "title": "Go module graph", + "area": "end-to-end", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "go", + "status": "pass", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "artifacts": [ + { + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", + "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + } + ], + "proves": [ + "The Go detector resolves a build-tool-backed module graph." + ], + "limitations": [ + "The result depends on a compatible Go toolchain." + ] + } + ], + "coverage": { + "ecosystems": [ + "Go", + "JavaScript" + ], + "rows": [ + { + "check_id": "smoke", + "title": "End-to-end scans of real projects", + "cells": { + "Go": "pass", + "JavaScript": "pass" + } + } + ] + }, + "environment": { + "runners": [ + { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + } + ], + "generated_by": "assurance-test" + } +} diff --git a/internal/assurance/testdata/golden/all-pass.summary.md b/internal/assurance/testdata/golden/all-pass.summary.md new file mode 100644 index 00000000..334e6ef4 --- /dev/null +++ b/internal/assurance/testdata/golden/all-pass.summary.md @@ -0,0 +1,28 @@ +## ✅ Release assurance v9.9.9 + +All 3 checks passed. + +| Stage | Result | Checks | Passed | Failed | Missing | +| --- | --- | --- | --- | --- | --- | +| Release prerequisites | ✅ pass | 1 | 1 | 0 | 0 | +| Final pre-release checks | ✅ pass | 1 | 1 | 0 | 0 | +| Post-release assessment | ✅ pass | 1 | 1 | 0 | 0 | + +### Release prerequisites + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| End-to-end scans of real projects | gate | ✅ pass | 2 instances passed. | + +### Final pre-release checks + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| Release files match their checksums | gate | ✅ pass | 23 downloaded assets match SHA256SUMS, which lists 23 files. | + +### Post-release assessment + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| Repeated scan speed and stability | advisory | ✅ pass | canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 412 ms, warm median 288 ms). | + diff --git a/internal/assurance/testdata/golden/mixed-failure.report.json b/internal/assurance/testdata/golden/mixed-failure.report.json new file mode 100644 index 00000000..29bfb088 --- /dev/null +++ b/internal/assurance/testdata/golden/mixed-failure.report.json @@ -0,0 +1,473 @@ +{ + "schema_version": "bomly.assurance-report/v1", + "generated_at": "2026-08-19T12:00:00Z", + "release": { + "tag": "v9.9.9", + "version": "9.9.9", + "commit": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + }, + "verdict": { + "overall": "fail", + "checks": 3, + "passed": 0, + "failed": 1, + "degraded": 1, + "skipped": 0, + "missing": 1, + "gates_failed": [ + "release-checksums" + ], + "advisories_failed": [ + "perf-samples" + ], + "missing_checks": [ + "smoke" + ] + }, + "stages": [ + { + "id": "prerequisites", + "title": "Release prerequisites", + "status": "missing", + "verdict": { + "overall": "missing", + "checks": 1, + "passed": 0, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 1, + "missing_checks": [ + "smoke" + ] + }, + "check_ids": [ + "smoke" + ] + }, + { + "id": "pre-release", + "title": "Final pre-release checks", + "status": "fail", + "verdict": { + "overall": "fail", + "checks": 1, + "passed": 0, + "failed": 1, + "degraded": 0, + "skipped": 0, + "missing": 0, + "gates_failed": [ + "release-checksums" + ] + }, + "check_ids": [ + "release-checksums" + ] + }, + { + "id": "post-release", + "title": "Post-release assessment", + "status": "degraded", + "verdict": { + "overall": "degraded", + "checks": 1, + "passed": 0, + "failed": 0, + "degraded": 1, + "skipped": 0, + "missing": 0, + "advisories_failed": [ + "perf-samples" + ] + }, + "check_ids": [ + "perf-samples" + ] + } + ], + "areas": [ + { + "id": "end-to-end", + "title": "End-to-end behavior", + "description": "Running the real command line against real projects.", + "status": "missing", + "verdict": { + "overall": "missing", + "checks": 1, + "passed": 0, + "failed": 0, + "degraded": 0, + "skipped": 0, + "missing": 1, + "missing_checks": [ + "smoke" + ] + } + }, + { + "id": "release-integrity", + "title": "Release integrity", + "description": "Whether the published files are complete and unmodified.", + "status": "fail", + "verdict": { + "overall": "fail", + "checks": 1, + "passed": 0, + "failed": 1, + "degraded": 0, + "skipped": 0, + "missing": 0, + "gates_failed": [ + "release-checksums" + ] + } + }, + { + "id": "performance", + "title": "Speed and stability", + "description": "How long a repeated scan takes and whether it returns the same answer.", + "status": "degraded", + "verdict": { + "overall": "degraded", + "checks": 1, + "passed": 0, + "failed": 0, + "degraded": 1, + "skipped": 0, + "missing": 0, + "advisories_failed": [ + "perf-samples" + ] + } + } + ], + "checks": [ + { + "id": "perf-samples", + "title": "Repeated scan speed and stability", + "area": "performance", + "stage": "post-release", + "level": "advisory", + "description": "Runs the same scan several times and records timing.", + "status": "degraded", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms).", + "duration_ms": 61000, + "source": { + "workflow": "assurance-assessment.yml", + "job": "perf-samples" + }, + "instances": [ + { + "name": "default", + "status": "degraded", + "summary": "canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms).", + "duration_ms": 61000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/11", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "cold_median_ms": 705, + "peak_memory_bytes": 120586240, + "samples_per_mode": 5, + "warm_median_ms": 402 + } + } + ], + "metrics": { + "cold_median_ms": 705, + "peak_memory_bytes": 120586240, + "samples_per_mode": 5, + "warm_median_ms": 402 + }, + "reproduce": [ + [ + "make", + "benchmark-samples" + ] + ], + "proves": [ + "Repeated runs produced identical normalized output." + ], + "limitations": [ + "Timings are observations from one machine, not limits." + ] + }, + { + "id": "release-checksums", + "title": "Release files match their checksums", + "area": "release-integrity", + "stage": "pre-release", + "level": "gate", + "description": "Hashes every release file and compares it with SHA256SUMS.", + "status": "fail", + "summary": "1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz.", + "duration_ms": 3900, + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "instances": [ + { + "name": "default", + "status": "fail", + "summary": "1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz.", + "duration_ms": 3900, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/10", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "listed": 23, + "mismatched": 1, + "verified": 22 + }, + "details": [ + { + "name": "bomly_9.9.9_linux_amd64.tar.gz", + "status": "fail", + "note": "hash does not match SHA256SUMS" + } + ] + } + ], + "metrics": { + "listed": 23, + "mismatched": 1, + "verified": 22 + }, + "reproduce": [ + [ + "sha256sum", + "--check", + "SHA256SUMS" + ] + ], + "proves": [ + "Every published file hashes to the recorded value." + ], + "limitations": [ + "Checksums prove the files were not altered after they were built." + ] + }, + { + "id": "smoke", + "title": "End-to-end scans of real projects", + "area": "end-to-end", + "stage": "prerequisites", + "level": "gate", + "description": "Runs scan against pinned public repositories.", + "status": "missing", + "summary": "1 instance passed, 1 instance reported nothing.", + "duration_ms": 60000, + "source": { + "workflow": "smoke.yml", + "job": "smoke" + }, + "instances": [ + { + "name": "go", + "status": "pass", + "summary": "18 of 18 tests passed.", + "duration_ms": 60000, + "run_url": "https://github.com/bomly-dev/bomly-cli/actions/runs/9", + "runner": { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + }, + "metrics": { + "tests_failed": 0, + "tests_passed": 18, + "tests_skipped": 0, + "tests_total": 18 + } + } + ], + "missing_instances": [ + "node" + ], + "metrics": { + "tests_failed": 0, + "tests_passed": 18, + "tests_skipped": 0, + "tests_total": 18 + }, + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "proves": [ + "Each ecosystem produces the dependency graph recorded in its golden file." + ], + "limitations": [ + "Golden files pin one revision of one example project per ecosystem." + ] + } + ], + "evidence": [ + { + "id": "graph-go", + "title": "Go module graph", + "area": "end-to-end", + "evidence_level": "pinned-input", + "check_id": "smoke", + "instance": "go", + "status": "pass", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "artifacts": [ + { + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", + "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + } + ], + "proves": [ + "The Go detector resolves a build-tool-backed module graph." + ], + "limitations": [ + "The result depends on a compatible Go toolchain." + ] + } + ], + "coverage": { + "ecosystems": [ + "Go", + "JavaScript" + ], + "rows": [ + { + "check_id": "smoke", + "title": "End-to-end scans of real projects", + "cells": { + "Go": "pass", + "JavaScript": "missing" + } + } + ] + }, + "trends": { + "previous_tag": "v9.9.9", + "metrics": [ + { + "check_id": "perf-samples", + "metric": "cold_median_ms", + "previous": 412, + "current": 705, + "delta": 293, + "delta_pct": 71.11650485436894, + "better": "lower" + }, + { + "check_id": "perf-samples", + "metric": "peak_memory_bytes", + "previous": 91234304, + "current": 120586240, + "delta": 29351936, + "delta_pct": 32.172039148783334, + "better": "lower" + }, + { + "check_id": "perf-samples", + "metric": "warm_median_ms", + "previous": 288, + "current": 402, + "delta": 114, + "delta_pct": 39.58333333333333, + "better": "lower" + }, + { + "check_id": "release-checksums", + "metric": "mismatched", + "previous": 0, + "current": 1, + "delta": 1, + "better": "neutral" + }, + { + "check_id": "release-checksums", + "metric": "verified", + "previous": 23, + "current": 22, + "delta": -1, + "delta_pct": -4.3478260869565215, + "better": "neutral" + }, + { + "check_id": "smoke", + "metric": "tests_passed", + "previous": 42, + "current": 18, + "delta": -24, + "delta_pct": -57.14285714285714, + "better": "higher" + }, + { + "check_id": "smoke", + "metric": "tests_total", + "previous": 42, + "current": 18, + "delta": -24, + "delta_pct": -57.14285714285714, + "better": "higher" + } + ], + "changed": [ + { + "check_id": "perf-samples", + "previous": "pass", + "current": "degraded" + }, + { + "check_id": "release-checksums", + "previous": "pass", + "current": "fail" + }, + { + "check_id": "smoke", + "previous": "pass", + "current": "missing" + } + ] + }, + "unknown_results": [ + { + "id": "mystery-check", + "stage": "post-release", + "status": "pass" + } + ], + "environment": { + "runners": [ + { + "os": "linux", + "arch": "amd64", + "go_version": "go1.26.3" + } + ], + "generated_by": "assurance-test" + } +} diff --git a/internal/assurance/testdata/golden/mixed-failure.summary.md b/internal/assurance/testdata/golden/mixed-failure.summary.md new file mode 100644 index 00000000..d4d8989c --- /dev/null +++ b/internal/assurance/testdata/golden/mixed-failure.summary.md @@ -0,0 +1,46 @@ +## ❌ Release assurance v9.9.9 + +0 of 3 checks passed; 1 failed and 1 reported nothing. + +| Stage | Result | Checks | Passed | Failed | Missing | +| --- | --- | --- | --- | --- | --- | +| Release prerequisites | ❓ missing | 1 | 0 | 0 | 1 | +| Final pre-release checks | ❌ fail | 1 | 0 | 1 | 0 | +| Post-release assessment | ⚠️ degraded | 1 | 0 | 0 | 0 | + +### Release prerequisites + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| End-to-end scans of real projects | gate | ❓ missing | 1 instance passed, 1 instance reported nothing. | + +### Final pre-release checks + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| Release files match their checksums | gate | ❌ fail | 1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz. | + +### Post-release assessment + +| Check | Level | Result | Summary | +| --- | --- | --- | --- | +| Repeated scan speed and stability | advisory | ⚠️ degraded | canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms). | + +### Needs attention + +- ⚠️ **Repeated scan speed and stability** (advisory, post-release) — canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms). +- ❌ **Release files match their checksums** (gate, pre-release) — 1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz. +- ❓ **End-to-end scans of real projects** (gate, prerequisites) — 1 instance passed, 1 instance reported nothing. Missing: node. +- ❓ Result `mystery-check` is not declared in the assurance catalog. + +### Compared with v9.9.9 + +- `perf-samples`: pass → degraded +- `release-checksums`: pass → fail +- `smoke`: pass → missing +- `perf-samples` cold_median_ms: 412.00 → 705.00 (+71.1%) +- `perf-samples` peak_memory_bytes: 91234304.00 → 120586240.00 (+32.2%) +- `perf-samples` warm_median_ms: 288.00 → 402.00 (+39.6%) +- `smoke` tests_passed: 42.00 → 18.00 (-57.1%) +- `smoke` tests_total: 42.00 → 18.00 (-57.1%) + diff --git a/internal/assurance/trends.go b/internal/assurance/trends.go new file mode 100644 index 00000000..998b36cb --- /dev/null +++ b/internal/assurance/trends.go @@ -0,0 +1,85 @@ +package assurance + +import ( + "sort" + "strings" +) + +// Direction names whether a rising metric is an improvement. +const ( + betterLower = "lower" + betterHigher = "higher" + betterNeutral = "neutral" +) + +// lowerIsBetterSuffixes name metrics where a smaller number is an improvement. +var lowerIsBetterSuffixes = []string{"_ms", "_bytes", "_seconds", "_failed"} + +// higherIsBetterSuffixes name metrics where a larger number is an improvement. +var higherIsBetterSuffixes = []string{"_passed", "_completed", "_total", "targets", "cases", "assets", "checks"} + +// metricDirection reports whether higher or lower is better for a metric. +func metricDirection(name string) string { + for _, suffix := range lowerIsBetterSuffixes { + if strings.HasSuffix(name, suffix) { + return betterLower + } + } + for _, suffix := range higherIsBetterSuffixes { + if strings.HasSuffix(name, suffix) { + return betterHigher + } + } + return betterNeutral +} + +// buildTrends compares the current report against the previous release's report +// and records metric deltas plus every check whose status changed. +func buildTrends(previous, current Report) *Trends { + trends := &Trends{PreviousTag: previous.Release.Tag} + previousChecks := make(map[string]ReportCheck, len(previous.Checks)) + for _, check := range previous.Checks { + previousChecks[check.ID] = check + } + for _, check := range current.Checks { + before, exists := previousChecks[check.ID] + if !exists { + continue + } + if before.Status != check.Status { + trends.Changed = append(trends.Changed, StatusChange{ + CheckID: check.ID, Previous: before.Status, Current: check.Status, + }) + } + names := make([]string, 0, len(check.Metrics)) + for name := range check.Metrics { + if _, present := before.Metrics[name]; present { + names = append(names, name) + } + } + sort.Strings(names) + for _, name := range names { + was := before.Metrics[name] + now := check.Metrics[name] + if was == now { + continue + } + trend := MetricTrend{ + CheckID: check.ID, Metric: name, Previous: was, Current: now, + Delta: now - was, Better: metricDirection(name), + } + if was != 0 { + trend.DeltaPct = (now - was) / was * 100 + } + trends.Metrics = append(trends.Metrics, trend) + } + } + sort.Slice(trends.Changed, func(i, j int) bool { return trends.Changed[i].CheckID < trends.Changed[j].CheckID }) + sort.Slice(trends.Metrics, func(i, j int) bool { + if trends.Metrics[i].CheckID != trends.Metrics[j].CheckID { + return trends.Metrics[i].CheckID < trends.Metrics[j].CheckID + } + return trends.Metrics[i].Metric < trends.Metrics[j].Metric + }) + return trends +} diff --git a/internal/tools/publicevidence/main.go b/internal/tools/publicevidence/main.go deleted file mode 100644 index 441cbc7d..00000000 --- a/internal/tools/publicevidence/main.go +++ /dev/null @@ -1,349 +0,0 @@ -// Command publicevidence validates and displays Bomly's public evidence -// catalog. -package main - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "os" - "path/filepath" - "regexp" - "sort" - "strings" -) - -const ( - catalogSchema = "bomly.public-evidence/v1" - defaultCatalog = "test/evidence/cases.json" -) - -var ( - caseIDPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) - revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) - hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) - containerDigestPattern = regexp.MustCompile(`@sha256:[0-9a-f]{64}$`) -) - -type catalog struct { - SchemaVersion string `json:"schema_version"` - Cases []evidenceCase `json:"cases"` -} - -type evidenceCase struct { - ID string `json:"id"` - Title string `json:"title"` - Area string `json:"area"` - EvidenceLevel string `json:"evidence_level"` - Inputs []input `json:"inputs"` - RequiredTools []string `json:"required_tools,omitempty"` - Reproduce [][]string `json:"reproduce"` - Evidence []artifact `json:"evidence"` - Proves []string `json:"proves"` - Limitations []string `json:"limitations"` -} - -type input struct { - Kind string `json:"kind"` - Location string `json:"location"` - Ref string `json:"ref,omitempty"` - Revision string `json:"revision,omitempty"` - SHA256 string `json:"sha256,omitempty"` -} - -type artifact struct { - Path string `json:"path"` - SHA256 string `json:"sha256"` -} - -func main() { - catalogPath := flag.String("catalog", defaultCatalog, "path to the public evidence catalog") - caseID := flag.String("case", "", "show one evidence case") - flag.Parse() - - root, err := repositoryRoot() - if err != nil { - exitError(err) - } - resolvedCatalog := resolveCatalogPath(root, *catalogPath) - loaded, err := loadCatalog(resolvedCatalog) - if err != nil { - exitError(err) - } - if err := validateCatalog(root, loaded); err != nil { - exitError(err) - } - - selected := loaded.Cases - if *caseID != "" { - selected = nil - for _, current := range loaded.Cases { - if current.ID == *caseID { - selected = []evidenceCase{current} - break - } - } - if len(selected) == 0 { - exitError(fmt.Errorf("unknown evidence case %q", *caseID)) - } - } - printCases(selected) -} - -func repositoryRoot() (string, error) { - current, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("resolve working directory: %w", err) - } - for { - if info, statErr := os.Stat(filepath.Join(current, "go.mod")); statErr == nil && !info.IsDir() { - return current, nil - } - parent := filepath.Dir(current) - if parent == current { - return "", errors.New("find repository root: go.mod not found") - } - current = parent - } -} - -func resolveCatalogPath(root, catalogPath string) string { - resolved := filepath.FromSlash(catalogPath) - if filepath.IsAbs(resolved) { - return resolved - } - return filepath.Join(root, resolved) -} - -func loadCatalog(path string) (catalog, error) { - file, err := os.Open(path) - if err != nil { - return catalog{}, fmt.Errorf("open evidence catalog: %w", err) - } - defer file.Close() - - decoder := json.NewDecoder(io.LimitReader(file, 2<<20)) - decoder.DisallowUnknownFields() - var loaded catalog - if err := decoder.Decode(&loaded); err != nil { - return catalog{}, fmt.Errorf("decode evidence catalog: %w", err) - } - if err := ensureJSONEOF(decoder); err != nil { - return catalog{}, err - } - return loaded, nil -} - -func ensureJSONEOF(decoder *json.Decoder) error { - var extra any - err := decoder.Decode(&extra) - if errors.Is(err, io.EOF) { - return nil - } - if err == nil { - return errors.New("decode evidence catalog: multiple JSON values") - } - return fmt.Errorf("decode evidence catalog trailing data: %w", err) -} - -func validateCatalog(root string, loaded catalog) error { - if loaded.SchemaVersion != catalogSchema { - return fmt.Errorf("unsupported evidence catalog schema %q", loaded.SchemaVersion) - } - if len(loaded.Cases) == 0 { - return errors.New("evidence catalog contains no cases") - } - seen := make(map[string]struct{}, len(loaded.Cases)) - previous := "" - for index, current := range loaded.Cases { - if !caseIDPattern.MatchString(current.ID) { - return fmt.Errorf("case %d has invalid id %q", index+1, current.ID) - } - if _, exists := seen[current.ID]; exists { - return fmt.Errorf("duplicate evidence case %q", current.ID) - } - seen[current.ID] = struct{}{} - if previous != "" && current.ID < previous { - return fmt.Errorf("evidence cases are not sorted: %q follows %q", current.ID, previous) - } - previous = current.ID - } - for _, current := range loaded.Cases { - if err := validateCase(root, current); err != nil { - return fmt.Errorf("case %q: %w", current.ID, err) - } - } - return nil -} - -func validateCase(root string, current evidenceCase) error { - if strings.TrimSpace(current.Title) == "" || strings.TrimSpace(current.Area) == "" { - return errors.New("title and area are required") - } - switch current.EvidenceLevel { - case "deterministic", "pinned-input", "live-service", "manual-assurance", "snapshot": - default: - return fmt.Errorf("unsupported evidence level %q", current.EvidenceLevel) - } - if len(current.Inputs) == 0 { - return errors.New("at least one input is required") - } - for _, item := range current.Inputs { - if err := validateInput(root, current.EvidenceLevel, item); err != nil { - return err - } - } - if len(current.Reproduce) == 0 { - return errors.New("at least one reproduction command is required") - } - for index, command := range current.Reproduce { - if len(command) == 0 { - return fmt.Errorf("reproduction command %d is empty", index+1) - } - for _, argument := range command { - if argument == "" { - return fmt.Errorf("reproduction command %d contains an empty argument", index+1) - } - } - } - if len(current.Evidence) == 0 { - return errors.New("at least one evidence artifact is required") - } - for _, item := range current.Evidence { - if err := validateArtifact(root, item); err != nil { - return err - } - } - if len(current.Proves) == 0 || len(current.Limitations) == 0 { - return errors.New("proves and limitations must both be explicit") - } - for _, claim := range current.Proves { - if strings.TrimSpace(claim) == "" { - return errors.New("proves and limitations cannot contain blank entries") - } - } - for _, limitation := range current.Limitations { - if strings.TrimSpace(limitation) == "" { - return errors.New("proves and limitations cannot contain blank entries") - } - } - return nil -} - -func validateInput(root, evidenceLevel string, current input) error { - if strings.TrimSpace(current.Location) == "" { - return errors.New("input location is required") - } - switch current.Kind { - case "git": - if !revisionPattern.MatchString(current.Revision) { - return errors.New("git input requires a full lowercase commit revision") - } - case "fixture": - if !hashPattern.MatchString(current.SHA256) { - return errors.New("fixture input requires a SHA-256 hash") - } - return validateArtifact(root, artifact{Path: current.Location, SHA256: current.SHA256}) - case "container": - if current.Ref == "" { - return errors.New("container input requires an image reference") - } - if evidenceLevel == "pinned-input" && !containerDigestPattern.MatchString(current.Ref) { - return errors.New("pinned container input requires an immutable sha256 digest") - } - case "workflow": - if !hashPattern.MatchString(current.SHA256) { - return errors.New("workflow input requires a SHA-256 hash") - } - return validateArtifact(root, artifact{Path: current.Location, SHA256: current.SHA256}) - default: - return fmt.Errorf("unsupported input kind %q", current.Kind) - } - return nil -} - -func validateArtifact(root string, item artifact) error { - if !hashPattern.MatchString(item.SHA256) { - return fmt.Errorf("artifact %q has an invalid SHA-256 hash", item.Path) - } - clean := filepath.Clean(filepath.FromSlash(item.Path)) - if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { - return fmt.Errorf("artifact path %q must stay inside the repository", item.Path) - } - resolvedRoot, err := filepath.EvalSymlinks(root) - if err != nil { - return fmt.Errorf("resolve repository root: %w", err) - } - path := filepath.Join(root, clean) - resolvedPath, err := filepath.EvalSymlinks(path) - if err != nil { - return fmt.Errorf("resolve artifact %q: %w", item.Path, err) - } - relative, err := filepath.Rel(resolvedRoot, resolvedPath) - if err != nil { - return fmt.Errorf("resolve artifact %q relative to repository: %w", item.Path, err) - } - if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { - return fmt.Errorf("artifact path %q resolves outside the repository", item.Path) - } - info, err := os.Stat(resolvedPath) - if err != nil { - return fmt.Errorf("inspect artifact %q: %w", item.Path, err) - } - if !info.Mode().IsRegular() { - return fmt.Errorf("artifact %q is not a regular file", item.Path) - } - data, err := os.ReadFile(resolvedPath) - if err != nil { - return fmt.Errorf("read artifact %q: %w", item.Path, err) - } - sum := sha256.Sum256(data) - actual := hex.EncodeToString(sum[:]) - if actual != item.SHA256 { - return fmt.Errorf("artifact %q hash is %s, want %s", item.Path, actual, item.SHA256) - } - return nil -} - -func printCases(cases []evidenceCase) { - sort.Slice(cases, func(i, j int) bool { return cases[i].ID < cases[j].ID }) - fmt.Printf("Verified %d public evidence case(s).\n", len(cases)) - for _, current := range cases { - fmt.Printf("\n%s — %s\n", current.ID, current.Title) - fmt.Printf(" Area: %s; evidence: %s\n", current.Area, current.EvidenceLevel) - for _, command := range current.Reproduce { - fmt.Printf(" Reproduce: %s\n", shellCommand(command)) - } - for _, limitation := range current.Limitations { - fmt.Printf(" Limitation: %s\n", limitation) - } - } -} - -func shellCommand(command []string) string { - quoted := make([]string, len(command)) - for index, argument := range command { - if argument != "" && strings.IndexFunc(argument, func(r rune) bool { - if (r >= 'a' && r <= 'z') || - (r >= 'A' && r <= 'Z') || - (r >= '0' && r <= '9') { - return false - } - return !strings.ContainsRune("@%_+=:,./-", r) - }) == -1 { - quoted[index] = argument - continue - } - quoted[index] = "'" + strings.ReplaceAll(argument, "'", "'\"'\"'") + "'" - } - return strings.Join(quoted, " ") -} - -func exitError(err error) { - fmt.Fprintln(os.Stderr, "public evidence:", err) - os.Exit(1) -} diff --git a/internal/tools/publicevidence/main_test.go b/internal/tools/publicevidence/main_test.go deleted file mode 100644 index 4b76beb4..00000000 --- a/internal/tools/publicevidence/main_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package main - -import ( - "crypto/sha256" - "encoding/hex" - "os" - "path/filepath" - "runtime" - "strings" - "testing" -) - -func TestLoadAndValidateCatalog(t *testing.T) { - root := t.TempDir() - artifactPath := filepath.Join(root, "result.json") - data := []byte(`{"ok":true}`) - if err := os.WriteFile(artifactPath, data, 0o600); err != nil { - t.Fatal(err) - } - sum := sha256.Sum256(data) - hash := hex.EncodeToString(sum[:]) - catalogPath := filepath.Join(root, "cases.json") - document := `{ - "schema_version": "bomly.public-evidence/v1", - "cases": [{ - "id": "example-case", - "title": "Example", - "area": "graph", - "evidence_level": "deterministic", - "inputs": [{"kind": "fixture", "location": "result.json", "sha256": "` + hash + `"}], - "reproduce": [["go", "test", "./..."]], - "evidence": [{"path": "result.json", "sha256": "` + hash + `"}], - "proves": ["The example succeeds."], - "limitations": ["The example covers one input."] - }] -}` - if err := os.WriteFile(catalogPath, []byte(document), 0o600); err != nil { - t.Fatal(err) - } - - loaded, err := loadCatalog(catalogPath) - if err != nil { - t.Fatal(err) - } - if err := validateCatalog(root, loaded); err != nil { - t.Fatal(err) - } - - loaded.Cases[0].Proves = []string{" "} - if err := validateCatalog(root, loaded); err == nil || !strings.Contains(err.Error(), "blank entries") { - t.Fatalf("validateCatalog() blank proof error = %v", err) - } - loaded.Cases[0].Proves = []string{"The example succeeds."} - loaded.Cases[0].Limitations = []string{"\t"} - if err := validateCatalog(root, loaded); err == nil || !strings.Contains(err.Error(), "blank entries") { - t.Fatalf("validateCatalog() blank limitation error = %v", err) - } -} - -func TestValidateCatalogRejectsUnpinnedGitAndChangedArtifact(t *testing.T) { - root := t.TempDir() - if err := os.WriteFile(filepath.Join(root, "result.json"), []byte("{}"), 0o600); err != nil { - t.Fatal(err) - } - current := catalog{ - SchemaVersion: catalogSchema, - Cases: []evidenceCase{{ - ID: "git-case", - Title: "Git case", - Area: "graph", - EvidenceLevel: "pinned-input", - Inputs: []input{{ - Kind: "git", - Location: "https://github.com/example/project", - Revision: "main", - }}, - Reproduce: [][]string{{"go", "test", "./..."}}, - Evidence: []artifact{{Path: "result.json", SHA256: strings.Repeat("0", 64)}}, - Proves: []string{"A result."}, - Limitations: []string{"One input."}, - }}, - } - if err := validateCatalog(root, current); err == nil || !strings.Contains(err.Error(), "full lowercase commit") { - t.Fatalf("validateCatalog() error = %v", err) - } - - current.Cases[0].Inputs[0].Revision = strings.Repeat("a", 40) - if err := validateCatalog(root, current); err == nil || !strings.Contains(err.Error(), "artifact") { - t.Fatalf("validateCatalog() error = %v", err) - } -} - -func TestValidateCatalogRejectsUnsortedAndUnknownFields(t *testing.T) { - root := t.TempDir() - current := catalog{ - SchemaVersion: catalogSchema, - Cases: []evidenceCase{ - {ID: "z-case"}, - {ID: "a-case"}, - }, - } - if err := validateCatalog(root, current); err == nil || !strings.Contains(err.Error(), "not sorted") { - t.Fatalf("validateCatalog() error = %v", err) - } - - path := filepath.Join(root, "cases.json") - if err := os.WriteFile(path, []byte(`{"schema_version":"bomly.public-evidence/v1","cases":[],"extra":true}`), 0o600); err != nil { - t.Fatal(err) - } - if _, err := loadCatalog(path); err == nil || !strings.Contains(err.Error(), "unknown field") { - t.Fatalf("loadCatalog() error = %v", err) - } -} - -func TestValidateInputRequiresDigestForPinnedContainer(t *testing.T) { - tagged := input{Kind: "container", Location: "Docker Hub", Ref: "alpine:3.20"} - if err := validateInput(t.TempDir(), "pinned-input", tagged); err == nil || - !strings.Contains(err.Error(), "immutable sha256 digest") { - t.Fatalf("validateInput() tagged pinned container error = %v", err) - } - if err := validateInput(t.TempDir(), "snapshot", tagged); err != nil { - t.Fatalf("validateInput() snapshot tag error = %v", err) - } - digested := tagged - digested.Ref = "alpine@sha256:" + strings.Repeat("a", 64) - if err := validateInput(t.TempDir(), "pinned-input", digested); err != nil { - t.Fatalf("validateInput() digest error = %v", err) - } -} - -func TestValidateArtifactRejectsSymlinkOutsideRepository(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("symlink creation requires elevated privileges on Windows") - } - root := t.TempDir() - data := []byte("outside") - outside := filepath.Join(t.TempDir(), "result.json") - if err := os.WriteFile(outside, data, 0o600); err != nil { - t.Fatal(err) - } - link := filepath.Join(root, "result.json") - if err := os.Symlink(outside, link); err != nil { - t.Fatal(err) - } - sum := sha256.Sum256(data) - err := validateArtifact(root, artifact{ - Path: "result.json", - SHA256: hex.EncodeToString(sum[:]), - }) - if err == nil || !strings.Contains(err.Error(), "resolves outside the repository") { - t.Fatalf("validateArtifact() error = %v", err) - } -} - -func TestResolveCatalogPathHonorsAbsolutePath(t *testing.T) { - root := t.TempDir() - absolute := filepath.Join(t.TempDir(), "cases.json") - if got := resolveCatalogPath(root, absolute); got != absolute { - t.Fatalf("resolveCatalogPath() absolute = %q, want %q", got, absolute) - } - wantRelative := filepath.Join(root, "test", "evidence", "cases.json") - if got := resolveCatalogPath(root, "test/evidence/cases.json"); got != wantRelative { - t.Fatalf("resolveCatalogPath() relative = %q, want %q", got, wantRelative) - } -} - -func TestShellCommandQuotesPatterns(t *testing.T) { - got := shellCommand([]string{"go", "test", "-run", "TestScan$/scan-npm$"}) - want := "go test -run 'TestScan$/scan-npm$'" - if got != want { - t.Fatalf("shellCommand() = %q, want %q", got, want) - } -} diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index 36bf6147..30aaeed5 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -7,6 +7,9 @@ FUZZTIME="${FUZZTIME:-60s}" # transport JSON) moved with the sdk package to the bomly-sdk repository and # run there. targets=( + "github.com/bomly-dev/bomly-cli/internal/assurance FuzzParseCatalog" + "github.com/bomly-dev/bomly-cli/internal/assurance FuzzParseCheckResult" + "github.com/bomly-dev/bomly-cli/internal/assurance FuzzParseGoTestEvents" "github.com/bomly-dev/bomly-cli/internal/config FuzzLoadFile" "github.com/bomly-dev/bomly-cli/internal/detectors/cargo FuzzDepGraphFromCargoLock" "github.com/bomly-dev/bomly-cli/internal/detectors/cocoapods FuzzDepGraphFromPodfileLock" diff --git a/test/assurance/PARSER_FUZZING.md b/test/assurance/PARSER_FUZZING.md index 9f7ddba1..e4dcf036 100644 --- a/test/assurance/PARSER_FUZZING.md +++ b/test/assurance/PARSER_FUZZING.md @@ -24,6 +24,7 @@ the reader inventory, cache behavior, and intentional exclusions. | Other lockfiles and manifests | Cargo, CocoaPods, Composer, Conan, Go list, Mix, NuGet lock and packages.config, Pub, Bundler, SwiftPM | | Workflow manifests | GitHub Actions workflow references | | Matcher evidence | vulnerability consolidation and advisory aliases | +| Release assurance | check-result documents, the assurance catalog, and `go test -json` streams | Seeds include valid minimal documents and malformed/truncated structures. The fuzz engine supplies invalid encodings, deep nesting, duplicate values, diff --git a/test/evidence/README.md b/test/evidence/README.md deleted file mode 100644 index 1c642b0a..00000000 --- a/test/evidence/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Public evidence catalog - -This directory maps Bomly claims to repeatable tests and checked-in evidence. -It is not a second test suite. The catalog points to the smoke tests, focused -unit tests, fixtures, and manually started assurance workflows that already -own each behavior. - -Run the catalog check from the repository root: - -```sh -make evidence -``` - -Show one case and its exact reproduction command: - -```sh -make evidence CASE=graph-npm -``` - -The checker verifies that: - -- every remote Git input has a full commit revision; -- every fixture, workflow, and result file has the recorded SHA-256 checksum; -- every case says what it proves and what it does not prove; -- case IDs are unique and stable. - -## Evidence levels - -- `deterministic` uses checked-in inputs or local services and should produce - the same normalized result. -- `pinned-input` uses a public repository revision. Package-manager tools or - registries can still affect build-tool-backed resolution. -- `snapshot` records the normalized result of an input that can move, such as - a container tag. -- `live-service` combines a pinned project with current advisory data. It is a - dated observation because advisory services change. -- `manual-assurance` starts a GitHub Actions workflow and stores the detailed - run report as a workflow artifact. - -The catalog schema belongs only to repository assurance metadata. It does not -change Bomly's CLI, MCP, SDK, or plugin schemas. From c876acb0c1b39d7413e66d6dd01dd1b2b3022fb2 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 19 Aug 2026 22:08:37 -0400 Subject: [PATCH 02/13] =?UTF-8?q?feat(assurance):=20stage=201=20=E2=80=94?= =?UTF-8?q?=20release=20prerequisites?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs every source-tree check before a version is tagged, and makes each one report into the assurance framework instead of only setting a job status. - smoke.yml, portable-assurance.yml, and fuzz.yml gain workflow_call inputs (ref, assurance, assurance_tag) alongside their existing triggers, and each slice, platform, and repeat loop now ends by writing a check result. - Smoke runs `go test -json` and summarises it with `assurance gotest --echo`, which replays the same output so the logs read as they did before. - The portable workflow's three hand-written GITHUB_STEP_SUMMARY blocks are replaced by `assurance emit --step-summary`: the summary and the published report are now rendered from the same data and cannot disagree. Cross-builds attempt every target instead of stopping at the first failure, so the report shows the whole matrix. - scripts/run-fuzz.sh gains FUZZ_RESULTS_JSONL, which records one line per target and lets the run continue past a failure while still exiting non-zero. - assurance-prerequisites.yml calls those three workflows, validates the catalog, and judges the stage with `assurance verdict`. Fuzz is advisory, so a finding is reported without blocking. - Auto Version runs the stage on the commit it is about to tag and only tags when it passes, with an explicit skip_prerequisites escape hatch that warns. Step summaries now include the catalog's reproduction command for the check. Co-Authored-By: Claude Fable 5 --- .github/workflows/assurance-prerequisites.yml | 167 ++++++++ .github/workflows/auto-version.yml | 28 ++ .github/workflows/fuzz.yml | 71 +++- .github/workflows/portable-assurance.yml | 357 +++++++++--------- .github/workflows/smoke.yml | 56 ++- dev-docs/CI.md | 30 +- internal/assurance/cmd/commands.go | 19 +- internal/assurance/cmd/main.go | 4 +- internal/assurance/render_markdown.go | 29 +- scripts/run-fuzz.sh | 32 +- test/assurance/BENCHMARK_RUNS.md | 18 +- 11 files changed, 610 insertions(+), 201 deletions(-) create mode 100644 .github/workflows/assurance-prerequisites.yml diff --git a/.github/workflows/assurance-prerequisites.yml b/.github/workflows/assurance-prerequisites.yml new file mode 100644 index 00000000..bd0b5138 --- /dev/null +++ b/.github/workflows/assurance-prerequisites.yml @@ -0,0 +1,167 @@ +# Stage 1 of Bomly's release assurance framework: everything that must hold +# before a version is tagged. It runs on the source tree, so a failure here is +# fixed by an ordinary pull request rather than by a broken release. +# +# Auto Version calls this workflow and only tags when it passes. Release +# preflight then looks for a successful run against the commit being released, +# so a hand-made tag cannot skip it. +# +# The checks it collects are declared in docs/assurance/catalog.json. +name: Release prerequisites + +on: + workflow_dispatch: + inputs: + ref: + description: Commit, branch, or tag to check (defaults to this ref). + type: string + required: false + default: "" + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check. + type: string + required: false + default: "" + assurance_tag: + description: Release tag recorded in the check results. + type: string + required: false + default: "" + +permissions: + contents: read + +concurrency: + group: assurance-prerequisites-${{ inputs.ref || github.ref }} + cancel-in-progress: false + +jobs: + smoke: + name: End-to-end scans + uses: ./.github/workflows/smoke.yml + with: + ref: ${{ inputs.ref || github.ref }} + assurance: true + assurance_tag: ${{ inputs.assurance_tag }} + + portable: + name: Platform stability + uses: ./.github/workflows/portable-assurance.yml + with: + ref: ${{ inputs.ref || github.ref }} + assurance: true + assurance_tag: ${{ inputs.assurance_tag }} + + # Fuzzing is advisory: a finding is recorded in the report and does not stop a + # release, so a failure here must not fail this workflow either. The verdict + # job below is what decides the stage. + fuzz: + name: Parser fuzzing + continue-on-error: true + uses: ./.github/workflows/fuzz.yml + with: + ref: ${{ inputs.ref || github.ref }} + fuzztime: 45s + assurance: true + assurance_tag: ${{ inputs.assurance_tag }} + + catalog: + name: Assurance catalog + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Validate the assurance catalog + id: catalog + continue-on-error: true + run: make assurance-catalog + + - name: Record the catalog result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + CATALOG_OUTCOME: ${{ steps.catalog.outcome }} + run: | + set -euo pipefail + exit_code=0 + summary="The assurance catalog is valid and every fixture and expected-result file it names still matches its recorded checksum." + if [ "${CATALOG_OUTCOME}" != "success" ]; then + exit_code=1 + summary="The assurance catalog is invalid or one of the files it names has changed. Open the validation step for the exact entry." + fi + checks="$(grep -c '"stage":' docs/assurance/catalog.json || true)" + go run ./internal/assurance/cmd emit \ + --id catalog-valid --exit-code "${exit_code}" --summary "${summary}" \ + --metric checks="${checks}" --out assurance-results --step-summary + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-catalog + path: assurance-results + if-no-files-found: error + retention-days: 7 + + - name: Fail the job when the catalog is invalid + if: steps.catalog.outcome == 'failure' + run: exit 1 + + verdict: + name: Prerequisites verdict + if: always() + needs: [smoke, portable, fuzz, catalog] + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Collect check results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: assurance-* + merge-multiple: true + path: assurance-results + + - name: Judge the prerequisites stage + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + run: | + go run ./internal/assurance/cmd verdict \ + --results assurance-results --stage prerequisites \ + --tag '${{ inputs.assurance_tag }}' \ + --json assurance-results/stage-prerequisites.json --step-summary + + - name: Upload the stage report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-stage-prerequisites + path: assurance-results + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/auto-version.yml b/.github/workflows/auto-version.yml index e6c7c051..e2d9e207 100644 --- a/.github/workflows/auto-version.yml +++ b/.github/workflows/auto-version.yml @@ -11,6 +11,11 @@ on: - patch - minor - major + skip_prerequisites: + description: Tag without running the release prerequisites stage (emergency use). + required: false + type: boolean + default: false permissions: contents: read @@ -20,7 +25,20 @@ concurrency: cancel-in-progress: false jobs: + # Stage 1 of release assurance, on the commit that is about to be tagged. + # Running it before the tag exists means a stale golden file or an + # intermittent failure is fixed by a normal pull request instead of leaving a + # broken release behind. + prerequisites: + name: Release prerequisites + if: ${{ !inputs.skip_prerequisites }} + uses: ./.github/workflows/assurance-prerequisites.yml + with: + ref: ${{ github.sha }} + bump-tag-and-release: + needs: prerequisites + if: ${{ always() && (needs.prerequisites.result == 'success' || inputs.skip_prerequisites) }} runs-on: ubuntu-latest environment: release @@ -41,6 +59,16 @@ jobs: token: ${{ steps.app-token.outputs.token }} persist-credentials: true + - name: Require the prerequisites stage to have run on this commit + shell: bash + env: + SKIPPED: ${{ inputs.skip_prerequisites }} + run: | + set -euo pipefail + if [[ "${SKIPPED}" == "true" ]]; then + echo "::warning::Release prerequisites were skipped for this tag. The release report will show every prerequisite check as missing." + fi + - name: Require main branch shell: bash run: | diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 4727dfde..444edb6f 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -8,6 +8,28 @@ on: workflow_dispatch: schedule: - cron: "0 8 * * *" + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check out. + type: string + required: false + default: "" + fuzztime: + description: Time budget per fuzz target. + type: string + required: false + default: 2m + assurance: + description: Upload a release assurance check result. + type: boolean + required: false + default: false + assurance_tag: + description: Release tag recorded in the check result. + type: string + required: false + default: "" permissions: contents: read @@ -24,6 +46,7 @@ jobs: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ inputs.ref || github.ref }} persist-credentials: false - name: Set up Go @@ -34,10 +57,50 @@ jobs: cache-dependency-path: go.sum - name: Run native Go fuzz targets - run: make fuzz FUZZTIME=2m + id: fuzz + continue-on-error: true + env: + FUZZ_RESULTS_JSONL: ${{ runner.temp }}/fuzz-details.jsonl + run: make fuzz FUZZTIME=${{ inputs.fuzztime || '2m' }} + + - name: Record the fuzz result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + FUZZ_OUTCOME: ${{ steps.fuzz.outcome }} + FUZZTIME: ${{ inputs.fuzztime || '2m' }} + run: | + set -euo pipefail + details="${RUNNER_TEMP}/fuzz-details.jsonl" + targets=0 + failed=0 + if [ -f "${details}" ]; then + targets="$(wc -l < "${details}" | tr -d ' ')" + failed="$(grep -c '"exit_code":[^0]' "${details}" || true)" + fi + exit_code=0 + summary="${targets} fuzz targets ran for ${FUZZTIME} each against their seed corpus without finding a crash." + if [ "${FUZZ_OUTCOME}" != "success" ]; then + exit_code=1 + summary="${failed} of ${targets} fuzz targets found a failing input. Reproducers are attached to this run." + fi + go run ./internal/assurance/cmd emit \ + --id fuzz --exit-code "${exit_code}" --summary "${summary}" \ + --metric targets="${targets}" --metric targets_failed="${failed}" \ + --details-jsonl "${details}" \ + --out assurance-results --step-summary + + - name: Upload check result + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-fuzz + path: assurance-results + if-no-files-found: error + retention-days: 7 - name: Upload fuzz failures - if: failure() + if: steps.fuzz.outcome == 'failure' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: go-fuzz-failures @@ -45,3 +108,7 @@ jobs: **/testdata/fuzz/** if-no-files-found: ignore retention-days: 7 + + - name: Fail the job when a fuzz target failed + if: steps.fuzz.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/portable-assurance.yml b/.github/workflows/portable-assurance.yml index 3522b404..69d67db0 100644 --- a/.github/workflows/portable-assurance.yml +++ b/.github/workflows/portable-assurance.yml @@ -1,7 +1,31 @@ +# Repeated unit tests across supported platforms and cross-builds of every +# release binary. Runs on its own when a maintainer starts it, and as part of +# the "Release prerequisites" stage before a version is tagged. +# +# Every step ends by writing a release assurance check result; the job summary +# is rendered from that same result, so the summary and the report can never +# disagree. Check ids used here must stay declared in docs/assurance/catalog.json. name: Portable stability assurance on: workflow_dispatch: + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check out. + type: string + required: false + default: "" + assurance: + description: Upload release assurance check results. + type: boolean + required: false + default: false + assurance_tag: + description: Release tag recorded in the check results. + type: string + required: false + default: "" permissions: contents: read @@ -17,6 +41,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -27,17 +53,23 @@ jobs: - name: Repeat portable suite twice id: portable-tests + continue-on-error: true shell: bash run: | - set -euo pipefail + set -uo pipefail completed=0 failed=0 + : > "${RUNNER_TEMP}/portable-details.jsonl" for iteration in 1 2; do echo "portable suite iteration ${iteration}" if go test ./... -count=1; then completed="${iteration}" + printf '{"name":"iteration %s","status":"pass"}\n' "${iteration}" \ + >> "${RUNNER_TEMP}/portable-details.jsonl" else failed="${iteration}" + printf '{"name":"iteration %s","status":"fail"}\n' "${iteration}" \ + >> "${RUNNER_TEMP}/portable-details.jsonl" break fi done @@ -47,10 +79,11 @@ jobs: exit 1 fi - - name: Explain this platform result + - name: Record the platform result if: always() shell: bash env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} TEST_RESULT: ${{ steps.portable-tests.outcome }} COMPLETED: ${{ steps.portable-tests.outputs.completed }} FAILED: ${{ steps.portable-tests.outputs.failed }} @@ -58,35 +91,32 @@ jobs: set -euo pipefail completed="${COMPLETED:-0}" failed="${FAILED:-0}" - if [[ "${TEST_RESULT}" == "success" ]]; then - result="Passed" - icon="✅" - else - result="Failed" - icon="❌" + exit_code=0 + summary="The complete Go test suite passed twice on ${RUNNER_OS}/${RUNNER_ARCH}, with no reused results." + if [[ "${TEST_RESULT}" != "success" ]]; then + exit_code=1 + summary="The complete Go test suite failed on run ${failed} of 2 on ${RUNNER_OS}/${RUNNER_ARCH}. Open the Repeat portable suite twice step for the failing package and test." fi - if [[ "${failed}" == "0" ]]; then - failed_display="—" - else - failed_display="${failed}" - fi - { - echo "## ${icon} ${RUNNER_OS}/${RUNNER_ARCH}: ${result}" - echo - echo "The complete Go test suite was scheduled twice on this platform. Each run used \`-count=1\` so Go did not reuse test results." - echo - echo "| Planned runs | Completed runs | Failed run |" - echo "|---:|---:|---:|" - echo "| 2 | ${completed} | ${failed_display} |" - echo - echo "### Need more detail?" - echo - echo "Open the **Repeat portable suite twice** step to find the failing package and test. To reproduce the same check locally:" - echo - echo '```sh' - echo "go test ./... -count=1" - echo '```' - } >> "${GITHUB_STEP_SUMMARY}" + go run ./internal/assurance/cmd emit \ + --id unit-portable --instance '${{ matrix.os }}' \ + --exit-code "${exit_code}" --summary "${summary}" \ + --metric planned_runs=2 --metric completed_runs="${completed}" \ + --details-jsonl "${RUNNER_TEMP}/portable-details.jsonl" \ + --out assurance-results --step-summary + + - name: Upload check result + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-portable-${{ matrix.os }} + path: assurance-results + if-no-files-found: error + retention-days: 7 + + - name: Fail the job when the suite failed + if: steps.portable-tests.outcome == 'failure' + shell: bash + run: exit 1 linux-stability: name: Linux repeated stability @@ -94,6 +124,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -110,16 +142,20 @@ jobs: - name: Repeat Java suites ten times id: java-tests + continue-on-error: true run: | - set -euo pipefail + set -uo pipefail completed=0 failed=0 + : > "${RUNNER_TEMP}/java-details.jsonl" for iteration in $(seq 1 10); do echo "Java suite iteration ${iteration}" if go test ./internal/detectors/gradle ./internal/detectors/maven ./internal/detectors/sbt -count=1; then completed="${iteration}" + printf '{"name":"iteration %s","status":"pass"}\n' "${iteration}" >> "${RUNNER_TEMP}/java-details.jsonl" else failed="${iteration}" + printf '{"name":"iteration %s","status":"fail"}\n' "${iteration}" >> "${RUNNER_TEMP}/java-details.jsonl" break fi done @@ -129,18 +165,44 @@ jobs: exit 1 fi + - name: Record the Java stability result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + JAVA_RESULT: ${{ steps.java-tests.outcome }} + COMPLETED: ${{ steps.java-tests.outputs.completed }} + FAILED: ${{ steps.java-tests.outputs.failed }} + run: | + set -euo pipefail + completed="${COMPLETED:-0}" + exit_code=0 + summary="The Gradle, Maven, and sbt detector suites passed ten times in a row." + if [[ "${JAVA_RESULT}" != "success" ]]; then + exit_code=1 + summary="The Java detector suites failed on run ${FAILED:-0} of 10, which points at an intermittent failure." + fi + go run ./internal/assurance/cmd emit \ + --id unit-repeat-java --exit-code "${exit_code}" --summary "${summary}" \ + --metric planned_runs=10 --metric completed_runs="${completed}" \ + --details-jsonl "${RUNNER_TEMP}/java-details.jsonl" \ + --out assurance-results --step-summary + - name: Repeat complete suite five times id: complete-tests + continue-on-error: true run: | - set -euo pipefail + set -uo pipefail completed=0 failed=0 + : > "${RUNNER_TEMP}/complete-details.jsonl" for iteration in $(seq 1 5); do echo "complete suite iteration ${iteration}" if go test ./... -count=1; then completed="${iteration}" + printf '{"name":"iteration %s","status":"pass"}\n' "${iteration}" >> "${RUNNER_TEMP}/complete-details.jsonl" else failed="${iteration}" + printf '{"name":"iteration %s","status":"fail"}\n' "${iteration}" >> "${RUNNER_TEMP}/complete-details.jsonl" break fi done @@ -150,12 +212,38 @@ jobs: exit 1 fi + - name: Record the repeated suite result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + COMPLETE_RESULT: ${{ steps.complete-tests.outcome }} + COMPLETED: ${{ steps.complete-tests.outputs.completed }} + FAILED: ${{ steps.complete-tests.outputs.failed }} + run: | + set -euo pipefail + completed="${COMPLETED:-0}" + exit_code=0 + summary="The complete Go test suite passed five more times on Linux." + if [[ "${COMPLETE_RESULT}" != "success" ]]; then + exit_code=1 + summary="The complete Go test suite failed on run ${FAILED:-0} of 5, which points at an intermittent failure." + fi + go run ./internal/assurance/cmd emit \ + --id unit-repeat-full --exit-code "${exit_code}" --summary "${summary}" \ + --metric planned_runs=5 --metric completed_runs="${completed}" \ + --details-jsonl "${RUNNER_TEMP}/complete-details.jsonl" \ + --out assurance-results --step-summary + + # Every target is attempted, even after one fails, so the report shows the + # complete build matrix instead of stopping at the first problem. - name: Cross-build release targets id: release-builds + continue-on-error: true run: | - set -euo pipefail + set -uo pipefail completed=0 - failed="" + planned=0 + : > "${RUNNER_TEMP}/build-details.jsonl" targets=( "linux amd64" "linux arm64" @@ -164,165 +252,78 @@ jobs: "windows amd64" "windows arm64" ) + record() { + printf '{"name":"%s","status":"%s"}\n' "${1}" "${2}" >> "${RUNNER_TEMP}/build-details.jsonl" + } for target in "${targets[@]}"; do read -r target_os target_arch <<<"${target}" suffix="" if [[ "${target_os}" == "windows" ]]; then suffix=".exe" fi - echo "cross-build ${target_os}/${target_arch}" - if GOOS="${target_os}" GOARCH="${target_arch}" CGO_ENABLED=0 \ - go build -o "bin/bomly-${target_os}-${target_arch}${suffix}" ./cmd/bomly; then - completed=$((completed + 1)) - else - failed="${target_os}/${target_arch} full" - break - fi - if GOOS="${target_os}" GOARCH="${target_arch}" CGO_ENABLED=0 \ - go build -tags "bomly_external_syft,bomly_external_grype" \ - -o "bin/bomly-lite-${target_os}-${target_arch}${suffix}" ./cmd/bomly; then - completed=$((completed + 1)) - else - failed="${target_os}/${target_arch} lite" - break - fi + for variant in full lite; do + planned=$((planned + 1)) + echo "cross-build ${target_os}/${target_arch} ${variant}" + status=fail + if [[ "${variant}" == "full" ]]; then + if GOOS="${target_os}" GOARCH="${target_arch}" CGO_ENABLED=0 \ + go build -o "bin/bomly-${target_os}-${target_arch}${suffix}" ./cmd/bomly; then + status=pass + fi + else + if GOOS="${target_os}" GOARCH="${target_arch}" CGO_ENABLED=0 \ + go build -tags "bomly_external_syft,bomly_external_grype" \ + -o "bin/bomly-lite-${target_os}-${target_arch}${suffix}" ./cmd/bomly; then + status=pass + fi + fi + if [[ "${status}" == "pass" ]]; then + completed=$((completed + 1)) + fi + record "${target_os}/${target_arch} ${variant}" "${status}" + done done echo "completed=${completed}" >> "${GITHUB_OUTPUT}" - echo "failed=${failed}" >> "${GITHUB_OUTPUT}" - if [[ -n "${failed}" ]]; then + echo "planned=${planned}" >> "${GITHUB_OUTPUT}" + if [[ "${completed}" != "${planned}" ]]; then exit 1 fi - - name: Explain Linux stability result + - name: Record the cross-build result if: always() env: - JAVA_RESULT: ${{ steps.java-tests.outcome }} - JAVA_COMPLETED: ${{ steps.java-tests.outputs.completed }} - JAVA_FAILED: ${{ steps.java-tests.outputs.failed }} - COMPLETE_RESULT: ${{ steps.complete-tests.outcome }} - COMPLETE_COMPLETED: ${{ steps.complete-tests.outputs.completed }} - COMPLETE_FAILED: ${{ steps.complete-tests.outputs.failed }} + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} BUILD_RESULT: ${{ steps.release-builds.outcome }} - BUILD_COMPLETED: ${{ steps.release-builds.outputs.completed }} - BUILD_FAILED: ${{ steps.release-builds.outputs.failed }} + COMPLETED: ${{ steps.release-builds.outputs.completed }} + PLANNED: ${{ steps.release-builds.outputs.planned }} run: | set -euo pipefail - if [[ "${JAVA_RESULT}" == "success" && "${COMPLETE_RESULT}" == "success" && "${BUILD_RESULT}" == "success" ]]; then - result="Passed" - icon="✅" - else - result="Failed" - icon="❌" + completed="${COMPLETED:-0}" + planned="${PLANNED:-12}" + exit_code=0 + summary="All ${planned} release binaries built: full and lite for Linux, macOS, and Windows on amd64 and arm64." + if [[ "${BUILD_RESULT}" != "success" ]]; then + exit_code=1 + summary="${completed} of ${planned} release binaries built. The failed targets are listed below." fi - show_result() { - case "${1}" in - success) printf "Passed" ;; - failure) printf "Failed" ;; - cancelled) printf "Cancelled" ;; - skipped) printf "Skipped" ;; - *) printf "Not run" ;; - esac - } - show_completed() { - case "${1}" in - success|failure) printf "%s" "${2:-0}" ;; - *) printf "—" ;; - esac - } - show_failure() { - case "${1}" in - success) printf "—" ;; - failure) - if [[ -z "${2:-}" || "${2}" == "0" ]]; then - printf "See step log" - else - printf "%s" "${2}" - fi - ;; - cancelled) printf "Cancelled" ;; - skipped) printf "Skipped" ;; - *) printf "Not run" ;; - esac - } - { - echo "## ${icon} Linux stability and release builds: ${result}" - echo - echo "This job repeats the areas most likely to reveal intermittent failures, then checks that every release binary can be built." - echo - echo "| Check | Result | Planned | Completed | Failed at |" - echo "|---|---|---:|---:|---|" - echo "| Java detector tests | $(show_result "${JAVA_RESULT}") | 10 runs | $(show_completed "${JAVA_RESULT}" "${JAVA_COMPLETED:-}") | $(show_failure "${JAVA_RESULT}" "${JAVA_FAILED:-}") |" - echo "| Complete test suite | $(show_result "${COMPLETE_RESULT}") | 5 runs | $(show_completed "${COMPLETE_RESULT}" "${COMPLETE_COMPLETED:-}") | $(show_failure "${COMPLETE_RESULT}" "${COMPLETE_FAILED:-}") |" - echo "| Release binaries | $(show_result "${BUILD_RESULT}") | 12 builds | $(show_completed "${BUILD_RESULT}" "${BUILD_COMPLETED:-}") | $(show_failure "${BUILD_RESULT}" "${BUILD_FAILED:-}") |" - echo - echo "The release check builds full and lite binaries for Linux, macOS, and Windows on amd64 and arm64." - echo - echo "### Need more detail?" - echo - echo "Open the failed step for the test name or release target. Reproduce the test checks with:" - echo - echo '```sh' - echo "go test ./internal/detectors/gradle ./internal/detectors/maven ./internal/detectors/sbt -count=1" - echo "go test ./... -count=1" - echo '```' - echo - echo "For a failed release build, use the operating system, architecture, and variant shown in the table with the build command from **Cross-build release targets**." - } >> "${GITHUB_STEP_SUMMARY}" + go run ./internal/assurance/cmd emit \ + --id cross-build --exit-code "${exit_code}" --summary "${summary}" \ + --metric builds_planned="${planned}" --metric builds_completed="${completed}" \ + --details-jsonl "${RUNNER_TEMP}/build-details.jsonl" \ + --out assurance-results --step-summary - summary: - name: Explain overall result - if: always() - needs: - - portable - - linux-stability - runs-on: ubuntu-latest - steps: - - name: Summarize workflow - env: - PORTABLE_RESULT: ${{ needs.portable.result }} - STABILITY_RESULT: ${{ needs.linux-stability.result }} - run: | - set -euo pipefail - if [[ "${PORTABLE_RESULT}" == "success" && "${STABILITY_RESULT}" == "success" ]]; then - result="Passed" - icon="✅" - else - result="Failed" - icon="❌" - fi - if [[ "${PORTABLE_RESULT}" == "success" ]]; then - portable_label="Passed" - else - portable_label="Failed" - fi - if [[ "${STABILITY_RESULT}" == "success" ]]; then - stability_label="Passed" - else - stability_label="Failed" - fi - { - echo "# ${icon} Portable stability assurance: ${result}" - echo - echo "This workflow checked that the test suite is repeatable across supported development platforms and that release binaries build for every target." - echo - echo "| Area | Result | What ran |" - echo "|---|---|---|" - echo "| Linux, macOS, and Windows | ${portable_label} | Complete suite twice on each platform |" - echo "| Linux stability and release builds | ${stability_label} | Java tests 10 times, complete suite 5 times, and 12 release builds |" - echo - echo "Each job has its own summary with completed counts and the exact point of failure." - echo - echo "## Need more detail?" - echo - echo "Show only failed logs:" - echo - echo '```sh' - echo "gh run view ${GITHUB_RUN_ID} --log-failed" - echo '```' - echo - echo "After fixing a problem, rerun only the failed jobs:" - echo - echo '```sh' - echo "gh run rerun ${GITHUB_RUN_ID} --failed" - echo '```' - } >> "${GITHUB_STEP_SUMMARY}" + - name: Upload check results + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-linux-stability + path: assurance-results + if-no-files-found: error + retention-days: 7 + + - name: Fail the job when any check failed + if: >- + steps.java-tests.outcome == 'failure' || + steps.complete-tests.outcome == 'failure' || + steps.release-builds.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 4eacfc56..46db33bf 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -7,6 +7,26 @@ on: schedule: - cron: "0 8 * * *" workflow_dispatch: + # Called by "Release prerequisites" (assurance-prerequisites.yml). The + # `assurance` input turns on per-slice check results, which that stage + # collects to decide whether the code is fit to be tagged. + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check out. + type: string + required: false + default: "" + assurance: + description: Emit release assurance check results for each slice. + type: boolean + required: false + default: false + assurance_tag: + description: Release tag recorded in the check results. + type: string + required: false + default: "" permissions: contents: read @@ -26,6 +46,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -122,6 +144,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -254,7 +278,37 @@ jobs: env: BOMLY_SMOKE_BINARY: ${{ runner.temp }}/bomly-bin/bomly BOMLY_SMOKE_LITE_BINARY: ${{ runner.temp }}/bomly-bin/bomly-lite - run: go test -tags smoke ./test/smoke/ -v -count=1 -timeout 15m -run '${{ matrix.slice.run }}' + # -json instead of -v so the run can be summarised into an assurance + # check result; `assurance gotest --echo` below replays the same + # output, so the logs read exactly as they did before. + run: | + set -o pipefail + go test -tags smoke ./test/smoke/ -json -count=1 -timeout 15m \ + -run '${{ matrix.slice.run }}' 2>&1 | tee "${RUNNER_TEMP}/smoke.jsonl" > /dev/null + + - name: Summarise this slice + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} + SMOKE_OUTCOME: ${{ steps.smoke_tests.outcome }} + run: | + exit_code=0 + if [ "${SMOKE_OUTCOME}" != "success" ]; then + exit_code=1 + fi + go run ./internal/assurance/cmd gotest \ + --id smoke --instance '${{ matrix.slice.name }}' \ + --input "${RUNNER_TEMP}/smoke.jsonl" --exit-code "${exit_code}" \ + --echo --out assurance-results + + - name: Upload check result + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-smoke-${{ matrix.slice.name }} + path: assurance-results + if-no-files-found: error + retention-days: 7 - name: Fail job if smoke tests failed if: steps.smoke_tests.outcome == 'failure' diff --git a/dev-docs/CI.md b/dev-docs/CI.md index 29d9045b..46047cb8 100644 --- a/dev-docs/CI.md +++ b/dev-docs/CI.md @@ -9,15 +9,22 @@ its own minimal CI in its own repository. | Workflow | Trigger | Purpose | |-------------------------|----------------------------------|----------------------------------------------------------------------| | `CI` | Pull requests, pushes to `main` | Lint, `go test ./...`, full and lite builds, npm wrapper tests, go.mod/go.sum tidy-drift and no-`replace` checks | -| `Smoke` | Pull requests (labeled), nightly | End-to-end smoke slices driving the built binary against pinned public repositories | +| `Release prerequisites` | Called by `Auto Version`, manual dispatch | Stage 1 of release assurance: calls `Smoke`, `Portable stability assurance`, and `Fuzz`, validates the assurance catalog, and judges the stage | +| `Smoke` | Merge queue, nightly, dispatch, `workflow_call` | End-to-end smoke slices driving the built binary against pinned public repositories | | `Update Smoke Goldens` | Manual dispatch | Regenerates smoke golden files per slice and opens a PR with the drift | -| `Fuzz` | Nightly schedule, manual dispatch | Native Go fuzzing over the `scripts/run-fuzz.sh` target list; uploads minimized failures as artifacts | +| `Portable stability assurance` | Manual dispatch, `workflow_call` | Repeated unit tests on Linux, macOS, and Windows plus cross-builds of every release binary | +| `Fuzz` | Nightly schedule, dispatch, `workflow_call` | Native Go fuzzing over the `scripts/run-fuzz.sh` target list; uploads minimized failures as artifacts | | `CodeQL` | Pull requests, pushes, schedule | Static analysis for Go and JavaScript | -| `SBOM Interoperability` | Schedule, manual dispatch | Binary-driven SBOM export/ingest checks against third-party tools | -| `Auto Version` | Pushes to `main` | Computes the next semver from the squash-commit prefix and creates a draft release tag | -| `Release` | Release tags | GoReleaser build/publish with signed checksums and SLSA provenance | +| `SBOM interoperability assurance` | Weekly schedule, dispatch, `workflow_call` | Validates generated SPDX and CycloneDX documents with checksum-pinned official tools | +| `Auto Version` | Manual dispatch | Runs `Release prerequisites`, then bumps the version, tags, and starts `Release` | +| `Release` | Release tags | GoReleaser build/publish with signed checksums and SLSA provenance, plus stage 2 of release assurance | +| `Release assessment` | `release: published`, dispatch | Stage 3 of release assurance against the published binaries; writes the per-release report | | `Scorecard`, `Dependency Review`, `Bomly Guard` | Various | Supply-chain posture checks and dogfooding | +Workflows that other workflows call (`Smoke`, `Portable stability assurance`, +`Fuzz`, `SBOM interoperability assurance`) keep their own triggers as well, so +each can still be run on its own. + Workflows use `actions/setup-go` with the Go version read from `go.mod`, so `gofmt`, compilation, and test behavior stay aligned between local development and GitHub Actions. Every workflow declares a top-level `permissions:` block @@ -44,12 +51,25 @@ target there. When Go minimizes a failure into `go test`, then commit the reproducer only after confirming it is a useful regression seed. +## Release assurance + +Every quality check belongs to a release stage and reports a +`bomly.assurance-check/v1` result, which the framework merges into a per-release +report. `dev-docs/RELEASE_ASSURANCE.md` describes the contracts and how to add a +check; `docs/assurance/catalog.json` is the list of checks; `docs/ASSURANCE.md` +is the public explanation. + +Workflow steps never hand-write result JSON. They call +`go run ./internal/assurance/cmd` (`emit`, `gotest`, `convert`, or +`verify-release`), which also renders the step summary from the same data. + ## Local parity - `make fmt` rewrites tracked Go files with `gofmt` - `make fmt-check` fails when tracked Go files are not formatted - `make lint` runs the repository-pinned `golangci-lint` - `make install-hooks` points Git at the `.githooks/` pre-commit hook +- `make assurance-catalog` validates the assurance catalog the way CI does ## Releases diff --git a/internal/assurance/cmd/commands.go b/internal/assurance/cmd/commands.go index bed30091..da98be47 100644 --- a/internal/assurance/cmd/commands.go +++ b/internal/assurance/cmd/commands.go @@ -114,7 +114,7 @@ func runEmit(args []string) error { } result.Links = append(result.Links, assurance.Link{Label: label, URL: url}) } - return writeResult(*out, result, *stepSummary) + return writeResult(*out, result, *stepSummary, reproduceFor(catalogCtx.catalog, result.ID)) } func parseDetail(entry string) (assurance.Detail, error) { @@ -235,7 +235,18 @@ func runGoTest(args []string) error { if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { return err } - return writeResult(*out, summary.ToCheckResult(result, *exitCode), *stepSummary) + return writeResult(*out, summary.ToCheckResult(result, *exitCode), *stepSummary, reproduceFor(catalogCtx.catalog, result.ID)) +} + +// reproduceFor returns the catalog's local reproduction command for a check. +func reproduceFor(catalog *assurance.Catalog, id string) [][]string { + if catalog == nil { + return nil + } + if check, found := catalog.Check(id); found { + return check.Reproduce + } + return nil } func writerOrNil(file *os.File) *os.File { @@ -289,7 +300,7 @@ func runConvert(args []string) error { if err != nil { return err } - return writeResult(*out, converted, *stepSummary) + return writeResult(*out, converted, *stepSummary, reproduceFor(catalogCtx.catalog, converted.ID)) } // -------------------------------------------------------- verify-release --- @@ -326,7 +337,7 @@ func runVerifyRelease(args []string) error { if result.Status == assurance.StatusFail { failures++ } - return writeResult(*out, result, *stepSummary) + return writeResult(*out, result, *stepSummary, reproduceFor(catalogCtx.catalog, result.ID)) } if *scope == "full" { diff --git a/internal/assurance/cmd/main.go b/internal/assurance/cmd/main.go index d833a1bc..40e381b5 100644 --- a/internal/assurance/cmd/main.go +++ b/internal/assurance/cmd/main.go @@ -200,7 +200,7 @@ func applyCatalog(result *assurance.CheckResult, catalog *assurance.Catalog, sta return nil } -func writeResult(outDir string, result assurance.CheckResult, stepSummary bool) error { +func writeResult(outDir string, result assurance.CheckResult, stepSummary bool, reproduce [][]string) error { if err := result.Validate(); err != nil { return err } @@ -217,7 +217,7 @@ func writeResult(outDir string, result assurance.CheckResult, stepSummary bool) } fmt.Printf("%s %s: %s\n", assurance.StatusIcon(result.Status), result.Key(), result.Summary) if stepSummary { - if err := appendStepSummary(assurance.RenderResultMarkdown(result)); err != nil { + if err := appendStepSummary(assurance.RenderResultMarkdown(result, reproduce)); err != nil { return err } } diff --git a/internal/assurance/render_markdown.go b/internal/assurance/render_markdown.go index 30c199cf..2bc55f5c 100644 --- a/internal/assurance/render_markdown.go +++ b/internal/assurance/render_markdown.go @@ -154,7 +154,9 @@ func renderTrends(trends Trends) string { } // RenderResultMarkdown renders one check result for a workflow step summary. -func RenderResultMarkdown(result CheckResult) string { +// reproduce is the catalog's local reproduction command for the check, shown so +// a reader can run the same thing without hunting for it. +func RenderResultMarkdown(result CheckResult, reproduce [][]string) string { var out strings.Builder title := result.ID if result.Instance != "" { @@ -176,9 +178,34 @@ func RenderResultMarkdown(result CheckResult) string { } out.WriteString("\n") } + if len(reproduce) > 0 { + out.WriteString("Reproduce locally:\n\n```sh\n") + for _, command := range reproduce { + out.WriteString(shellCommand(command) + "\n") + } + out.WriteString("```\n\n") + } return out.String() } +// shellCommand renders one argument list as a copyable shell command. +func shellCommand(command []string) string { + quoted := make([]string, len(command)) + for index, argument := range command { + if argument != "" && strings.IndexFunc(argument, func(r rune) bool { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + return false + } + return !strings.ContainsRune("@%_+=:,./-", r) + }) == -1 { + quoted[index] = argument + continue + } + quoted[index] = "'" + strings.ReplaceAll(argument, "'", "'\"'\"'") + "'" + } + return strings.Join(quoted, " ") +} + func markdownCell(value string) string { cleaned := strings.ReplaceAll(strings.TrimSpace(value), "|", "\\|") cleaned = strings.ReplaceAll(cleaned, "\r", " ") diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index 30aaeed5..203659d0 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -3,6 +3,12 @@ set -euo pipefail FUZZTIME="${FUZZTIME:-60s}" +# When FUZZ_RESULTS_JSONL is set, every target is attempted and one JSON line +# per target is written to that file, so the release assurance framework can +# record the whole run instead of stopping at the first failure. The script +# still exits non-zero when any target failed. +FUZZ_RESULTS_JSONL="${FUZZ_RESULTS_JSONL:-}" + # The SDK's own fuzz targets (package URL canonicalization, graph/registry # transport JSON) moved with the sdk package to the bomly-sdk repository and # run there. @@ -39,9 +45,33 @@ targets=( "github.com/bomly-dev/bomly-cli/internal/plugin FuzzPluginPathSanitizers" ) +if [ -n "${FUZZ_RESULTS_JSONL}" ]; then + : > "${FUZZ_RESULTS_JSONL}" +fi + +failures=0 for target in "${targets[@]}"; do package="${target%% *}" fuzz="${target#* }" echo "==> go test ${package} -run=^$ -fuzz=^${fuzz}$ -fuzztime=${FUZZTIME}" - go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" + started="$(date -u +%s)" + status=0 + if [ -n "${FUZZ_RESULTS_JSONL}" ]; then + go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" || status=$? + else + go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" + fi + if [ "${status}" -ne 0 ]; then + failures=$((failures + 1)) + fi + if [ -n "${FUZZ_RESULTS_JSONL}" ]; then + printf '{"name":"%s %s","exit_code":%s,"duration_s":%s}\n' \ + "${package##*/}" "${fuzz}" "${status}" "$(( $(date -u +%s) - started ))" \ + >> "${FUZZ_RESULTS_JSONL}" + fi done + +if [ "${failures}" -ne 0 ]; then + echo "${failures} fuzz target(s) failed" >&2 + exit 1 +fi diff --git a/test/assurance/BENCHMARK_RUNS.md b/test/assurance/BENCHMARK_RUNS.md index 10ee3702..55969c78 100644 --- a/test/assurance/BENCHMARK_RUNS.md +++ b/test/assurance/BENCHMARK_RUNS.md @@ -1,6 +1,7 @@ # Measuring speed and stability -`make benchmark-samples` measures the same offline Bomly scan ten times: +`make benchmark-samples` (the `perfrun` tool under `internal/assurance/`) +measures the same offline Bomly scan ten times: - five runs start with an empty cache; - five runs share a cache, like repeated scans normally do. @@ -38,8 +39,9 @@ checksum. This comparison format is named ## Checking supported systems -The `Portable stability assurance` workflow runs only when someone starts it -from GitHub Actions. It: +The `Portable stability assurance` workflow runs when someone starts it from +GitHub Actions, and as part of the `Release prerequisites` stage before a +version is tagged. It: - runs the Go unit tests twice on Linux, macOS, and Windows; - runs the Java-related unit tests ten times to catch intermittent failures; @@ -57,10 +59,12 @@ when investigating platform-specific or intermittent failures. ## Reading a portable run -Open the workflow run's **Summary** page first. The overall section explains -what ran and whether each area passed. Each platform also has a short section -showing how many test runs completed and which run failed, if any. The Linux -section does the same for repeated tests and release builds. +Open the workflow run's **Summary** page first. Each check writes its own +section there — the repeated suites per platform, the repeated Java suites, the +repeated complete suite, and the cross-build matrix — with the number of runs +planned and completed and the exact point of failure. Those sections are +rendered from the same check results the release assurance report is built +from, so the summary and the published report always agree. If something fails, open the named job and failed step for the test or build output. To show only failed logs with the GitHub CLI, run: From 7a3b821f5816d3f933bf9550b4a98bcae05e7d0a Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 19 Aug 2026 22:12:31 -0400 Subject: [PATCH 03/13] =?UTF-8?q?feat(assurance):=20stage=202=20=E2=80=94?= =?UTF-8?q?=20pre-release=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Published GitHub releases are immutable, so the draft release is the last point where a bad artifact can be stopped. This adds that gate. release.yml: - preflight now refuses to build a release whose commit (or its parent, which is where the version-bump commit lands) has no successful "Release prerequisites" run, with an actionable message and a RELEASE_ASSURANCE_ENFORCE escape hatch. - verify-draft downloads the draft's assets by release id on Linux, macOS, and Windows and checks asset completeness, SHA256SUMS on each platform, the cosign signature, SLSA provenance, and that both released binaries start and report the tag version. - gate judges the stage from the check results rather than job statuses, and publish now depends on it, so a failed gate leaves an unpublished draft instead of a bad release. sbom-interoperability.yml drops ~60 lines of inline download/validate shell in favour of internal/assurance/sbominterop, which already pins the validators by checksum and records every command in a manifest; it gains a workflow_call interface and can validate the binary a release actually shipped. notify-landing-yank.yml now ignores draft releases: deleting an abandoned draft fires the same event as yanking a live version, and nothing was published for that version. Docs: the release checklist covers the new failure paths, and the SBOM interoperability note is corrected (CycloneDX 1.7, weekly plus post-release). Co-Authored-By: Claude Fable 5 --- .github/workflows/notify-landing-yank.yml | 14 +- .github/workflows/release.yml | 286 +++++++++++++++++++- .github/workflows/sbom-interoperability.yml | 132 +++++---- dev-docs/RELEASE_CHECKLIST.md | 17 +- docs/assurance/catalog.json | 16 +- internal/assurance/cmd/commands.go | 5 +- test/assurance/SBOM_INTEROPERABILITY.md | 34 ++- 7 files changed, 430 insertions(+), 74 deletions(-) diff --git a/.github/workflows/notify-landing-yank.yml b/.github/workflows/notify-landing-yank.yml index 3b44fcd2..dd5272f7 100644 --- a/.github/workflows/notify-landing-yank.yml +++ b/.github/workflows/notify-landing-yank.yml @@ -32,7 +32,7 @@ jobs: permission-contents: write - name: Trigger docs sync on landing page (published) - if: github.event.action == 'published' + if: github.event.action == 'published' && github.event.release.draft == false env: GH_TOKEN: ${{ steps.landing-token.outputs.token }} TAG: ${{ github.event.release.tag_name }} @@ -45,8 +45,14 @@ jobs: {"event_type":"bomly-release","client_payload":{"version":"${TAG}","publishedAt":"${PUBLISHED_AT}"}} EOF + # A failed pre-release assurance gate leaves a draft release that a + # maintainer deletes. Deleting a draft fires the same `deleted` event as + # yanking a live release, so both yank paths ignore drafts: nothing was + # ever published for that version. - name: Trigger docs removal on landing page (yanked) - if: github.event.action == 'deleted' || github.event.action == 'unpublished' + if: >- + (github.event.action == 'deleted' || github.event.action == 'unpublished') && + github.event.release.draft == false env: GH_TOKEN: ${{ steps.landing-token.outputs.token }} TAG: ${{ github.event.release.tag_name }} @@ -59,7 +65,9 @@ jobs: EOF winget-yank: - if: github.event.action == 'deleted' || github.event.action == 'unpublished' + if: >- + (github.event.action == 'deleted' || github.event.action == 'unpublished') && + github.event.release.draft == false runs-on: ubuntu-latest timeout-minutes: 15 steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 35936efe..6c689479 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,10 @@ concurrency: jobs: preflight: + # Fail within seconds on the two things that strand a release: an expired + # publishing token, and a tag whose commit never passed the release + # prerequisites stage. + # # Fail within seconds if WINGET_GITHUB_TOKEN is expired or revoked, # instead of 14 minutes into GoReleaser — after it has already created # the draft release and opened the Homebrew/Scoop PRs — which strands @@ -25,6 +29,35 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: + # Stage 1 of release assurance runs on the source tree before a tag + # exists (see assurance-prerequisites.yml). Auto Version runs it and then + # commits the version bump, so the tagged commit is usually the child of + # the commit that was checked: accept either. + - name: Require a passing release prerequisites run + if: vars.RELEASE_ASSURANCE_ENFORCE != 'false' + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + + tag_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${RELEASE_TAG}" --jq '.sha')" + parent_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${RELEASE_TAG}" --jq '.parents[0].sha // ""')" + + for candidate in "${tag_sha}" "${parent_sha}"; do + [ -n "${candidate}" ] || continue + conclusion="$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/assurance-prerequisites.yml/runs?head_sha=${candidate}&per_page=20" \ + --jq '[.workflow_runs[] | select(.conclusion == "success")] | length')" + if [ "${conclusion}" != "0" ]; then + echo "Release prerequisites passed for ${candidate}." + exit 0 + fi + done + + echo "::error::No successful \"Release prerequisites\" run exists for ${RELEASE_TAG} (${tag_sha}) or its parent. Run it first with: gh workflow run assurance-prerequisites.yml -f ref=${tag_sha}. Set the repository variable RELEASE_ASSURANCE_ENFORCE=false to release without it." + exit 1 + - name: Check winget token validity and expiry env: WINGET_GITHUB_TOKEN: ${{ secrets.WINGET_GITHUB_TOKEN }} @@ -168,9 +201,260 @@ jobs: base64-subjects: ${{ needs.release.outputs.hashes }} upload-assets: false - publish: + # Stage 2 of release assurance: the release exists as a draft, so its files + # can be checked before anyone can download them. Published releases are + # immutable, so this is the last point where a bad artifact can be stopped. + verify-draft: needs: [release, provenance] if: startsWith(github.ref, 'refs/tags/') + name: Verify draft release (${{ matrix.platform.os }}) + runs-on: ${{ matrix.platform.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + platform: + - os: ubuntu-latest + scope: full + - os: macos-latest + scope: native + - os: windows-latest + scope: native + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Mint release token + # Draft release assets are not publicly downloadable, so this job needs + # the same app token the publish job uses to read them by release id. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: release-token + with: + client-id: ${{ vars.RELEASE_BOT_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} + owner: bomly-dev + repositories: bomly-cli + permission-contents: read + + - name: Download draft release assets + shell: bash + env: + GH_TOKEN: ${{ steps.release-token.outputs.token }} + RELEASE_TAG: ${{ github.ref_name }} + SCOPE: ${{ matrix.platform.scope }} + run: | + set -euo pipefail + version="${RELEASE_TAG#v}" + mkdir -p "${RUNNER_TEMP}/assets" + + release_id="$(gh api "repos/bomly-dev/bomly-cli/releases?per_page=100" | \ + jq -r --arg tag "$RELEASE_TAG" '.[] | select(.tag_name == $tag and .draft == true) | .id' | head -n1)" + if [ -z "${release_id}" ]; then + echo "::error::No draft release found for tag ${RELEASE_TAG}" + exit 1 + fi + + case "${RUNNER_OS}" in + Linux) native_os=linux ;; + macOS) native_os=darwin ;; + Windows) native_os=windows ;; + *) echo "::error::Unsupported runner ${RUNNER_OS}"; exit 1 ;; + esac + case "${RUNNER_ARCH}" in + X64) native_arch=amd64 ;; + ARM64) native_arch=arm64 ;; + *) echo "::error::Unsupported runner architecture ${RUNNER_ARCH}"; exit 1 ;; + esac + extension="tar.gz" + if [ "${native_os}" = "windows" ]; then + extension="zip" + fi + + gh api "repos/bomly-dev/bomly-cli/releases/${release_id}/assets?per_page=100" \ + --jq '.[] | [.id, .name] | @tsv' > "${RUNNER_TEMP}/assets.tsv" + + while IFS=$'\t' read -r asset_id asset_name; do + case "${SCOPE}" in + full) ;; + native) + case "${asset_name}" in + SHA256SUMS|"bomly_${version}_${native_os}_${native_arch}.${extension}"|"bomly-lite_${version}_${native_os}_${native_arch}.${extension}") ;; + *) continue ;; + esac + ;; + esac + echo "downloading ${asset_name}" + curl -fsSL -H "Authorization: Bearer ${GH_TOKEN}" -H "Accept: application/octet-stream" \ + -o "${RUNNER_TEMP}/assets/${asset_name}" \ + "https://api.github.com/repos/bomly-dev/bomly-cli/releases/assets/${asset_id}" + done < "${RUNNER_TEMP}/assets.tsv" + + - name: Verify release files and binaries + id: verify + continue-on-error: true + shell: bash + env: + BOMLY_ASSURANCE_TAG: ${{ github.ref_name }} + run: | + go run ./internal/assurance/cmd verify-release \ + --dir "${RUNNER_TEMP}/assets" --version "${GITHUB_REF_NAME#v}" \ + --scope '${{ matrix.platform.scope }}' \ + --out assurance-results --step-summary + + - name: Install cosign + if: matrix.platform.scope == 'full' + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: Verify the checksum signature + id: signature + if: matrix.platform.scope == 'full' + continue-on-error: true + shell: bash + run: | + set -euo pipefail + cosign verify-blob "${RUNNER_TEMP}/assets/SHA256SUMS" \ + --bundle "${RUNNER_TEMP}/assets/SHA256SUMS.sigstore.json" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + --certificate-identity-regexp '^https://github.com/bomly-dev/bomly-cli/\.github/workflows/release\.yml@refs/tags/' + + - name: Record the signature result + if: matrix.platform.scope == 'full' && always() + shell: bash + env: + BOMLY_ASSURANCE_TAG: ${{ github.ref_name }} + SIGNATURE_OUTCOME: ${{ steps.signature.outcome }} + run: | + set -euo pipefail + exit_code=0 + summary="The checksum list carries a valid Sigstore signature issued to this repository's release workflow." + if [ "${SIGNATURE_OUTCOME}" != "success" ]; then + exit_code=1 + summary="cosign could not verify the signature over SHA256SUMS. Nothing about this release's files can be trusted until that is explained." + fi + go run ./internal/assurance/cmd emit \ + --id release-signature --exit-code "${exit_code}" --summary "${summary}" \ + --out assurance-results --step-summary + + - name: Install slsa-verifier + if: matrix.platform.scope == 'full' + uses: slsa-framework/slsa-verifier/actions/installer@ea584f4502babc6f60d9bc799dbbb13c1caa9ee6 # v2.7.1 + + - name: Verify build provenance + id: provenance + if: matrix.platform.scope == 'full' + continue-on-error: true + shell: bash + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#v}" + for archive in "bomly_${version}_linux_amd64.tar.gz" "bomly-lite_${version}_linux_amd64.tar.gz"; do + slsa-verifier verify-artifact "${RUNNER_TEMP}/assets/${archive}" \ + --provenance-path "${RUNNER_TEMP}/assets/multiple.intoto.jsonl" \ + --source-uri github.com/bomly-dev/bomly-cli \ + --source-tag "${GITHUB_REF_NAME}" + done + + - name: Record the provenance result + if: matrix.platform.scope == 'full' && always() + shell: bash + env: + BOMLY_ASSURANCE_TAG: ${{ github.ref_name }} + PROVENANCE_OUTCOME: ${{ steps.provenance.outcome }} + run: | + set -euo pipefail + exit_code=0 + summary="SLSA build provenance verifies the linux/amd64 archives against this repository and tag." + if [ "${PROVENANCE_OUTCOME}" != "success" ]; then + exit_code=1 + summary="slsa-verifier could not verify the build provenance attached to this release." + fi + go run ./internal/assurance/cmd emit \ + --id release-provenance --exit-code "${exit_code}" --summary "${summary}" \ + --out assurance-results --step-summary + + - name: Upload check results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-release-${{ matrix.platform.os }} + path: assurance-results + if-no-files-found: error + retention-days: 30 + + # The gate that decides whether the draft may be published. It reads the + # check results rather than job statuses, so an advisory check can fail + # without stopping a release and a missing result can never look like a pass. + gate: + needs: verify-draft + if: always() && startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + verdict: ${{ steps.verdict.outputs.verdict }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Collect check results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: assurance-release-* + merge-multiple: true + path: assurance-results + + - name: Judge the pre-release stage + id: verdict + continue-on-error: true + env: + BOMLY_ASSURANCE_TAG: ${{ github.ref_name }} + run: | + set -o pipefail + if go run ./internal/assurance/cmd verdict \ + --results assurance-results --stage pre-release \ + --tag "${GITHUB_REF_NAME}" --step-summary; then + echo "verdict=pass" >> "${GITHUB_OUTPUT}" + else + echo "verdict=fail" >> "${GITHUB_OUTPUT}" + fi + + - name: Upload the stage report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-stage-pre-release + path: assurance-results + if-no-files-found: error + retention-days: 30 + + - name: Fail when the stage did not pass + if: steps.verdict.outputs.verdict != 'pass' && vars.RELEASE_ASSURANCE_ENFORCE != 'false' + run: | + echo "::error::The pre-release stage did not pass. The draft release was left unpublished; fix the cause, delete the tag and the draft, then tag again." + exit 1 + + publish: + needs: [release, provenance, gate] + if: | + startsWith(github.ref, 'refs/tags/') && + (needs.gate.outputs.verdict == 'pass' || vars.RELEASE_ASSURANCE_ENFORCE == 'false') runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/sbom-interoperability.yml b/.github/workflows/sbom-interoperability.yml index 7d3af05e..5910bd96 100644 --- a/.github/workflows/sbom-interoperability.yml +++ b/.github/workflows/sbom-interoperability.yml @@ -1,21 +1,39 @@ +# Generates canonical SBOMs and validates them with the official SPDX and +# CycloneDX tools, pinned by checksum. Runs weekly on its own, and as part of +# the post-release assessment against the binary the release actually shipped. +# +# The download, generation, and validation logic lives in +# internal/assurance/sbominterop, which records every command, exit code, and +# checksum in a bomly.sbom-assurance-run/v1 manifest. This workflow only +# arranges the inputs and turns that manifest into a check result. name: SBOM interoperability assurance on: schedule: - cron: "41 5 * * 1" workflow_dispatch: + workflow_call: + inputs: + ref: + description: Commit, branch, or tag to check out. + type: string + required: false + default: "" + release_tag: + description: Validate the binary published for this release tag instead of a freshly built one. + type: string + required: false + default: "" + assurance: + description: Upload a release assurance check result. + type: boolean + required: false + default: false permissions: contents: read env: - SPDX_TOOLS_VERSION: "2.0.7" - SPDX_TOOLS_URL: https://github.com/spdx/tools-java/releases/download/v2.0.7/tools-java-2.0.7.zip - SPDX_TOOLS_SHA256: 2dc63c3399c5178058b1be8a3de6f13b9f24981cd86c4292ef98f4a7e90de36d - SPDX_TOOLS_JAR: tools-java-2.0.7-jar-with-dependencies.jar - CDX_CLI_VERSION: "0.32.0" - CDX_CLI_URL: https://github.com/CycloneDX/cyclonedx-cli/releases/download/v0.32.0/cyclonedx-linux-x64 - CDX_CLI_SHA256: 454879e6a4a405c8a13bff49b8982adcb0596f3019b26b0811c66e4d7f0783e1 FIXTURE_SBOM: test/smoke/testdata/sboms/go.spdx.json OUTPUT_DIR: sbom-assurance-artifacts @@ -23,9 +41,13 @@ jobs: validate: name: Validate generated SBOMs runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -41,58 +63,62 @@ jobs: java-version: "21" - name: Build the Bomly binary + if: inputs.release_tag == '' run: make build-full - - name: Download checksum-pinned validators - run: | - set -euo pipefail - mkdir -p "${OUTPUT_DIR}/validators" - curl -fsSL --retry 3 -o "${OUTPUT_DIR}/validators/tools-java.zip" "${SPDX_TOOLS_URL}" - echo "${SPDX_TOOLS_SHA256} ${OUTPUT_DIR}/validators/tools-java.zip" | sha256sum -c - - unzip -j -o "${OUTPUT_DIR}/validators/tools-java.zip" "${SPDX_TOOLS_JAR}" -d "${OUTPUT_DIR}/validators" - curl -fsSL --retry 3 -o "${OUTPUT_DIR}/validators/cyclonedx-cli" "${CDX_CLI_URL}" - echo "${CDX_CLI_SHA256} ${OUTPUT_DIR}/validators/cyclonedx-cli" | sha256sum -c - - chmod +x "${OUTPUT_DIR}/validators/cyclonedx-cli" - - - name: Generate SBOMs with the built binary - run: | - set -euo pipefail - ./bin/bomly scan --sbom --path "${FIXTURE_SBOM}" --detectors sbom \ - --format spdx > "${OUTPUT_DIR}/bomly.spdx.json" - ./bin/bomly scan --sbom --path "${FIXTURE_SBOM}" --detectors sbom \ - --format cyclonedx > "${OUTPUT_DIR}/bomly.cdx.json" - sha256sum "${OUTPUT_DIR}/bomly.spdx.json" "${OUTPUT_DIR}/bomly.cdx.json" \ - | tee "${OUTPUT_DIR}/checksums.txt" - - - name: Validate SPDX 2.3 output (spdx/tools-java) + # The post-release assessment validates what users downloaded, not a + # rebuild of it, so the released archive is verified against the published + # checksum list before it is used. + - name: Download the released binary + if: inputs.release_tag != '' + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.release_tag }} run: | set -euo pipefail - java -jar "${OUTPUT_DIR}/validators/${SPDX_TOOLS_JAR}" Verify \ - "${OUTPUT_DIR}/bomly.spdx.json" 2>&1 | tee "${OUTPUT_DIR}/spdx-validation.log" + version="${RELEASE_TAG#v}" + archive="bomly_${version}_linux_amd64.tar.gz" + mkdir -p "${RUNNER_TEMP}/release" bin + gh release download "${RELEASE_TAG}" --repo bomly-dev/bomly-cli \ + --pattern "${archive}" --pattern SHA256SUMS --dir "${RUNNER_TEMP}/release" + (cd "${RUNNER_TEMP}/release" && sha256sum --check --ignore-missing SHA256SUMS) + tar -xzf "${RUNNER_TEMP}/release/${archive}" -C "${RUNNER_TEMP}/release" bomly + install -m 0755 "${RUNNER_TEMP}/release/bomly" bin/bomly + ./bin/bomly version - - name: Validate CycloneDX 1.7 output (cyclonedx-cli) + - name: Generate and validate SBOMs + id: sbom + continue-on-error: true run: | - set -euo pipefail - "${OUTPUT_DIR}/validators/cyclonedx-cli" validate \ - --input-file "${OUTPUT_DIR}/bomly.cdx.json" \ - --input-format json --input-version v1_7 --fail-on-errors \ - 2>&1 | tee "${OUTPUT_DIR}/cyclonedx-validation.log" + go run ./internal/assurance/sbominterop \ + -bomly ./bin/bomly -input "${FIXTURE_SBOM}" -output "${OUTPUT_DIR}" - - name: Summarize the result + - name: Record the interoperability result if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ inputs.release_tag }} run: | set -euo pipefail - { - printf '# SBOM interoperability assurance\n\n' - printf 'The built binary generated SPDX 2.3 and CycloneDX 1.7 SBOMs from a pinned fixture, and the official validators checked them.\n\n' - printf -- '- **Revision:** `%s`\n' "${GITHUB_SHA}" - printf -- '- **Validators:** spdx/tools-java `%s`, cyclonedx-cli `%s` (both checksum-pinned)\n\n' "${SPDX_TOOLS_VERSION}" "${CDX_CLI_VERSION}" - if [[ -f "${OUTPUT_DIR}/checksums.txt" ]]; then - printf '## Generated file checksums\n\n```\n' - cat "${OUTPUT_DIR}/checksums.txt" - printf '```\n' - fi - } >> "${GITHUB_STEP_SUMMARY}" + manifest="${OUTPUT_DIR}/run-manifest.json" + if [ -f "${manifest}" ]; then + go run ./internal/assurance/cmd convert sbom-assurance \ + --id sbom-interoperability --input "${manifest}" \ + --out assurance-results --step-summary + else + go run ./internal/assurance/cmd emit \ + --id sbom-interoperability --status fail \ + --summary "The SBOM interoperability run stopped before it could write a manifest. Open the generate step for the cause." \ + --out assurance-results --step-summary + fi + + - name: Upload check result + if: always() && inputs.assurance + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-sbom-interoperability + path: assurance-results + if-no-files-found: error + retention-days: 30 - name: Upload assurance evidence if: always() @@ -102,8 +128,10 @@ jobs: path: | sbom-assurance-artifacts/bomly.spdx.json sbom-assurance-artifacts/bomly.cdx.json - sbom-assurance-artifacts/checksums.txt - sbom-assurance-artifacts/spdx-validation.log - sbom-assurance-artifacts/cyclonedx-validation.log + sbom-assurance-artifacts/run-manifest.json if-no-files-found: warn retention-days: 14 + + - name: Fail the job when validation failed + if: steps.sbom.outcome == 'failure' + run: exit 1 diff --git a/dev-docs/RELEASE_CHECKLIST.md b/dev-docs/RELEASE_CHECKLIST.md index 3586c8f2..8901cc8d 100644 --- a/dev-docs/RELEASE_CHECKLIST.md +++ b/dev-docs/RELEASE_CHECKLIST.md @@ -5,14 +5,16 @@ Use this checklist when publishing a tagged Bomly CLI release. ## Before tagging - Confirm `main` is green for required checks. -- Run the smoke workflow (or confirm its latest scheduled smoke result is healthy). -- Confirm `cmd/bomly/main.go` contains the intended version after the `Auto Version` workflow. - Confirm release publishing credentials are configured in GitHub Actions. +- `Auto Version` runs the `Release prerequisites` stage (smoke, platform stability, cross-builds, fuzz, catalog) on the commit it is about to tag and refuses to tag when it fails, so there is no separate smoke run to start by hand. To pre-flight without tagging: `gh workflow run assurance-prerequisites.yml -f ref=main`. +- If the stage fails, fix the cause on `main` — for stale golden files, run `Update Smoke Goldens` and merge its PR — then start `Auto Version` again. No tag and no release exist yet. ## Release workflow - Run `Auto Version` from `main`, choosing `patch`, `minor`, or `major`. -- Wait for `Release` to finish. +- Wait for `Release` to finish. It builds the draft release, then runs the final pre-release checks against the draft (asset completeness, checksums on three platforms, cosign signature, SLSA provenance, and the released binaries) and only publishes when they pass. +- If the pre-release gate fails, nothing is published. Fix the cause, delete the tag and the draft release, then tag again. Deleting a draft does not trigger the yanking workflow. +- Confirm `cmd/bomly/main.go` contains the intended version. - Review the published GitHub release: - `bomly` archives exist for Linux, macOS, and Windows on `amd64` and `arm64`. - `bomly-lite` archives exist for the same platforms. @@ -21,9 +23,16 @@ Use this checklist when publishing a tagged Bomly CLI release. - Homebrew, Scoop, and WinGet manifest PRs were opened or updated. - The landing-page sync PR updates `/install.sh` and `/install.ps1` from this tag when those scripts changed. +## After publishing + +- `Release assessment` starts automatically once the release is published: it runs the install scripts on all three operating systems, re-downloads the public files, scans real projects with the released binary, validates its SBOM output with the official tools, and records repeated-scan timings. +- Read the report it publishes at [bomly.dev/assurance](https://bomly.dev/assurance), or the JSON it commits to `docs/assurance/reports/.json`. +- If it opens a `Release assurance: ` issue, triage it: the release is already live, so the fix is a follow-up release, not an edit to this one. + ## Verification -Run the checks against the published release tag. Replace `VERSION` in the examples below with the actual release tag, such as `v0.2.0`. +The assessment runs these automatically. Run them by hand when investigating a +report, replacing `VERSION` with the release tag, such as `v0.2.0`. ```bash gh release download VERSION --pattern SHA256SUMS --pattern 'bomly_VERSION_linux_amd64.tar.gz' diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json index b83548f9..ce769c4f 100644 --- a/docs/assurance/catalog.json +++ b/docs/assurance/catalog.json @@ -324,11 +324,25 @@ "area": "release-integrity", "stage": "pre-release", "level": "gate", - "description": "Hashes every downloaded release file and compares it with the published SHA256SUMS list.", + "description": "Hashes every release file each platform downloads and compares it with the published SHA256SUMS list.", "source": { "workflow": "release.yml", "job": "verify-draft" }, + "expected_instances": [ + { + "name": "linux-amd64", + "platform": "linux" + }, + { + "name": "darwin-arm64", + "platform": "darwin" + }, + { + "name": "windows-amd64", + "platform": "windows" + } + ], "reproduce": [ [ "sha256sum", diff --git a/internal/assurance/cmd/commands.go b/internal/assurance/cmd/commands.go index da98be47..9187a3a1 100644 --- a/internal/assurance/cmd/commands.go +++ b/internal/assurance/cmd/commands.go @@ -387,7 +387,8 @@ func runVerifyRelease(args []string) error { if err != nil { return err } - checksums := baseResult("release-checksums", "") + platform := runtime.GOOS + "-" + runtime.GOARCH + checksums := baseResult("release-checksums", platform) checksums.FinishedAt = time.Now().UTC().Format(time.RFC3339) if err := applyCatalog(&checksums, catalogCtx.catalog, "", ""); err != nil { return err @@ -435,7 +436,7 @@ func runVerifyRelease(args []string) error { if err != nil { return err } - binaries := baseResult("release-binaries", runtime.GOOS+"-"+runtime.GOARCH) + binaries := baseResult("release-binaries", platform) binaries.FinishedAt = time.Now().UTC().Format(time.RFC3339) if err := applyCatalog(&binaries, catalogCtx.catalog, "", ""); err != nil { return err diff --git a/test/assurance/SBOM_INTEROPERABILITY.md b/test/assurance/SBOM_INTEROPERABILITY.md index b265a88b..0c115872 100644 --- a/test/assurance/SBOM_INTEROPERABILITY.md +++ b/test/assurance/SBOM_INTEROPERABILITY.md @@ -5,27 +5,39 @@ read the SBOM files that Bomly creates. This catches compatibility problems that Bomly's own tests might miss. The workflow uses the same checked-in sample input each time. Bomly creates an -SPDX 2.3 file and a CycloneDX 1.6 file from that input. The workflow then asks +SPDX 2.3 file and a CycloneDX 1.7 file from that input. The workflow then asks the official SPDX and CycloneDX validators to check those files. -This workflow runs only when someone starts it from GitHub Actions. It is kept -separate from normal tests because it downloads the validators and takes +It runs three ways: on a weekly schedule, when someone starts it from GitHub +Actions, and as part of the post-release assessment. In the post-release run it +downloads the binary the release actually shipped, verifies it against the +published checksum list, and validates the SBOMs that binary produces. It is +kept separate from normal tests because it downloads the validators and takes longer to run. Bomly never downloads or installs these tools during normal CLI use. +The download, generation, and validation steps live in +`internal/assurance/sbominterop`, so the same run can be reproduced locally: + +```sh +make build-full +go run ./internal/assurance/sbominterop -bomly ./bin/bomly +``` + ## Reading the result Open the workflow run's **Summary** page first. It shows: - whether validation passed; -- the version and result of each validator; -- the size and checksum of each generated file; -- any messages returned by the validators. - -If validation fails, open the **Generate and validate canonical SBOMs** step -to see its full output. The summary also provides a command that downloads the -saved evidence. The downloaded `run-manifest.json` records every command, exit -code, duration, validator message, version, and checksum. +- the version and checksum of each validator; +- the result of each command that ran; +- the size and checksum of each generated file. + +If validation fails, open the **Generate and validate SBOMs** step to see its +full output. The saved evidence is attached to the run as an artifact. The +downloaded `run-manifest.json` records every command, exit code, duration, +validator message, version, and checksum, and is also what the release +assurance report is built from. ## What the workflow saves From 7030f220a0f39cdc000ecd09341eb5083b350c57 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 19 Aug 2026 22:16:34 -0400 Subject: [PATCH 04/13] =?UTF-8?q?feat(assurance):=20stage=203=20=E2=80=94?= =?UTF-8?q?=20post-release=20assessment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exhaustive pass, run against the binaries users actually download, and the step that publishes the per-release report. assurance-assessment.yml runs on `release: published` (and on dispatch for any tag) and checks: - the published install scripts on Linux, macOS, and Windows; - every published file downloaded unauthenticated from its public URL and checked against SHA256SUMS; - scan, diff, and explain driven by the released binary against the same golden files the source tree is held to; - SBOM interoperability, with the released binary's output validated by the official SPDX and CycloneDX tools; - repeated cold and warm scan timings (advisory). The report job then merges these results with the prerequisites and pre-release results pulled from their own runs, builds the per-release report against the catalog as it stood at that tag, commits it to main under docs/assurance/ (releases are immutable, so it cannot be a release asset), tells the landing page a report is available, and opens, updates, or closes a single "Release assurance: " tracking issue. Checks whose results cannot be found stay missing rather than being assumed to have passed. A new test feeds the repository catalog a synthetic passing result for every check and instance it declares, so an evidence claim pointing at an instance no check can report is caught here instead of in a published report. Co-Authored-By: Claude Fable 5 --- .github/workflows/assurance-assessment.yml | 611 +++++++++++++++++++++ dev-docs/RELEASE_ASSURANCE.md | 16 +- internal/assurance/catalog_test.go | 65 +++ 3 files changed, 691 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/assurance-assessment.yml diff --git a/.github/workflows/assurance-assessment.yml b/.github/workflows/assurance-assessment.yml new file mode 100644 index 00000000..72fec030 --- /dev/null +++ b/.github/workflows/assurance-assessment.yml @@ -0,0 +1,611 @@ +# Stage 3 of Bomly's release assurance framework: the exhaustive pass, run +# against the binaries users actually download. It cannot change the release — +# published releases are immutable — so its job is to measure and to publish an +# honest report, and to open a tracking issue when something is wrong. +# +# The report it writes (docs/assurance/reports/.json) plus the index beside +# it are the only data source for bomly.dev/assurance. +name: Release assessment + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: Release tag to assess, such as v0.24.0. + type: string + required: true + +permissions: + contents: read + +concurrency: + group: release-assessment-${{ github.event.release.tag_name || inputs.tag }} + cancel-in-progress: false + +jobs: + resolve: + name: Resolve the release + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + tag: ${{ steps.release.outputs.tag }} + version: ${{ steps.release.outputs.version }} + commit: ${{ steps.release.outputs.commit }} + url: ${{ steps.release.outputs.url }} + published_at: ${{ steps.release.outputs.published_at }} + steps: + - name: Look up the release + id: release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name || inputs.tag }} + run: | + set -euo pipefail + release_json="$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}")" + published_at="$(echo "${release_json}" | jq -r '.published_at // ""')" + url="$(echo "${release_json}" | jq -r '.html_url')" + draft="$(echo "${release_json}" | jq -r '.draft')" + if [ "${draft}" = "true" ]; then + echo "::error::${TAG} is still a draft. The assessment describes published releases." + exit 1 + fi + commit="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${TAG}" --jq '.sha')" + { + echo "tag=${TAG}" + echo "version=${TAG#v}" + echo "commit=${commit}" + echo "url=${url}" + echo "published_at=${published_at}" + } >> "${GITHUB_OUTPUT}" + + install-scripts: + name: Install script (${{ matrix.platform.name }}) + needs: resolve + strategy: + fail-fast: false + matrix: + platform: + - name: ubuntu + os: ubuntu-latest + - name: macos + os: macos-latest + - name: windows + os: windows-latest + runs-on: ${{ matrix.platform.os }} + timeout-minutes: 20 + steps: + - name: Check out the released source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.tag }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Run the published install script + id: install + continue-on-error: true + shell: bash + env: + BOMLY_VERSION: ${{ needs.resolve.outputs.tag }} + BOMLY_INSTALL_DIR: ${{ runner.temp }}/bomly-install + run: | + set -euo pipefail + mkdir -p "${BOMLY_INSTALL_DIR}" + if [ "${RUNNER_OS}" = "Windows" ]; then + pwsh -NoProfile -File scripts/install.ps1 \ + -Version "${BOMLY_VERSION}" -InstallDir "${BOMLY_INSTALL_DIR}" + "${BOMLY_INSTALL_DIR}/bomly.exe" version | tee "${RUNNER_TEMP}/version.txt" + else + sh scripts/install.sh + "${BOMLY_INSTALL_DIR}/bomly" version | tee "${RUNNER_TEMP}/version.txt" + fi + + - name: Record the install result + if: always() + shell: bash + env: + BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} + INSTALL_OUTCOME: ${{ steps.install.outcome }} + VERSION: ${{ needs.resolve.outputs.version }} + run: | + set -euo pipefail + exit_code=0 + reported="$(head -n 1 "${RUNNER_TEMP}/version.txt" 2>/dev/null || echo "")" + summary="The published install script installed Bomly ${VERSION} on ${RUNNER_OS} and the binary reported: ${reported}" + if [ "${INSTALL_OUTCOME}" != "success" ]; then + exit_code=1 + summary="The published install script failed on ${RUNNER_OS}. Open the install step for the exact command that failed." + elif ! printf '%s' "${reported}" | grep -q "${VERSION}"; then + exit_code=1 + summary="The install script ran on ${RUNNER_OS} but the installed binary reported '${reported}' instead of ${VERSION}." + fi + go run ./internal/assurance/cmd emit \ + --id install-script --instance '${{ matrix.platform.name }}' \ + --exit-code "${exit_code}" --summary "${summary}" \ + --out assurance-results --step-summary + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-install-${{ matrix.platform.name }} + path: assurance-results + if-no-files-found: error + retention-days: 30 + + public-download: + name: Public download + needs: resolve + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out the released source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.tag }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + # Unauthenticated, from the public URL, exactly as a user would. + - name: Download every published file + id: download + continue-on-error: true + env: + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/assets" + base="https://github.com/${GITHUB_REPOSITORY}/releases/download/${TAG}" + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/SHA256SUMS" "${base}/SHA256SUMS" + while read -r _ name; do + name="${name#\*}" + echo "downloading ${name}" + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/${name}" "${base}/${name}" + done < "${RUNNER_TEMP}/assets/SHA256SUMS" + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/SHA256SUMS.sigstore.json" "${base}/SHA256SUMS.sigstore.json" + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/multiple.intoto.jsonl" "${base}/multiple.intoto.jsonl" + + - name: Verify what a user would download + id: verify + continue-on-error: true + env: + BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} + run: | + go run ./internal/assurance/cmd verify-release \ + --dir "${RUNNER_TEMP}/assets" --version '${{ needs.resolve.outputs.version }}' \ + --scope full --out "${RUNNER_TEMP}/release-results" + + - name: Record the public download result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} + DOWNLOAD_OUTCOME: ${{ steps.download.outcome }} + VERIFY_OUTCOME: ${{ steps.verify.outcome }} + run: | + set -euo pipefail + exit_code=0 + count="$(find "${RUNNER_TEMP}/assets" -type f | wc -l | tr -d ' ')" + summary="All ${count} published files downloaded from their public URLs and match the published checksum list." + if [ "${DOWNLOAD_OUTCOME}" != "success" ]; then + exit_code=1 + summary="At least one published file could not be downloaded from its public URL." + elif [ "${VERIFY_OUTCOME}" != "success" ]; then + exit_code=1 + summary="The published files downloaded, but ${count} of them did not match the published checksum list." + fi + go run ./internal/assurance/cmd emit \ + --id public-download --exit-code "${exit_code}" --summary "${summary}" \ + --metric files="${count}" --out assurance-results --step-summary + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-public-download + path: assurance-results + if-no-files-found: error + retention-days: 30 + + released-scan: + name: Released binary scan (${{ matrix.slice.name }}) + needs: resolve + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + slice: + - name: go + run: 'TestScan$/scan-go$|TestDiff/diff-go|TestExplain/explain-go' + - name: node + run: 'TestScan$/scan-npm$' + - name: sbom + run: 'TestScan$/scan-sbom' + steps: + - name: Check out the released source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.tag }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Download the released binaries + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.resolve.outputs.tag }} + VERSION: ${{ needs.resolve.outputs.version }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/release" + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" --dir "${RUNNER_TEMP}/release" \ + --pattern "bomly_${VERSION}_linux_amd64.tar.gz" \ + --pattern "bomly-lite_${VERSION}_linux_amd64.tar.gz" \ + --pattern SHA256SUMS + (cd "${RUNNER_TEMP}/release" && sha256sum --check --ignore-missing SHA256SUMS) + tar -xzf "${RUNNER_TEMP}/release/bomly_${VERSION}_linux_amd64.tar.gz" -C "${RUNNER_TEMP}/release" bomly + tar -xzf "${RUNNER_TEMP}/release/bomly-lite_${VERSION}_linux_amd64.tar.gz" -C "${RUNNER_TEMP}/release" bomly-lite + chmod +x "${RUNNER_TEMP}/release/bomly" "${RUNNER_TEMP}/release/bomly-lite" + + # The same golden files the source tree is checked against, driven by the + # binary the release shipped. + - name: Scan real projects with the released binary + id: scan + continue-on-error: true + env: + BOMLY_SMOKE_BINARY: ${{ runner.temp }}/release/bomly + BOMLY_SMOKE_LITE_BINARY: ${{ runner.temp }}/release/bomly-lite + run: | + set -o pipefail + go test -tags smoke ./test/smoke/ -json -count=1 -timeout 30m \ + -run '${{ matrix.slice.run }}' 2>&1 | tee "${RUNNER_TEMP}/scan.jsonl" > /dev/null + + - name: Record the scan result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} + SCAN_OUTCOME: ${{ steps.scan.outcome }} + run: | + exit_code=0 + if [ "${SCAN_OUTCOME}" != "success" ]; then + exit_code=1 + fi + go run ./internal/assurance/cmd gotest \ + --id released-scan --instance '${{ matrix.slice.name }}' \ + --input "${RUNNER_TEMP}/scan.jsonl" --exit-code "${exit_code}" \ + --echo --out assurance-results --step-summary + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-released-scan-${{ matrix.slice.name }} + path: assurance-results + if-no-files-found: error + retention-days: 30 + + sbom: + name: SBOM interoperability + needs: resolve + uses: ./.github/workflows/sbom-interoperability.yml + with: + ref: ${{ needs.resolve.outputs.tag }} + release_tag: ${{ needs.resolve.outputs.tag }} + assurance: true + + perf-samples: + name: Repeated scan timing + needs: resolve + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out the released source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.tag }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Download the released lite binary + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.resolve.outputs.tag }} + VERSION: ${{ needs.resolve.outputs.version }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/release" bin + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" --dir "${RUNNER_TEMP}/release" \ + --pattern "bomly-lite_${VERSION}_linux_amd64.tar.gz" --pattern SHA256SUMS + (cd "${RUNNER_TEMP}/release" && sha256sum --check --ignore-missing SHA256SUMS) + tar -xzf "${RUNNER_TEMP}/release/bomly-lite_${VERSION}_linux_amd64.tar.gz" \ + -C "${RUNNER_TEMP}/release" bomly-lite + install -m 0755 "${RUNNER_TEMP}/release/bomly-lite" bin/bomly-lite + + - name: Measure repeated scans + id: perf + continue-on-error: true + run: | + go run ./internal/assurance/perfrun -output .benchmark-runs/performance \ + -case canonical-sbom-scan -samples 5 -network-state offline -- \ + ./bin/bomly-lite scan --sbom --path test/smoke/testdata/sboms/go.spdx.json \ + --detectors sbom --format json + + - name: Record the timing result + if: always() + env: + BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + manifest=".benchmark-runs/performance/run-manifest.json" + if [ -f "${manifest}" ]; then + go run ./internal/assurance/cmd convert benchmark-run \ + --id perf-samples --input "${manifest}" \ + --out assurance-results --step-summary + else + go run ./internal/assurance/cmd emit \ + --id perf-samples --status fail \ + --summary "The performance run stopped before it could write a manifest." \ + --out assurance-results --step-summary + fi + + - name: Upload check result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: assurance-perf-samples + path: assurance-results + if-no-files-found: error + retention-days: 30 + + report: + name: Publish the release report + if: always() + needs: [resolve, install-scripts, public-download, released-scan, sbom, perf-samples] + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Mint release bot token + # Commits the report to main and manages the tracking issue. The same + # app already pushes the version-bump commits. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: app-token + with: + client-id: ${{ vars.RELEASE_BOT_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} + owner: bomly-dev + repositories: | + bomly-cli + bomly-landing-page + permission-contents: write + permission-issues: write + + - name: Check out main + # The report is committed to main, and the report tooling comes from + # main so a fix can be applied to an old release's report. The catalog, + # though, must be the one the checks actually ran against — see below. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + persist-credentials: true + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Collect this stage's check results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: assurance-* + merge-multiple: true + path: assurance-results + + # Stages 1 and 2 ran in other workflow runs. Their results are pulled in + # so the report covers the whole release, and anything that cannot be + # found stays absent — the report then shows those checks as missing + # rather than pretending they passed. + - name: Collect the earlier stages' check results + continue-on-error: true + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + TAG: ${{ needs.resolve.outputs.tag }} + COMMIT: ${{ needs.resolve.outputs.commit }} + run: | + set -uo pipefail + parent="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${COMMIT}" --jq '.parents[0].sha // ""')" + + prerequisites_run="" + for candidate in "${COMMIT}" "${parent}"; do + [ -n "${candidate}" ] || continue + prerequisites_run="$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/assurance-prerequisites.yml/runs?head_sha=${candidate}&per_page=20" \ + --jq '[.workflow_runs[] | select(.conclusion == "success")][0].id // ""')" + [ -n "${prerequisites_run}" ] && break + done + if [ -n "${prerequisites_run}" ]; then + gh run download "${prerequisites_run}" --repo "${GITHUB_REPOSITORY}" \ + --name assurance-stage-prerequisites --dir assurance-results || true + echo "prerequisites_run_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${prerequisites_run}" >> "${GITHUB_ENV}" + fi + + release_run="$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/release.yml/runs?per_page=50" \ + --jq --arg tag "${TAG}" '[.workflow_runs[] | select(.head_branch == $tag)][0].id // ""')" + if [ -n "${release_run}" ]; then + gh run download "${release_run}" --repo "${GITHUB_REPOSITORY}" \ + --name assurance-stage-pre-release --dir assurance-results || true + echo "pre_release_run_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${release_run}" >> "${GITHUB_ENV}" + fi + + # A release is judged against the catalog that existed when it was made, + # so a check added later is not counted as missing from an older release. + - name: Take the catalog from the released tag + env: + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + git fetch --quiet origin "refs/tags/${TAG}:refs/tags/${TAG}" || true + if git show "${TAG}:docs/assurance/catalog.json" > "${RUNNER_TEMP}/catalog.json" 2>/dev/null; then + echo "catalog=${RUNNER_TEMP}/catalog.json" >> "${GITHUB_ENV}" + else + echo "${TAG} predates the assurance catalog; using the one on main." + echo "catalog=docs/assurance/catalog.json" >> "${GITHUB_ENV}" + fi + + - name: Build the release report + id: report + continue-on-error: true + env: + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -o pipefail + go run ./internal/assurance/cmd report \ + --results assurance-results \ + --catalog "${catalog}" --out docs/assurance \ + --tag "${TAG}" \ + --commit '${{ needs.resolve.outputs.commit }}' \ + --url '${{ needs.resolve.outputs.url }}' \ + --published-at '${{ needs.resolve.outputs.published_at }}' \ + --prerequisites-run "${prerequisites_run_url:-}" \ + --pre-release-run "${pre_release_run_url:-}" \ + --assessment-run "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --summary-out "${RUNNER_TEMP}/assurance-summary.md" \ + --step-summary --allow-unknown + echo "verdict=$(jq -r '.verdict.overall' "docs/assurance/reports/${TAG}.json")" >> "${GITHUB_OUTPUT}" + + - name: Commit the report to main + if: steps.report.outcome == 'success' + env: + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + git config user.name "${{ steps.app-token.outputs.app-slug }}[bot]" + git config user.email "${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com" + git add "docs/assurance/reports/${TAG}.json" docs/assurance/index.json + if git diff --cached --quiet; then + echo "The report is already committed and unchanged." + exit 0 + fi + git commit -m "docs(assurance): release report for ${TAG} [skip ci]" + # main moves while this runs; rebase onto whatever landed and retry. + for attempt in 1 2 3 4 5; do + if git push origin HEAD:main; then + exit 0 + fi + echo "push attempt ${attempt} failed; rebasing onto origin/main" + git fetch origin main + git rebase origin/main + done + echo "::error::Could not push the assurance report to main after five attempts." + exit 1 + + - name: Tell the landing page a report is available + if: steps.report.outcome == 'success' + continue-on-error: true + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + gh api repos/bomly-dev/bomly-landing-page/dispatches \ + --method POST \ + --input - </dev/null 2>&1 || true + + existing="$(gh issue list --repo "${GITHUB_REPOSITORY}" --state all \ + --label release-assurance --search "\"${title}\" in:title" \ + --json number,title,state --jq \ + ".[] | select(.title == \"${title}\") | \"\(.number) \(.state)\"" | head -n1)" + number="${existing%% *}" + state="${existing##* }" + + body="${RUNNER_TEMP}/assurance-summary.md" + if [ ! -f "${body}" ]; then + printf '# %s\n\nThe assessment could not build a report for this release. Open the run for the cause: %s/%s/actions/runs/%s\n' \ + "${title}" "${GITHUB_SERVER_URL}" "${GITHUB_REPOSITORY}" "${GITHUB_RUN_ID}" > "${body}" + fi + + if [ "${REPORT_OUTCOME}" = "success" ] && [ "${VERDICT}" = "pass" ]; then + if [ -n "${number}" ] && [ "${state}" = "OPEN" ]; then + gh issue comment "${number}" --repo "${GITHUB_REPOSITORY}" \ + --body "A later assessment run of ${TAG} passed every check. Closing." + gh issue close "${number}" --repo "${GITHUB_REPOSITORY}" + fi + exit 0 + fi + + if [ -n "${number}" ]; then + gh issue edit "${number}" --repo "${GITHUB_REPOSITORY}" --body-file "${body}" + if [ "${state}" != "OPEN" ]; then + gh issue reopen "${number}" --repo "${GITHUB_REPOSITORY}" + fi + gh issue comment "${number}" --repo "${GITHUB_REPOSITORY}" \ + --body "The assessment ran again and still reports problems. See the updated description." + else + gh issue create --repo "${GITHUB_REPOSITORY}" --title "${title}" \ + --label release-assurance --body-file "${body}" + fi + + - name: Fail when the release did not pass + if: always() && (steps.report.outcome != 'success' || steps.report.outputs.verdict != 'pass') + run: | + echo "::error::The post-release assessment did not pass. The release is already public; follow the tracking issue." + exit 1 diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index 287badf0..8489219d 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -89,7 +89,21 @@ go run ./internal/assurance/cmd report --results assurance-results \ `docs/assurance/index.json`, prints a markdown summary, and compares the release against the previous one listed in the index. It exits with code 3 when a result arrives for a check the catalog does not declare, so a renamed check cannot -silently disappear from the report. +silently disappear from the report (`--allow-unknown` downgrades that to a note, +which the assessment uses because the matching declared check already shows as +missing). + +The assessment judges a release against **the catalog as it was at that tag**, +not the one on `main`, so a check added later is not counted as missing from an +older release. The report tooling itself comes from `main`, which is what makes +`gh workflow run assurance-assessment.yml -f tag=` able to re-render an +old release's report after a generator fix. + +The report is committed to `main` under `docs/assurance/`, because published +GitHub releases are immutable and the assessment runs after publication. The +commit uses the release app token, is marked `[skip ci]`, and retries onto +`origin/main` if the branch moved. A `bomly-assurance-report` repository +dispatch then tells the landing page a new report is available. ## Adding a check diff --git a/internal/assurance/catalog_test.go b/internal/assurance/catalog_test.go index 0b2987cc..eae43274 100644 --- a/internal/assurance/catalog_test.go +++ b/internal/assurance/catalog_test.go @@ -185,3 +185,68 @@ func mustJSON(t *testing.T, value any) []byte { } return data } + +// TestRepositoryCatalogProducesACompleteReport feeds the catalog a synthetic +// passing result for every check and instance it declares. It guards the wiring +// between the two: an evidence claim pointing at an instance no check can ever +// report, or a check whose instances cannot all be satisfied, shows up here +// instead of as a permanently "missing" entry in a published release report. +func TestRepositoryCatalogProducesACompleteReport(t *testing.T) { + catalog := repositoryCatalog(t) + var results []CheckResult + for _, check := range catalog.Checks { + instances := []string{""} + if len(check.ExpectedInstances) > 0 { + instances = nil + for _, instance := range check.ExpectedInstances { + instances = append(instances, instance.Name) + } + } + for _, instance := range instances { + results = append(results, CheckResult{ + SchemaVersion: CheckSchema, ID: check.ID, Instance: instance, + Stage: check.Stage, Level: check.Level, Status: StatusPass, + Summary: "Synthetic passing result.", + }) + } + } + report := BuildReport(catalog, results, BuildOptions{ + Release: Release{Tag: "v0.0.0", Version: "0.0.0"}, + IncludeEvidence: true, + }) + if report.Verdict.Overall != StatusPass { + t.Fatalf("verdict = %s, want pass; missing=%v gates=%v", + report.Verdict.Overall, report.Verdict.MissingChecks, report.Verdict.GatesFailed) + } + if len(report.Unknown) != 0 { + t.Fatalf("unknown results = %+v", report.Unknown) + } + if len(report.Checks) != len(catalog.Checks) { + t.Fatalf("reported %d checks, catalog declares %d", len(report.Checks), len(catalog.Checks)) + } + if len(report.Evidence) != len(catalog.Evidence) { + t.Fatalf("reported %d evidence claims, catalog declares %d", len(report.Evidence), len(catalog.Evidence)) + } + for _, evidence := range report.Evidence { + if evidence.Status != StatusPass { + t.Errorf("evidence %q resolved to %s even though every check passed", evidence.ID, evidence.Status) + } + } + for _, stage := range Stages() { + var found bool + for _, reported := range report.Stages { + if reported.ID == stage { + found = true + if reported.Verdict.Checks == 0 { + t.Errorf("stage %s reported no checks", stage) + } + } + } + if !found { + t.Errorf("stage %s is missing from the report", stage) + } + } + if len(report.Coverage.Ecosystems) == 0 { + t.Fatal("the coverage matrix is empty") + } +} From d9fc9afb250f423b241f1fee50e5a6ecab9901f3 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 19 Aug 2026 22:34:53 -0400 Subject: [PATCH 05/13] fix(assurance): correct the workflow wiring an adversarial review found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A review pass over the framework found three things that would have broken the first real run, plus several ways a problem could have been reported as success. Blockers: - `continue-on-error` is not allowed on a job that calls a reusable workflow, which made assurance-prerequisites.yml an invalid workflow file — and with it Auto Version, since it calls that workflow. Fuzzing stays advisory by skipping its failing step when called in assurance mode instead. - A workflow invoked with `uses:` produces no run of its own (verified against a public repository: 416 invocations, `total_count: 0`), so looking up runs of assurance-prerequisites.yml would never find the stage Auto Version ran, and every release would have been refused. Both lookups now search the runs for the commit for a successful job named "Prerequisites verdict". - `gh api --jq --arg` is not valid (`accepts 1 arg(s), received 4`), so the assessment silently found no release run and reported all five pre-release checks as missing on every release. Reported-as-success holes: - A test command that exits 0 having run no tests — a `-run` pattern that stopped matching — was recorded as skipped, which no gate blocks on. It now fails and says so, and a skipped gate check counts as blocking. - The report step's exit status came from a trailing `echo`, so a failed report was treated as success and the commit step failed confusingly. - `catalog-valid` looked its own stage up in the catalog it was validating, so an unparseable catalog produced no result at all instead of a failure. Also: the draft guard on the yank workflow is now limited to `deleted`, since unpublishing turns a release back into a draft and the guard would have killed that path; the draft-release job mints a contents:write token, because drafts are only listed to identities with push access; the issues token is minted separately so an app without that grant loses the tracking issue rather than the whole report; the report job requires a resolved release; and Update Smoke Goldens now refreshes the catalog checksums it invalidates, which would otherwise have blocked the next release after every golden update. Co-Authored-By: Claude Fable 5 --- .github/workflows/assurance-assessment.yml | 57 ++++++++++--- .github/workflows/assurance-prerequisites.yml | 11 ++- .github/workflows/fuzz.yml | 5 +- .github/workflows/notify-landing-yank.yml | 13 +-- .github/workflows/release.yml | 35 ++++++-- .github/workflows/update-smoke-goldens.yml | 15 +++- dev-docs/RELEASE_ASSURANCE.md | 15 ++++ internal/assurance/aggregate.go | 5 +- internal/assurance/assurance_test.go | 37 +++++++++ internal/assurance/catalog.go | 80 ++++++++++++++++--- internal/assurance/catalog_test.go | 31 +++++++ internal/assurance/cmd/report.go | 22 +++++ internal/assurance/gotest.go | 15 ++-- 13 files changed, 290 insertions(+), 51 deletions(-) diff --git a/.github/workflows/assurance-assessment.yml b/.github/workflows/assurance-assessment.yml index 72fec030..3495e80a 100644 --- a/.github/workflows/assurance-assessment.yml +++ b/.github/workflows/assurance-assessment.yml @@ -384,14 +384,16 @@ jobs: report: name: Publish the release report - if: always() + # always(), so a failed check still gets a published report — but only when + # the release itself was resolved, since everything below needs its tag. + if: always() && needs.resolve.result == 'success' needs: [resolve, install-scripts, public-download, released-scan, sbom, perf-samples] runs-on: ubuntu-latest timeout-minutes: 20 steps: - name: Mint release bot token - # Commits the report to main and manages the tracking issue. The same - # app already pushes the version-bump commits. + # Commits the report to main and dispatches to the landing page. The + # same app already pushes the version-bump commits. uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 id: app-token with: @@ -402,6 +404,19 @@ jobs: bomly-cli bomly-landing-page permission-contents: write + + - name: Mint issue token + # Issues is a newer grant on the release app than contents. Minting it + # separately means an app that has not been granted it yet loses the + # tracking issue, not the whole report. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: issue-token + continue-on-error: true + with: + client-id: ${{ vars.RELEASE_BOT_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} + owner: bomly-dev + repositories: bomly-cli permission-issues: write - name: Check out main @@ -443,13 +458,31 @@ jobs: set -uo pipefail parent="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${COMMIT}" --jq '.parents[0].sha // ""')" + # A workflow called with `uses:` produces no run of its own — the + # jobs live in the caller's run (Auto Version). So the search is over + # every run for the commit, looking for the verdict job by name. + find_verdict_run() { + local sha="$1" + [ -n "${sha}" ] || return 1 + local run_id + for run_id in $(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${sha}&per_page=50" \ + --jq '.workflow_runs[].id'); do + if gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?per_page=100" \ + --jq '.jobs[] | select(.conclusion == "success") | .name' \ + | grep -q 'Prerequisites verdict$'; then + echo "${run_id}" + return 0 + fi + done + return 1 + } + prerequisites_run="" for candidate in "${COMMIT}" "${parent}"; do - [ -n "${candidate}" ] || continue - prerequisites_run="$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/assurance-prerequisites.yml/runs?head_sha=${candidate}&per_page=20" \ - --jq '[.workflow_runs[] | select(.conclusion == "success")][0].id // ""')" - [ -n "${prerequisites_run}" ] && break + if prerequisites_run="$(find_verdict_run "${candidate}")"; then + break + fi + prerequisites_run="" done if [ -n "${prerequisites_run}" ]; then gh run download "${prerequisites_run}" --repo "${GITHUB_REPOSITORY}" \ @@ -459,7 +492,7 @@ jobs: release_run="$(gh api \ "repos/${GITHUB_REPOSITORY}/actions/workflows/release.yml/runs?per_page=50" \ - --jq --arg tag "${TAG}" '[.workflow_runs[] | select(.head_branch == $tag)][0].id // ""')" + | jq -r --arg tag "${TAG}" '[.workflow_runs[] | select(.head_branch == $tag)][0].id // ""')" if [ -n "${release_run}" ]; then gh run download "${release_run}" --repo "${GITHUB_REPOSITORY}" \ --name assurance-stage-pre-release --dir assurance-results || true @@ -487,7 +520,7 @@ jobs: env: TAG: ${{ needs.resolve.outputs.tag }} run: | - set -o pipefail + set -eo pipefail go run ./internal/assurance/cmd report \ --results assurance-results \ --catalog "${catalog}" --out docs/assurance \ @@ -558,9 +591,9 @@ jobs: # blocked. One issue per release: opened when something fails, updated on # a re-run, and closed when a re-run comes back clean. - name: Track the outcome - if: always() + if: always() && steps.issue-token.outputs.token != '' env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} + GH_TOKEN: ${{ steps.issue-token.outputs.token }} TAG: ${{ needs.resolve.outputs.tag }} VERDICT: ${{ steps.report.outputs.verdict }} REPORT_OUTCOME: ${{ steps.report.outcome }} diff --git a/.github/workflows/assurance-prerequisites.yml b/.github/workflows/assurance-prerequisites.yml index bd0b5138..f39a1aed 100644 --- a/.github/workflows/assurance-prerequisites.yml +++ b/.github/workflows/assurance-prerequisites.yml @@ -55,11 +55,11 @@ jobs: assurance_tag: ${{ inputs.assurance_tag }} # Fuzzing is advisory: a finding is recorded in the report and does not stop a - # release, so a failure here must not fail this workflow either. The verdict - # job below is what decides the stage. + # release. `continue-on-error` is not allowed on a job that calls a reusable + # workflow, so fuzz.yml itself skips its failing step when it is called in + # assurance mode; the verdict job below is what decides the stage. fuzz: name: Parser fuzzing - continue-on-error: true uses: ./.github/workflows/fuzz.yml with: ref: ${{ inputs.ref || github.ref }} @@ -104,8 +104,11 @@ jobs: summary="The assurance catalog is invalid or one of the files it names has changed. Open the validation step for the exact entry." fi checks="$(grep -c '"stage":' docs/assurance/catalog.json || true)" + # --stage and --level are explicit: when the catalog is the thing + # that is broken, the emitter cannot look them up in it. go run ./internal/assurance/cmd emit \ - --id catalog-valid --exit-code "${exit_code}" --summary "${summary}" \ + --id catalog-valid --stage prerequisites --level gate \ + --exit-code "${exit_code}" --summary "${summary}" \ --metric checks="${checks}" --out assurance-results --step-summary - name: Upload check result diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 444edb6f..aa9c32e2 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -109,6 +109,9 @@ jobs: if-no-files-found: ignore retention-days: 7 + # In the release prerequisites stage fuzzing is advisory: the finding is + # recorded in the check result and the stage verdict decides what happens. + # On the nightly and manual runs a finding still fails the job. - name: Fail the job when a fuzz target failed - if: steps.fuzz.outcome == 'failure' + if: steps.fuzz.outcome == 'failure' && !inputs.assurance run: exit 1 diff --git a/.github/workflows/notify-landing-yank.yml b/.github/workflows/notify-landing-yank.yml index dd5272f7..0a0d89f1 100644 --- a/.github/workflows/notify-landing-yank.yml +++ b/.github/workflows/notify-landing-yank.yml @@ -47,12 +47,13 @@ jobs: # A failed pre-release assurance gate leaves a draft release that a # maintainer deletes. Deleting a draft fires the same `deleted` event as - # yanking a live release, so both yank paths ignore drafts: nothing was - # ever published for that version. + # yanking a live release, so deletions of drafts are ignored: nothing was + # ever published for that version. `unpublished` always refers to a live + # release (it is what turns one back into a draft), so it is not guarded. - name: Trigger docs removal on landing page (yanked) if: >- - (github.event.action == 'deleted' || github.event.action == 'unpublished') && - github.event.release.draft == false + github.event.action == 'unpublished' || + (github.event.action == 'deleted' && github.event.release.draft == false) env: GH_TOKEN: ${{ steps.landing-token.outputs.token }} TAG: ${{ github.event.release.tag_name }} @@ -66,8 +67,8 @@ jobs: winget-yank: if: >- - (github.event.action == 'deleted' || github.event.action == 'unpublished') && - github.event.release.draft == false + github.event.action == 'unpublished' || + (github.event.action == 'deleted' && github.event.release.draft == false) runs-on: ubuntu-latest timeout-minutes: 15 steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6c689479..a8435aed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,13 +44,28 @@ jobs: tag_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${RELEASE_TAG}" --jq '.sha')" parent_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${RELEASE_TAG}" --jq '.parents[0].sha // ""')" + # A workflow called with `uses:` produces no run of its own — the + # jobs live in the caller's run (Auto Version). So the search is over + # every run for the commit, looking for the verdict job by name. + find_verdict_run() { + local sha="$1" + [ -n "${sha}" ] || return 1 + local run_id + for run_id in $(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${sha}&per_page=50" \ + --jq '.workflow_runs[].id'); do + if gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?per_page=100" \ + --jq '.jobs[] | select(.conclusion == "success") | .name' \ + | grep -q 'Prerequisites verdict$'; then + echo "${run_id}" + return 0 + fi + done + return 1 + } + for candidate in "${tag_sha}" "${parent_sha}"; do - [ -n "${candidate}" ] || continue - conclusion="$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/assurance-prerequisites.yml/runs?head_sha=${candidate}&per_page=20" \ - --jq '[.workflow_runs[] | select(.conclusion == "success")] | length')" - if [ "${conclusion}" != "0" ]; then - echo "Release prerequisites passed for ${candidate}." + if run_id="$(find_verdict_run "${candidate}")"; then + echo "Release prerequisites passed for ${candidate} in run ${run_id}." exit 0 fi done @@ -243,7 +258,9 @@ jobs: private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} owner: bomly-dev repositories: bomly-cli - permission-contents: read + # Draft releases are only listed to identities with push access, so + # read is not enough here even though this job only reads. + permission-contents: write - name: Download draft release assets shell: bash @@ -441,7 +458,9 @@ jobs: with: name: assurance-stage-pre-release path: assurance-results - if-no-files-found: error + # warn, not error: when nothing was collected the gate step's message + # is the useful one, and an upload failure would hide it. + if-no-files-found: warn retention-days: 30 - name: Fail when the stage did not pass diff --git a/.github/workflows/update-smoke-goldens.yml b/.github/workflows/update-smoke-goldens.yml index c245916d..972980ee 100644 --- a/.github/workflows/update-smoke-goldens.yml +++ b/.github/workflows/update-smoke-goldens.yml @@ -327,6 +327,13 @@ jobs: with: ref: ${{ inputs.source_ref }} + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + - name: Download regenerated goldens uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -376,7 +383,13 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git checkout -b "${branch_name}" - git add test/smoke/testdata/golden + + # Public evidence claims record the checksum of the golden file that + # backs them. Regenerating a golden without refreshing the catalog + # would fail the catalog-valid gate and block the next release. + go run ./internal/assurance/cmd catalog-validate --refresh + + git add test/smoke/testdata/golden docs/assurance/catalog.json git commit -m "test: update smoke golden files" git push --set-upstream origin "${branch_name}" diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index 8489219d..ec78a7a9 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -155,6 +155,21 @@ tracking issue and the published report records the failure honestly. the report without blocking a release. Use it for the first release after a framework change, then turn it back on. +## What the automation needs + +- The Bomly Release app needs **Issues: Read and write** on `bomly-cli` for the + per-release tracking issue. Without it the assessment still publishes the + report and simply skips the issue. +- The prerequisites stage is found by looking for a successful job whose name + ends in `Prerequisites verdict` among the runs for a commit. A workflow called + with `uses:` produces no run of its own, so searching for runs of + `assurance-prerequisites.yml` would miss every stage that Auto Version + triggered. Keep that job name stable, or update the lookup in `release.yml` + and `assurance-assessment.yml` with it. +- Regenerating smoke goldens changes files the evidence claims are pinned to. + `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the + catalog alongside them; do the same when refreshing goldens by hand. + ## Related documents - [`docs/ASSURANCE.md`](../docs/ASSURANCE.md) — the public explanation diff --git a/internal/assurance/aggregate.go b/internal/assurance/aggregate.go index 5229eada..99e7837b 100644 --- a/internal/assurance/aggregate.go +++ b/internal/assurance/aggregate.go @@ -278,7 +278,10 @@ func summarize(checks []ReportCheck) Verdict { } if check.Level == LevelGate { gateStatus = Worse(gateStatus, check.Status) - if check.Status == StatusFail || check.Status == StatusDegraded { + // A gate that was skipped did not hold; it simply did not run, and + // treating that as a pass is exactly the silence this framework is + // meant to prevent. + if check.Status == StatusFail || check.Status == StatusDegraded || check.Status == StatusSkip { verdict.GatesFailed = append(verdict.GatesFailed, check.ID) } continue diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go index ebc0d721..0dac97f1 100644 --- a/internal/assurance/assurance_test.go +++ b/internal/assurance/assurance_test.go @@ -331,3 +331,40 @@ func TestParseGoTestEventsBuildFailure(t *testing.T) { t.Fatalf("summary = %q", result.Summary) } } + +// TestGoTestZeroTestsFailsInsteadOfSkipping guards the case where a -run +// pattern stops matching: the command succeeds, no test runs, and the check +// must not be able to slide past a gate. +func TestGoTestZeroTestsFailsInsteadOfSkipping(t *testing.T) { + summary, err := ParseGoTestEvents(strings.NewReader(""), nil) + if err != nil { + t.Fatalf("parse: %v", err) + } + result := summary.ToCheckResult(CheckResult{ + SchemaVersion: CheckSchema, ID: "smoke", Instance: "go", + Stage: StagePrerequisites, Level: LevelGate, + }, 0) + if result.Status != StatusFail { + t.Fatalf("status = %s, want fail when no test ran", result.Status) + } + if !strings.Contains(result.Summary, "proved nothing") { + t.Fatalf("summary = %q", result.Summary) + } +} + +// TestSkippedGateBlocks keeps a gate check that did not run from being counted +// as a pass. +func TestSkippedGateBlocks(t *testing.T) { + results := loadFixture(t, "all-pass") + for index := range results { + if results[index].ID == "release-checksums" { + results[index].Status = StatusSkip + } + } + report := BuildReport(fixtureCatalog(t), results, BuildOptions{ + Release: Release{Tag: "v9.9.9"}, Now: time.Unix(0, 0).UTC(), + }) + if !report.Verdict.Blocking() { + t.Fatal("a skipped gate check must block a release") + } +} diff --git a/internal/assurance/catalog.go b/internal/assurance/catalog.go index 3a89da99..2e754241 100644 --- a/internal/assurance/catalog.go +++ b/internal/assurance/catalog.go @@ -399,6 +399,52 @@ func validateRepoPath(path string) error { return nil } +// RefreshArtifacts rewrites every recorded hash from the file it names and +// reports how many changed. Golden files are regenerated by their own tooling, +// so the catalog has to be able to catch up without anyone editing hashes by +// hand. +func (c *Catalog) RefreshArtifacts(root string) (int, error) { + changed := 0 + for evidenceIndex := range c.Evidence { + evidence := &c.Evidence[evidenceIndex] + for artifactIndex := range evidence.Artifacts { + artifact := &evidence.Artifacts[artifactIndex] + actual, err := hashRepositoryFile(root, artifact.Path) + if err != nil { + return changed, fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + if actual != artifact.SHA256 { + artifact.SHA256 = actual + changed++ + } + } + for inputIndex := range evidence.Inputs { + input := &evidence.Inputs[inputIndex] + if input.Kind != "fixture" { + continue + } + actual, err := hashRepositoryFile(root, input.Location) + if err != nil { + return changed, fmt.Errorf("evidence %q: %w", evidence.ID, err) + } + if actual != input.SHA256 { + input.SHA256 = actual + changed++ + } + } + } + return changed, nil +} + +// Encode renders the catalog as indented JSON with a trailing newline. +func (c Catalog) Encode() ([]byte, error) { + data, err := json.MarshalIndent(c, "", " ") + if err != nil { + return nil, fmt.Errorf("encode assurance catalog: %w", err) + } + return append(data, '\n'), nil +} + // VerifyArtifacts confirms every catalog artifact and fixture input still // matches its recorded hash inside root. func (c Catalog) VerifyArtifacts(root string) error { @@ -421,40 +467,50 @@ func (c Catalog) VerifyArtifacts(root string) error { } func verifyHashedFile(root, path, want string) error { - if err := validateRepoPath(path); err != nil { + actual, err := hashRepositoryFile(root, path) + if err != nil { return err } + if actual != want { + return fmt.Errorf("%q hash is %s, want %s", path, actual, want) + } + return nil +} + +// hashRepositoryFile reads one repository file and returns its SHA-256, after +// confirming the path stays inside root. +func hashRepositoryFile(root, path string) (string, error) { + if err := validateRepoPath(path); err != nil { + return "", err + } resolvedRoot, err := filepath.EvalSymlinks(root) if err != nil { - return fmt.Errorf("resolve repository root: %w", err) + return "", fmt.Errorf("resolve repository root: %w", err) } resolved, err := filepath.EvalSymlinks(filepath.Join(root, filepath.FromSlash(path))) if err != nil { - return fmt.Errorf("resolve %q: %w", path, err) + return "", fmt.Errorf("resolve %q: %w", path, err) } relative, err := filepath.Rel(resolvedRoot, resolved) if err != nil { - return fmt.Errorf("resolve %q relative to repository: %w", path, err) + return "", fmt.Errorf("resolve %q relative to repository: %w", path, err) } if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { - return fmt.Errorf("path %q resolves outside the repository", path) + return "", fmt.Errorf("path %q resolves outside the repository", path) } info, err := os.Stat(resolved) if err != nil { - return fmt.Errorf("inspect %q: %w", path, err) + return "", fmt.Errorf("inspect %q: %w", path, err) } if !info.Mode().IsRegular() { - return fmt.Errorf("%q is not a regular file", path) + return "", fmt.Errorf("%q is not a regular file", path) } data, err := os.ReadFile(resolved) if err != nil { - return fmt.Errorf("read %q: %w", path, err) + return "", fmt.Errorf("read %q: %w", path, err) } sum := sha256.Sum256(data) - if actual := hex.EncodeToString(sum[:]); actual != want { - return fmt.Errorf("%q hash is %s, want %s", path, actual, want) - } - return nil + return hex.EncodeToString(sum[:]), nil } // ChecksForStage returns the catalog checks that belong to one stage. diff --git a/internal/assurance/catalog_test.go b/internal/assurance/catalog_test.go index eae43274..48b66ff0 100644 --- a/internal/assurance/catalog_test.go +++ b/internal/assurance/catalog_test.go @@ -250,3 +250,34 @@ func TestRepositoryCatalogProducesACompleteReport(t *testing.T) { t.Fatal("the coverage matrix is empty") } } + +func TestRefreshArtifactsRewritesDriftedHashes(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } + catalog := Catalog{Evidence: []Evidence{{ + ID: "example", + Artifacts: []EvidenceArtifact{{ + Path: "golden.json", SHA256: "1111111111111111111111111111111111111111111111111111111111111111", + }}, + Inputs: []Input{{ + Kind: "fixture", Location: "golden.json", + SHA256: "2222222222222222222222222222222222222222222222222222222222222222", + }}, + }}} + changed, err := catalog.RefreshArtifacts(root) + if err != nil { + t.Fatalf("refresh: %v", err) + } + if changed != 2 { + t.Fatalf("refreshed %d hashes, want 2", changed) + } + if err := catalog.VerifyArtifacts(root); err != nil { + t.Fatalf("refreshed catalog still fails verification: %v", err) + } + again, err := catalog.RefreshArtifacts(root) + if err != nil || again != 0 { + t.Fatalf("second refresh changed %d hashes (err=%v), want 0", again, err) + } +} diff --git a/internal/assurance/cmd/report.go b/internal/assurance/cmd/report.go index 4e01fa18..7a36a233 100644 --- a/internal/assurance/cmd/report.go +++ b/internal/assurance/cmd/report.go @@ -238,6 +238,7 @@ func runCatalogValidate(args []string) error { checkID := flags.String("check", "", "print one check") evidenceID := flags.String("evidence", "", "print one evidence claim") skipArtifacts := flags.Bool("skip-artifacts", false, "skip repository artifact hash verification") + refresh := flags.Bool("refresh", false, "rewrite recorded checksums from the files they name") if err := flags.Parse(args); err != nil { return err } @@ -245,6 +246,27 @@ func runCatalogValidate(args []string) error { if err != nil { return err } + if *refresh { + root, rootErr := repositoryRoot() + if rootErr != nil { + return rootErr + } + changed, refreshErr := catalog.RefreshArtifacts(root) + if refreshErr != nil { + return refreshErr + } + if changed > 0 { + data, encodeErr := catalog.Encode() + if encodeErr != nil { + return encodeErr + } + if err := os.WriteFile(catalogFile, data, 0o644); err != nil { + return fmt.Errorf("write assurance catalog: %w", err) + } + } + fmt.Printf("Refreshed %d recorded checksum(s) in %s.\n", changed, relativeToWorkingDir(catalogFile)) + return nil + } if !*skipArtifacts { root, rootErr := repositoryRoot() if rootErr != nil { diff --git a/internal/assurance/gotest.go b/internal/assurance/gotest.go index 95773ced..4ddbb3cf 100644 --- a/internal/assurance/gotest.go +++ b/internal/assurance/gotest.go @@ -178,7 +178,10 @@ func (s GoTestSummary) ToCheckResult(base CheckResult, exitCode int) CheckResult case s.Failed > 0 || exitCode != 0: result.Status = StatusFail case s.Total == 0: - result.Status = StatusSkip + // A test command that ran cleanly but executed nothing proves nothing: + // usually a -run pattern that no longer matches. Reporting that as a + // skip would let it slide past a gate, so it fails instead. + result.Status = StatusFail } result.Metrics = map[string]float64{ "tests_total": float64(s.Total), @@ -197,14 +200,14 @@ func (s GoTestSummary) ToCheckResult(base CheckResult, exitCode int) CheckResult func (s GoTestSummary) summaryLine(exitCode int) string { if s.Total == 0 { + detail := "" + if len(s.Anomalies) > 0 { + detail = " " + s.Anomalies[0] + } if exitCode != 0 { - detail := "" - if len(s.Anomalies) > 0 { - detail = " " + s.Anomalies[0] - } return fmt.Sprintf("No tests ran and the command exited with code %d.%s", exitCode, detail) } - return "No tests ran." + return "No tests ran, so this check proved nothing. The test selection most likely no longer matches any test." + detail } parts := []string{fmt.Sprintf("%d of %d tests passed", s.Passed, s.Total)} if s.Failed > 0 { From 75f9581109c0b8df334bdddfbd3e407af6f08757 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 19 Aug 2026 23:25:35 -0400 Subject: [PATCH 06/13] refactor(assurance): link counts to their jobs, and drop what repeats itself Two revisions after reading the report end to end. Every reported count now links to the job behind it. Each instance of a matrix check shares one workflow run, so a run-level link could not tell a reader which platform or slice a number came from; the emitter now resolves the job it is running inside by matching the running job on this runner, and records that. It reads public workflow metadata once, falls back to the run URL whenever that does not work, and honours ASSURANCE_JOB_URL. Markdown summaries link the failing jobs only, so a red summary reaches the log in one click without burying it among passing rows. Removed as redundant: - `unit-repeat-full` ran the whole suite five more times on Linux on top of the two runs `unit-portable` already does on each of three platforms, and the suite that CI and release validation each run again. Five full-suite runs was the largest single cost in the prerequisites stage, and the repeats that have actually caught intermittent failures are the Java detector ones, which stay at ten. - Five evidence claims that only restated their backing check (portable-platforms, performance-stability, sbom-interoperability, release-integrity, supported-install-paths). What makes an evidence claim worth a second entry is the pinned input and the committed artifact it adds; without those it is the check card said twice. The contract now enforces that: artifacts are required, and the input kinds and evidence levels that existed only for workflow-backed claims are gone. - `public-download` re-extracted and re-ran the released binaries that the pre-release stage had already extracted and run. It now checks what is actually new after publication: that the public copies are complete and match the checksums they were signed with. 16 checks and 21 evidence claims, from 17 and 26. Co-Authored-By: Claude Fable 5 --- .github/workflows/assurance-assessment.yml | 17 +- .github/workflows/portable-assurance.yml | 52 +---- dev-docs/RELEASE_ASSURANCE.md | 23 ++- docs/ASSURANCE.md | 23 ++- docs/assurance/catalog.json | 195 +----------------- internal/assurance/assurance_test.go | 43 ++++ internal/assurance/catalog.go | 36 +--- internal/assurance/catalog_test.go | 19 +- internal/assurance/cmd/joburl.go | 73 +++++++ internal/assurance/cmd/main.go | 2 +- internal/assurance/joburl.go | 52 +++++ internal/assurance/render_markdown.go | 29 +++ .../testdata/golden/mixed-failure.summary.md | 4 +- test/assurance/BENCHMARK_RUNS.md | 4 +- 14 files changed, 273 insertions(+), 299 deletions(-) create mode 100644 internal/assurance/cmd/joburl.go create mode 100644 internal/assurance/joburl.go diff --git a/.github/workflows/assurance-assessment.yml b/.github/workflows/assurance-assessment.yml index 3495e80a..5f52fb81 100644 --- a/.github/workflows/assurance-assessment.yml +++ b/.github/workflows/assurance-assessment.yml @@ -179,15 +179,16 @@ jobs: curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/SHA256SUMS.sigstore.json" "${base}/SHA256SUMS.sigstore.json" curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/multiple.intoto.jsonl" "${base}/multiple.intoto.jsonl" - - name: Verify what a user would download + # Only the checksums are re-checked here: the pre-release stage already + # extracted and ran the binaries, so what is new after publication is + # whether the public copies are complete and unaltered. + - name: Check the public copies against the published checksums id: verify continue-on-error: true - env: - BOMLY_ASSURANCE_TAG: ${{ needs.resolve.outputs.tag }} run: | - go run ./internal/assurance/cmd verify-release \ - --dir "${RUNNER_TEMP}/assets" --version '${{ needs.resolve.outputs.version }}' \ - --scope full --out "${RUNNER_TEMP}/release-results" + set -euo pipefail + cd "${RUNNER_TEMP}/assets" + sha256sum --check --strict SHA256SUMS - name: Record the public download result if: always() @@ -199,13 +200,13 @@ jobs: set -euo pipefail exit_code=0 count="$(find "${RUNNER_TEMP}/assets" -type f | wc -l | tr -d ' ')" - summary="All ${count} published files downloaded from their public URLs and match the published checksum list." + summary="All ${count} published files downloaded from their public URLs without credentials and match the published checksum list." if [ "${DOWNLOAD_OUTCOME}" != "success" ]; then exit_code=1 summary="At least one published file could not be downloaded from its public URL." elif [ "${VERIFY_OUTCOME}" != "success" ]; then exit_code=1 - summary="The published files downloaded, but ${count} of them did not match the published checksum list." + summary="The published files downloaded, but at least one did not match the published checksum list." fi go run ./internal/assurance/cmd emit \ --id public-download --exit-code "${exit_code}" --summary "${summary}" \ diff --git a/.github/workflows/portable-assurance.yml b/.github/workflows/portable-assurance.yml index 69d67db0..23811733 100644 --- a/.github/workflows/portable-assurance.yml +++ b/.github/workflows/portable-assurance.yml @@ -1,5 +1,7 @@ # Repeated unit tests across supported platforms and cross-builds of every -# release binary. Runs on its own when a maintainer starts it, and as part of +# release binary. The full suite is repeated by the `portable` job (twice on +# each of three platforms); this job repeats only the Java detector suites, +# which is where intermittent failures have actually appeared. Runs on its own when a maintainer starts it, and as part of # the "Release prerequisites" stage before a version is tagged. # # Every step ends by writing a release assurance check result; the job summary @@ -187,53 +189,6 @@ jobs: --details-jsonl "${RUNNER_TEMP}/java-details.jsonl" \ --out assurance-results --step-summary - - name: Repeat complete suite five times - id: complete-tests - continue-on-error: true - run: | - set -uo pipefail - completed=0 - failed=0 - : > "${RUNNER_TEMP}/complete-details.jsonl" - for iteration in $(seq 1 5); do - echo "complete suite iteration ${iteration}" - if go test ./... -count=1; then - completed="${iteration}" - printf '{"name":"iteration %s","status":"pass"}\n' "${iteration}" >> "${RUNNER_TEMP}/complete-details.jsonl" - else - failed="${iteration}" - printf '{"name":"iteration %s","status":"fail"}\n' "${iteration}" >> "${RUNNER_TEMP}/complete-details.jsonl" - break - fi - done - echo "completed=${completed}" >> "${GITHUB_OUTPUT}" - echo "failed=${failed}" >> "${GITHUB_OUTPUT}" - if [[ "${failed}" != "0" ]]; then - exit 1 - fi - - - name: Record the repeated suite result - if: always() - env: - BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} - COMPLETE_RESULT: ${{ steps.complete-tests.outcome }} - COMPLETED: ${{ steps.complete-tests.outputs.completed }} - FAILED: ${{ steps.complete-tests.outputs.failed }} - run: | - set -euo pipefail - completed="${COMPLETED:-0}" - exit_code=0 - summary="The complete Go test suite passed five more times on Linux." - if [[ "${COMPLETE_RESULT}" != "success" ]]; then - exit_code=1 - summary="The complete Go test suite failed on run ${FAILED:-0} of 5, which points at an intermittent failure." - fi - go run ./internal/assurance/cmd emit \ - --id unit-repeat-full --exit-code "${exit_code}" --summary "${summary}" \ - --metric planned_runs=5 --metric completed_runs="${completed}" \ - --details-jsonl "${RUNNER_TEMP}/complete-details.jsonl" \ - --out assurance-results --step-summary - # Every target is attempted, even after one fails, so the report shows the # complete build matrix instead of stopping at the first problem. - name: Cross-build release targets @@ -324,6 +279,5 @@ jobs: - name: Fail the job when any check failed if: >- steps.java-tests.outcome == 'failure' || - steps.complete-tests.outcome == 'failure' || steps.release-builds.outcome == 'failure' run: exit 1 diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index ec78a7a9..be818400 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -66,10 +66,17 @@ step only passes `--stage` when it emits something the catalog does not declare (which the report will then flag as unknown). `--details-jsonl` reads sub-results from a file, which keeps Windows command lines short. -Every command fills the release tag, commit, run URL, job, and runner from the +Every command fills the release tag, commit, job URL, job, and runner from the GitHub Actions environment. Set `BOMLY_ASSURANCE_TAG` when a stage runs for a tag that is not the checked-out ref. +The job URL is resolved by reading the run's job list and matching the job +running on this runner, because every instance of a matrix check shares one +run and a run-level link cannot tell a reader which platform or slice a number +came from. It costs one read of public workflow metadata, falls back to the run +URL whenever that does not work, and can be set directly with +`ASSURANCE_JOB_URL`. + ## Judging a stage and building the report ```sh @@ -128,12 +135,14 @@ Evidence claims are the public "we prove X, we do not prove Y" statements that used to live in `test/evidence/cases.json`. They keep the same rigor: pinned Git revisions, checksummed fixtures and expected-result files, explicit reproduction commands, and mandatory limitations. `make assurance-catalog` -re-hashes every file a claim names, so a golden file cannot drift away from the -claim it supports. - -Claims proven by running a workflow against a release (`release-artifact` and -`platform-matrix` levels) carry no repository checksum, because the per-release -check result is the record that they ran. +verifies every hash a claim names, so a golden file cannot drift away from the +claim it supports, and `--refresh` rewrites them when the goldens are +deliberately regenerated. + +A claim must carry a pinned input **and** a committed artifact. That is the line +between the two layers: if a statement would only restate what its check already +reports — "the workflow ran and passed" — it belongs in the check's `proves`, +not in a second entry that says the same thing again. ## Re-running things diff --git a/docs/ASSURANCE.md b/docs/ASSURANCE.md index 8f03ab3b..9a54149b 100644 --- a/docs/ASSURANCE.md +++ b/docs/ASSURANCE.md @@ -32,8 +32,10 @@ Highlights: - **End-to-end scans.** Every supported ecosystem is scanned from a pinned public example project and compared against a checked-in expected result. -- **Platform stability.** The unit tests run repeatedly on Linux, macOS, and - Windows, and every release binary is cross-compiled. +- **Platform stability.** The full unit test suite runs twice on Linux, macOS, + and Windows, the Java detector suites run ten times because that is where + intermittent failures have appeared, and every release binary is + cross-compiled. - **Parser safety.** Fuzz targets feed malformed project, configuration, baseline, and SBOM files to the parsers that read them. - **Release integrity.** Checksums, the Sigstore signature, and SLSA build @@ -46,15 +48,22 @@ Highlights: cache to record timing and confirm the output does not change. Checks are either **gates**, which stop a release, or **advisory**, which are -reported but never block. The report always says which is which. +reported but never block. The report always says which is which, and every +count links to the job that produced it, so any number on the page is one click +from its log. ## Evidence claims Alongside the checks, the catalog carries **evidence claims**: specific -statements about Bomly's behavior, each with the pinned input, the exact -command to reproduce it, the expected result file, and its stated limitations. -Each claim names the check that backs it, so the report can show whether that -claim still held for the release you are looking at. +statements about Bomly's behavior, each with a pinned input — a public +repository at a recorded commit, or a checked-in fixture — the exact command to +reproduce it, the checksummed result file it is compared against, and its +stated limitations. Each claim names the check that backs it, so the report can +show whether that claim still held for the release you are looking at. + +A claim earns its own entry only when it adds something a check result cannot: +the pinned input and the committed file. Claims that would just restate their +check are not repeated here. Reproduce any claim from a repository checkout: diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json index ce769c4f..f3e3dbc5 100644 --- a/docs/assurance/catalog.json +++ b/docs/assurance/catalog.json @@ -220,7 +220,7 @@ "area": "release-integrity", "stage": "post-release", "level": "gate", - "description": "Downloads release files from their public URLs after publication and checks them against the published checksum list.", + "description": "Downloads every published file from its public URL, without credentials, and checks each one against the published checksum list.", "source": { "workflow": "assurance-assessment.yml", "job": "public-download" @@ -236,10 +236,11 @@ ] ], "proves": [ - "The files a user downloads from the public release page are present and match the checksums published with them." + "Every file a user downloads from the public release page is present, reachable without credentials, and byte-identical to what was signed before publication." ], "limitations": [ - "This checks availability and integrity at one moment from one network location." + "This checks availability and integrity at one moment from one network location.", + "Whether each archive contains a working binary is checked separately, against the draft release and by the install scripts." ] }, { @@ -654,7 +655,7 @@ "area": "stability", "stage": "prerequisites", "level": "gate", - "description": "Runs the full Go unit test suite twice on Linux, macOS, and Windows.", + "description": "Runs the full Go unit test suite twice on Linux, macOS, and Windows, so a pass is never a single lucky run.", "source": { "workflow": "portable-assurance.yml", "job": "portable" @@ -688,39 +689,13 @@ "Unit tests run in process against fixtures. They do not download repositories or call external services." ] }, - { - "id": "unit-repeat-full", - "title": "Repeated full test runs are stable", - "area": "stability", - "stage": "prerequisites", - "level": "gate", - "description": "Runs the complete unit test suite five more times on Linux to surface intermittent failures.", - "source": { - "workflow": "portable-assurance.yml", - "job": "linux-stability" - }, - "reproduce": [ - [ - "go", - "test", - "./...", - "-count=5" - ] - ], - "proves": [ - "The full suite passed five consecutive times, so the tests recorded here are not intermittently passing." - ], - "limitations": [ - "Repetition raises confidence in stability; it cannot rule out a rare failure." - ] - }, { "id": "unit-repeat-java", "title": "Java detector tests are stable", "area": "stability", "stage": "prerequisites", "level": "gate", - "description": "Runs the Gradle, Maven, and sbt detector test suites ten times, because build-tool-backed detectors are the most timing sensitive.", + "description": "Runs the Gradle, Maven, and sbt detector test suites ten times, because build-tool-backed detectors are where intermittent failures have actually appeared.", "source": { "workflow": "portable-assurance.yml", "job": "linux-stability" @@ -1239,35 +1214,6 @@ "The tests prove policy evaluation after license data is present; they do not prove the accuracy of every upstream license source." ] }, - { - "id": "performance-stability", - "title": "Repeated cold and warm scan measurements", - "area": "performance", - "evidence_level": "release-artifact", - "check_id": "perf-samples", - "inputs": [ - { - "kind": "fixture", - "location": "test/smoke/testdata/sboms/go.spdx.json", - "sha256": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" - } - ], - "required_tools": [ - "go" - ], - "reproduce": [ - [ - "make", - "benchmark-samples" - ] - ], - "proves": [ - "The runner records five isolated cold and five shared-cache warm samples with output hashes, timing, memory, and dispersion." - ], - "limitations": [ - "Wall time and memory are machine-specific observations and are not fixed pass-or-fail limits." - ] - }, { "id": "persisted-risk", "title": "Risk that persists across a version change", @@ -1307,38 +1253,6 @@ "Persistence uses the canonical package-finding identity; a genuinely different advisory or rule remains a distinct finding." ] }, - { - "id": "portable-platforms", - "title": "Repeated unit tests and release builds", - "area": "stability", - "evidence_level": "platform-matrix", - "check_id": "unit-portable", - "inputs": [ - { - "kind": "workflow", - "location": ".github/workflows/portable-assurance.yml" - } - ], - "required_tools": [ - "gh" - ], - "reproduce": [ - [ - "gh", - "workflow", - "run", - "portable-assurance.yml", - "--ref", - "v0.20.0" - ] - ], - "proves": [ - "The manual workflow repeats Go unit tests on Linux, macOS, and Windows and cross-builds every release target." - ], - "limitations": [ - "This workflow runs unit tests and builds; it does not run smoke tests against remote repositories or services." - ] - }, { "id": "reachability-go", "title": "Go vulnerability reachability", @@ -1527,45 +1441,6 @@ "This is package-tier evidence, reflective imports can reduce confidence, and live advisory results may change." ] }, - { - "id": "release-integrity", - "title": "Published release files are complete and signed", - "area": "release-integrity", - "evidence_level": "release-artifact", - "check_id": "release-checksums", - "inputs": [ - { - "kind": "release", - "location": "https://github.com/bomly-dev/bomly-cli/releases" - } - ], - "required_tools": [ - "cosign", - "slsa-verifier" - ], - "reproduce": [ - [ - "gh", - "release", - "download", - "", - "--repo", - "bomly-dev/bomly-cli" - ], - [ - "sha256sum", - "--check", - "SHA256SUMS", - "--ignore-missing" - ] - ], - "proves": [ - "Every file this release publishes was checked against its checksum, its signature, and its build provenance before the release was made public." - ], - "limitations": [ - "These checks describe the files as published. Anyone can re-run them at any time with the commands above." - ] - }, { "id": "remediation-read-only", "title": "Canonical read-only remediation guidance", @@ -1650,38 +1525,6 @@ "The fixture covers one document shape and does not imply lossless conversion from every producer." ] }, - { - "id": "sbom-interoperability", - "title": "External SBOM validation", - "area": "sbom", - "evidence_level": "release-artifact", - "check_id": "sbom-interoperability", - "inputs": [ - { - "kind": "workflow", - "location": ".github/workflows/sbom-interoperability.yml" - } - ], - "required_tools": [ - "gh" - ], - "reproduce": [ - [ - "gh", - "workflow", - "run", - "sbom-interoperability.yml", - "--ref", - "v0.20.0" - ] - ], - "proves": [ - "Checksum-pinned official SPDX and CycloneDX validators can check Bomly's generated canonical SBOMs." - ], - "limitations": [ - "The workflow validates canonical fixtures, not every possible receiving tool or document conversion." - ] - }, { "id": "sbom-spdx-ingest", "title": "SPDX 2.3 ingestion", @@ -1780,32 +1623,6 @@ "A source change is a review signal, not proof of malicious intent, and unknown source evidence cannot be classified." ] }, - { - "id": "supported-install-paths", - "title": "The documented install paths work", - "area": "installation", - "evidence_level": "release-artifact", - "check_id": "install-script", - "inputs": [ - { - "kind": "release", - "location": "https://bomly.dev/install.sh" - } - ], - "reproduce": [ - [ - "sh", - "-c", - "curl -fsSL https://bomly.dev/install.sh | sh" - ] - ], - "proves": [ - "The install script published for this release downloads, verifies, and installs a working binary on Linux, macOS, and Windows." - ], - "limitations": [ - "Package manager installs (Homebrew, Scoop, WinGet, and the Linux packages) are published by the same release but are not exercised by this check." - ] - }, { "id": "vulnerability-policy", "title": "Vulnerability policy constraints", diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go index 0dac97f1..285dec9b 100644 --- a/internal/assurance/assurance_test.go +++ b/internal/assurance/assurance_test.go @@ -368,3 +368,46 @@ func TestSkippedGateBlocks(t *testing.T) { t.Fatal("a skipped gate check must block a release") } } + +func TestMatchJobURLPicksTheRunningJobOnThisRunner(t *testing.T) { + payload := []byte(`{"jobs":[ + {"name":"Portable suite (ubuntu-latest)","status":"completed","runner_name":"GitHub Actions 3","html_url":"https://example.test/job/1"}, + {"name":"Portable suite (macos-latest)","status":"in_progress","runner_name":"GitHub Actions 7","html_url":"https://example.test/job/2"}, + {"name":"Portable suite (windows-latest)","status":"in_progress","runner_name":"GitHub Actions 9","html_url":"https://example.test/job/3"} + ]}`) + url, err := MatchJobURL(payload, "GitHub Actions 7") + if err != nil { + t.Fatalf("match: %v", err) + } + if url != "https://example.test/job/2" { + t.Fatalf("url = %q, want the running job on this runner", url) + } + + for name, args := range map[string][2]string{ + "unknown runner": {string(payload), "GitHub Actions 42"}, + "no runner name": {string(payload), ""}, + "empty payload": {`{"jobs":[]}`, "GitHub Actions 7"}, + } { + t.Run(name, func(t *testing.T) { + url, err := MatchJobURL([]byte(args[0]), args[1]) + if err != nil { + t.Fatalf("match: %v", err) + } + if url != "" { + t.Fatalf("url = %q, want no match so the caller links the run", url) + } + }) + } + + ambiguous := []byte(`{"jobs":[ + {"name":"a","status":"in_progress","runner_name":"shared","html_url":"https://example.test/job/1"}, + {"name":"b","status":"in_progress","runner_name":"shared","html_url":"https://example.test/job/2"} + ]}`) + if url, err := MatchJobURL(ambiguous, "shared"); err != nil || url != "" { + t.Fatalf("ambiguous match returned %q (err=%v), want no link", url, err) + } + + if _, err := MatchJobURL([]byte("not json"), "runner"); err == nil { + t.Fatal("expected malformed payload to be reported") + } +} diff --git a/internal/assurance/catalog.go b/internal/assurance/catalog.go index 2e754241..d5cae7eb 100644 --- a/internal/assurance/catalog.go +++ b/internal/assurance/catalog.go @@ -39,17 +39,12 @@ const ( EvidenceSnapshot EvidenceLevel = "snapshot" // EvidenceLiveService depends on data from a live service. EvidenceLiveService EvidenceLevel = "live-service" - // EvidenceReleaseArtifact is measured against published release artifacts. - EvidenceReleaseArtifact EvidenceLevel = "release-artifact" - // EvidencePlatformMatrix comes from repeated runs across platforms. - EvidencePlatformMatrix EvidenceLevel = "platform-matrix" ) // Valid reports whether the evidence level is known. func (e EvidenceLevel) Valid() bool { switch e { - case EvidenceDeterministic, EvidencePinnedInput, EvidenceSnapshot, - EvidenceLiveService, EvidenceReleaseArtifact, EvidencePlatformMatrix: + case EvidenceDeterministic, EvidencePinnedInput, EvidenceSnapshot, EvidenceLiveService: return true default: return false @@ -117,8 +112,10 @@ type EvidenceArtifact struct { SHA256 string `json:"sha256"` } -// Evidence is one public claim about Bomly's behavior, backed by a check whose -// per-release status shows whether the claim still holds. +// Evidence is one public claim about Bomly's behavior, proven by a pinned input +// and a committed result file, and backed by a check whose per-release status +// shows whether the claim still holds. A claim that would only restate what its +// check already reports does not belong here — the check card says it once. type Evidence struct { ID string `json:"id"` Title string `json:"title"` @@ -129,7 +126,7 @@ type Evidence struct { Inputs []Input `json:"inputs"` RequiredTools []string `json:"required_tools,omitempty"` Reproduce [][]string `json:"reproduce"` - Artifacts []EvidenceArtifact `json:"artifacts,omitempty"` + Artifacts []EvidenceArtifact `json:"artifacts"` Proves []string `json:"proves"` Limitations []string `json:"limitations"` } @@ -298,8 +295,8 @@ func validateEvidence(evidence Evidence, areas map[string]struct{}, checks map[s if err := validateCommands(evidence.Reproduce); err != nil { return err } - if len(evidence.Artifacts) == 0 && requiresArtifact(evidence) { - return errCatalog("repository-backed evidence requires at least one hashed artifact") + if len(evidence.Artifacts) == 0 { + return errCatalog("evidence requires at least one hashed artifact") } for _, artifact := range evidence.Artifacts { if !hashPattern.MatchString(artifact.SHA256) { @@ -312,19 +309,6 @@ func validateEvidence(evidence Evidence, areas map[string]struct{}, checks map[s return validateClaims(evidence.Proves, evidence.Limitations) } -// requiresArtifact reports whether an evidence claim must point at a hashed -// repository file. Claims proven by committed fixtures and goldens must; claims -// proven by running a workflow against a release are recorded by the per-release -// check result instead. -func requiresArtifact(evidence Evidence) bool { - switch evidence.EvidenceLevel { - case EvidenceDeterministic, EvidencePinnedInput, EvidenceSnapshot, EvidenceLiveService: - return true - default: - return false - } -} - func hasInstance(check Check, name string) bool { for _, instance := range check.ExpectedInstances { if instance.Name == name { @@ -355,10 +339,6 @@ func validateInput(level EvidenceLevel, input Input) error { if level == EvidencePinnedInput && !containerDigestPattern.MatchString(input.Ref) { return errCatalog("pinned container input requires an immutable sha256 digest") } - case "release", "service", "workflow": - // A published release asset, an external service, or a CI workflow: - // identified by location. The per-release check result is the record - // that it ran, so there is no repository hash to pin. default: return fmt.Errorf("unsupported input kind %q", input.Kind) } diff --git a/internal/assurance/catalog_test.go b/internal/assurance/catalog_test.go index 48b66ff0..44bf3863 100644 --- a/internal/assurance/catalog_test.go +++ b/internal/assurance/catalog_test.go @@ -105,9 +105,15 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { }}, Evidence: []Evidence{{ ID: "graph-go", Title: "Go graph", Area: "end-to-end", - EvidenceLevel: EvidenceReleaseArtifact, CheckID: "smoke", - Inputs: []Input{{Kind: "release", Location: "https://example.test/release"}}, - Reproduce: [][]string{{"make", "smoke"}}, + EvidenceLevel: EvidencePinnedInput, CheckID: "smoke", + Inputs: []Input{{ + Kind: "git", Location: "https://example.test/repo", + Revision: "0f2103c7e671653e519cf5edb0d3e86020202ecf", + }}, + Reproduce: [][]string{{"make", "smoke"}}, + Artifacts: []EvidenceArtifact{{ + Path: "go.mod", SHA256: "0000000000000000000000000000000000000000000000000000000000000000", + }}, Proves: []string{"It resolves."}, Limitations: []string{"One toolchain."}, }}, @@ -133,6 +139,10 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { "git without revision": func(c *Catalog) { c.Evidence[0].Inputs = []Input{{Kind: "git", Location: "https://example.test/repo"}} }, + "unsupported input kind": func(c *Catalog) { + c.Evidence[0].Inputs = []Input{{Kind: "workflow", Location: ".github/workflows/smoke.yml"}} + }, + "evidence without an artifact": func(c *Catalog) { c.Evidence[0].Artifacts = nil }, "fixture without hash": func(c *Catalog) { c.Evidence[0].Inputs = []Input{{Kind: "fixture", Location: "go.mod"}} }, @@ -142,9 +152,6 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { SHA256: "0000000000000000000000000000000000000000000000000000000000000000", }} }, - "deterministic evidence needs an artifact": func(c *Catalog) { - c.Evidence[0].EvidenceLevel = EvidenceDeterministic - }, "unsorted checks": func(c *Catalog) { c.Checks = append(c.Checks, c.Checks[0]) c.Checks[1].ID = "aaa" diff --git a/internal/assurance/cmd/joburl.go b/internal/assurance/cmd/joburl.go new file mode 100644 index 00000000..a7276569 --- /dev/null +++ b/internal/assurance/cmd/joburl.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/bomly-dev/bomly-cli/internal/assurance" +) + +// jobURL returns the URL a check result should point at: the job it ran in when +// that can be resolved, and the workflow run otherwise. +// +// Every instance of a matrix check shares one run, so run-level links cannot +// tell a reader which platform or slice a number came from. Resolving the job +// costs one read of public workflow metadata and is skipped silently whenever +// it does not work. +func jobURL(runURL string) string { + if override := os.Getenv("ASSURANCE_JOB_URL"); override != "" { + return override + } + repository := os.Getenv("GITHUB_REPOSITORY") + runID := os.Getenv("GITHUB_RUN_ID") + runnerName := os.Getenv("RUNNER_NAME") + if repository == "" || runID == "" || runnerName == "" { + return runURL + } + endpoint := fmt.Sprintf("https://api.github.com/repos/%s/actions/runs/%s/jobs?per_page=100", repository, runID) + if attempt := os.Getenv("GITHUB_RUN_ATTEMPT"); attempt != "" { + endpoint = fmt.Sprintf("https://api.github.com/repos/%s/actions/runs/%s/attempts/%s/jobs?per_page=100", + repository, runID, attempt) + } + payload, err := fetchJSON(endpoint) + if err != nil { + fmt.Fprintf(os.Stderr, "assurance: could not resolve the job URL (%v); linking the run instead\n", err) + return runURL + } + resolved, err := assurance.MatchJobURL(payload, runnerName) + if err != nil || resolved == "" { + return runURL + } + return resolved +} + +func fetchJSON(endpoint string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + request.Header.Set("Accept", "application/vnd.github+json") + // The token lifts the rate limit; the endpoint is readable without one on a + // public repository, so a missing token is not an error. + for _, name := range []string{"GH_TOKEN", "GITHUB_TOKEN"} { + if token := os.Getenv(name); token != "" { + request.Header.Set("Authorization", "Bearer "+token) + break + } + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("workflow jobs request returned %s", response.Status) + } + return io.ReadAll(io.LimitReader(response.Body, 8<<20)) +} diff --git a/internal/assurance/cmd/main.go b/internal/assurance/cmd/main.go index 40e381b5..e3eabb5a 100644 --- a/internal/assurance/cmd/main.go +++ b/internal/assurance/cmd/main.go @@ -158,7 +158,7 @@ func baseResult(id, instance string) assurance.CheckResult { StartedAt: time.Now().UTC().Format(time.RFC3339), } result.Version = strings.TrimPrefix(result.Tag, "v") - result.RunURL = runURL() + result.RunURL = jobURL(runURL()) return result } diff --git a/internal/assurance/joburl.go b/internal/assurance/joburl.go new file mode 100644 index 00000000..aca6b864 --- /dev/null +++ b/internal/assurance/joburl.go @@ -0,0 +1,52 @@ +package assurance + +import ( + "encoding/json" + "fmt" +) + +// maxJobsPayloadBytes bounds the workflow-jobs response this package parses. +const maxJobsPayloadBytes = 8 << 20 + +type jobsPayload struct { + Jobs []struct { + Name string `json:"name"` + Status string `json:"status"` + RunnerName string `json:"runner_name"` + HTMLURL string `json:"html_url"` + } `json:"jobs"` +} + +// MatchJobURL picks the URL of the job a check is running inside, so a reported +// count links to the exact log that produced it rather than to the whole run. +// +// A runner executes one job at a time, so the running job on this runner is the +// caller's own job — that holds for matrix legs and for jobs contributed by a +// called workflow, which is why the runner name is matched rather than the job +// name (display names differ between direct and nested invocations). Returns an +// empty string when there is no confident match; callers fall back to the run. +func MatchJobURL(payload []byte, runnerName string) (string, error) { + if len(payload) > maxJobsPayloadBytes { + return "", fmt.Errorf("workflow jobs payload is %d bytes, limit is %d", len(payload), maxJobsPayloadBytes) + } + if runnerName == "" { + return "", nil + } + var jobs jobsPayload + if err := json.Unmarshal(payload, &jobs); err != nil { + return "", fmt.Errorf("decode workflow jobs: %w", err) + } + match := "" + for _, job := range jobs.Jobs { + if job.RunnerName != runnerName || job.Status != "in_progress" || job.HTMLURL == "" { + continue + } + if match != "" { + // Two running jobs claiming the same runner should not happen; if + // it does, a wrong link is worse than the run-level one. + return "", nil + } + match = job.HTMLURL + } + return match, nil +} diff --git a/internal/assurance/render_markdown.go b/internal/assurance/render_markdown.go index 2bc55f5c..cc781de0 100644 --- a/internal/assurance/render_markdown.go +++ b/internal/assurance/render_markdown.go @@ -120,6 +120,12 @@ func attentionLines(report Report) []string { if len(check.MissingInstances) > 0 { line += fmt.Sprintf(" Missing: %s.", strings.Join(check.MissingInstances, ", ")) } + // Only the parts that need attention are linked: a reader of a red + // summary should reach the failing log in one click, and linking the + // passing rows too would bury it. + if logs := failingLogs(check); logs != "" { + line += " Logs: " + logs + "." + } lines = append(lines, line) } for _, unknown := range report.Unknown { @@ -128,6 +134,29 @@ func attentionLines(report Report) []string { return lines } +// failingLogs renders links to the jobs behind a check's failing instances. +func failingLogs(check ReportCheck) string { + var links []string + for _, instance := range check.Instances { + if instance.Status == StatusPass || instance.RunURL == "" { + continue + } + name := instance.Name + if name == "default" { + name = "job" + } + links = append(links, fmt.Sprintf("[%s](%s)", name, instance.RunURL)) + } + if len(links) == 0 { + return "" + } + if len(links) > 5 { + remaining := len(links) - 5 + links = append(links[:5], fmt.Sprintf("and %d more", remaining)) + } + return strings.Join(links, ", ") +} + func renderTrends(trends Trends) string { if len(trends.Changed) == 0 && len(trends.Metrics) == 0 { return "" diff --git a/internal/assurance/testdata/golden/mixed-failure.summary.md b/internal/assurance/testdata/golden/mixed-failure.summary.md index d4d8989c..dfebf9b8 100644 --- a/internal/assurance/testdata/golden/mixed-failure.summary.md +++ b/internal/assurance/testdata/golden/mixed-failure.summary.md @@ -28,8 +28,8 @@ ### Needs attention -- ⚠️ **Repeated scan speed and stability** (advisory, post-release) — canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms). -- ❌ **Release files match their checksums** (gate, pre-release) — 1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz. +- ⚠️ **Repeated scan speed and stability** (advisory, post-release) — canonical-sbom-scan completed 5 samples per cache mode with identical normalized output (cold median 705 ms, warm median 402 ms). Logs: [job](https://github.com/bomly-dev/bomly-cli/actions/runs/11). +- ❌ **Release files match their checksums** (gate, pre-release) — 1 assets do not match SHA256SUMS: bomly_9.9.9_linux_amd64.tar.gz. Logs: [job](https://github.com/bomly-dev/bomly-cli/actions/runs/10). - ❓ **End-to-end scans of real projects** (gate, prerequisites) — 1 instance passed, 1 instance reported nothing. Missing: node. - ❓ Result `mystery-check` is not declared in the assurance catalog. diff --git a/test/assurance/BENCHMARK_RUNS.md b/test/assurance/BENCHMARK_RUNS.md index 55969c78..b0f7fe6d 100644 --- a/test/assurance/BENCHMARK_RUNS.md +++ b/test/assurance/BENCHMARK_RUNS.md @@ -44,8 +44,8 @@ GitHub Actions, and as part of the `Release prerequisites` stage before a version is tagged. It: - runs the Go unit tests twice on Linux, macOS, and Windows; -- runs the Java-related unit tests ten times to catch intermittent failures; -- runs all Go unit tests on Linux five more times; +- runs the Java-related unit tests ten times to catch intermittent failures, + because those are the suites where intermittent failures have appeared; - builds both Bomly binaries for every supported Linux, macOS, and Windows processor target. From a38946e4751644410cf2a1090f11e7fd7c9e5140 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 19 Aug 2026 23:40:09 -0400 Subject: [PATCH 07/13] feat(assurance): make subject areas the report's structure The published report is organised by subject rather than by the release stage a check ran in, because "what does this tell me about Bomly" is the question a reader arrives with; when a check ran is a label on the check, not the shape of the report. - AreaReport now carries the checks and evidence claims that belong to it, and an area earns a section when it has either. An area proven only by evidence takes the status of the checks backing those claims, so it can never look better than they are. - Areas are emitted in catalog order, which is now a deliberate reading order, and their titles and descriptions are written for someone technical who does not work on Bomly: "Scanning real projects", "What you download", "Handling broken input", rather than internal shorthand. Co-Authored-By: Claude Fable 5 --- dev-docs/RELEASE_ASSURANCE.md | 5 +- docs/ASSURANCE.md | 14 +++- docs/assurance/catalog.json | 84 +++++++++---------- internal/assurance/aggregate.go | 45 +++++++--- internal/assurance/report.go | 17 ++-- .../testdata/golden/all-pass.report.json | 18 +++- .../testdata/golden/mixed-failure.report.json | 18 +++- 7 files changed, 134 insertions(+), 67 deletions(-) diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index be818400..9822d643 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -117,7 +117,10 @@ dispatch then tells the landing page a new report is available. 1. Add the entry to `docs/assurance/catalog.json`: `id`, `title`, `area`, `stage`, `level`, `description`, `source`, optional `expected_instances`, `reproduce`, and — required — `proves` and `limitations` in plain language. - Checks are sorted by `id`. + Checks are sorted by `id`. The `area` decides which section of the published + page the check appears in, and the order of `areas` in the catalog is the + order those sections are read in, so put a new area where it belongs in the + narrative rather than at the end. 2. Emit a result from the workflow named in `source`, and upload it as an `assurance-` artifact. 3. If the check backs a public claim, add an `evidence` entry pointing at it diff --git a/docs/ASSURANCE.md b/docs/ASSURANCE.md index 9a54149b..3e72d6ac 100644 --- a/docs/ASSURANCE.md +++ b/docs/ASSURANCE.md @@ -21,12 +21,22 @@ need a fix in the source tree, run **before** a version number exists, so a problem is fixed by a normal pull request instead of by a broken release. Checks that describe the published files can only run once those files exist. -## What each check covers +## How the report is organised + +The report is grouped by subject, not by release stage: scanning real projects, +dependency graphs, policy decisions, reachability, upgrade guidance, comparing +two scans, other scan targets, SBOM compatibility, running everywhere, handling +broken input, speed and repeatability, what you download, and installing it. +Each section holds the checks that cover it and the evidence claims made about +it. Every check still states which stage it ran in, because that is what +decides how a failure gets fixed. The full list, with what every check proves and what it does not, lives in the machine-readable catalog at [`docs/assurance/catalog.json`](assurance/catalog.json). The assurance page -renders the same catalog, so the page and the repository can never disagree. +renders the same catalog, so the page and the repository can never disagree — +including the section order, which is the order the catalog declares its areas +in. Highlights: diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json index f3e3dbc5..dfd6be37 100644 --- a/docs/assurance/catalog.json +++ b/docs/assurance/catalog.json @@ -2,74 +2,74 @@ "schema_version": "bomly.assurance-catalog/v1", "areas": [ { - "id": "dependency-graph", - "title": "Dependency graphs", - "description": "Resolving the packages a project depends on, and how they got there." + "id": "end-to-end", + "title": "Scanning real projects", + "description": "Running the real command line against pinned example projects and comparing the entire result with a recorded one." }, { - "id": "diff", - "title": "Change detection", - "description": "Comparing two scans and classifying what changed." + "id": "dependency-graph", + "title": "Dependency graphs", + "description": "Working out which packages a project depends on, and how each one got there." }, { - "id": "end-to-end", - "title": "End-to-end behavior", - "description": "Running the real command line against real projects and comparing the whole result." + "id": "policy", + "title": "Policy decisions", + "description": "Deciding what counts as a problem: licences, vulnerabilities, baselines, and packages that change where they come from." }, { - "id": "framework", - "title": "Assurance framework", - "description": "The catalog and contracts this report is built from." + "id": "reachability", + "title": "Reachability", + "description": "Judging whether a vulnerable package is actually used by the code that depends on it." }, { - "id": "installation", - "title": "Installation", - "description": "Getting the published release onto a machine the documented ways." + "id": "remediation", + "title": "Upgrade guidance", + "description": "The fixes Bomly recommends, worked out without changing anything in your project." }, { - "id": "parsers", - "title": "Input parsing safety", - "description": "Reading untrusted project, configuration, and SBOM files without crashing." + "id": "diff", + "title": "Comparing two scans", + "description": "Telling what genuinely changed between two scans, and what only looks new." }, { - "id": "performance", - "title": "Speed and stability", - "description": "How long a repeated scan takes and whether it returns the same answer." + "id": "targets", + "title": "Other scan targets", + "description": "Scanning things that are not a local source tree, such as container images." }, { - "id": "policy", - "title": "Policy and guidance", - "description": "Evaluating license, vulnerability, baseline, and source-change policy." + "id": "sbom", + "title": "SBOM compatibility", + "description": "Reading SBOM documents other tools produce, and writing ones they accept." }, { - "id": "reachability", - "title": "Reachability", - "description": "Deciding whether a vulnerable package is actually used." + "id": "stability", + "title": "Running everywhere", + "description": "Behaving the same on Linux, macOS, and Windows, and building for every processor a release supports." }, { - "id": "release-integrity", - "title": "Release integrity", - "description": "Whether the published files are complete, unmodified, and signed." + "id": "parsers", + "title": "Handling broken input", + "description": "Reading damaged or hostile project, configuration, and SBOM files without crashing." }, { - "id": "remediation", - "title": "Remediation", - "description": "The upgrade guidance Bomly derives, without changing your project." + "id": "performance", + "title": "Speed and repeatability", + "description": "How long a repeated scan takes, and whether it returns the same answer every time." }, { - "id": "sbom", - "title": "SBOM interoperability", - "description": "Reading and writing SBOM documents other tools accept." + "id": "release-integrity", + "title": "What you download", + "description": "Whether the published files are complete, unaltered, and signed by this project's release pipeline." }, { - "id": "stability", - "title": "Platform stability", - "description": "Behaving the same on every supported operating system and processor." + "id": "installation", + "title": "Installing it", + "description": "Whether the documented ways of installing this release actually work." }, { - "id": "targets", - "title": "Scan targets", - "description": "Scanning things other than a local source tree, such as container images." + "id": "framework", + "title": "This report itself", + "description": "The catalog and contracts the report is built from, and the check that keeps them honest." } ], "checks": [ diff --git a/internal/assurance/aggregate.go b/internal/assurance/aggregate.go index 99e7837b..546fa804 100644 --- a/internal/assurance/aggregate.go +++ b/internal/assurance/aggregate.go @@ -104,25 +104,37 @@ func BuildReport(catalog Catalog, results []CheckResult, opts BuildOptions) Repo } report.Verdict = summarize(report.Checks) + if opts.IncludeEvidence { + report.Evidence = buildEvidence(catalog, report, selected) + } + + // Areas are listed in catalog order, which is the order the published page + // reads in. An area earns a section when it has checks or evidence. for _, area := range catalog.Areas { + reported := AreaReport{ID: area.ID, Title: area.Title, Description: area.Description} var areaChecks []ReportCheck for _, check := range report.Checks { if check.Area == area.ID { areaChecks = append(areaChecks, check) + reported.CheckIDs = append(reported.CheckIDs, check.ID) } } - if len(areaChecks) == 0 { + for _, evidence := range report.Evidence { + if evidence.Area == area.ID { + reported.EvidenceIDs = append(reported.EvidenceIDs, evidence.ID) + } + } + if len(reported.CheckIDs) == 0 && len(reported.EvidenceIDs) == 0 { continue } - verdict := summarize(areaChecks) - report.Areas = append(report.Areas, AreaReport{ - ID: area.ID, Title: area.Title, Description: area.Description, - Status: verdict.Overall, Verdict: verdict, - }) - } - - if opts.IncludeEvidence { - report.Evidence = buildEvidence(catalog, report, selected) + reported.Verdict = summarize(areaChecks) + reported.Status = reported.Verdict.Overall + // An area proven only by evidence takes the status of the checks + // backing those claims, so it can never look better than they are. + if len(areaChecks) == 0 { + reported.Status = evidenceAreaStatus(report, reported.EvidenceIDs) + } + report.Areas = append(report.Areas, reported) } report.Coverage = buildCoverage(catalog, report, selected) if opts.Previous != nil { @@ -298,6 +310,19 @@ func summarize(checks []ReportCheck) Verdict { return verdict } +// evidenceAreaStatus is the worst status among the claims in an area. +func evidenceAreaStatus(report Report, ids []string) Status { + status := StatusPass + for _, id := range ids { + for _, evidence := range report.Evidence { + if evidence.ID == id { + status = Worse(status, evidence.Status) + } + } + } + return status +} + func buildEvidence(catalog Catalog, report Report, selected map[Stage]struct{}) []ReportEvidence { var entries []ReportEvidence for _, evidence := range catalog.Evidence { diff --git a/internal/assurance/report.go b/internal/assurance/report.go index c78d02b4..fbba3d80 100644 --- a/internal/assurance/report.go +++ b/internal/assurance/report.go @@ -51,13 +51,18 @@ type StageReport struct { CheckIDs []string `json:"check_ids"` } -// AreaReport summarises one catalog area. +// AreaReport summarises one subject area: the checks that cover it and the +// evidence claims made about it. Areas are what the published report is +// organised by, because "what does this tell me about Bomly" is a more useful +// question for a reader than "when in the release did this run". type AreaReport struct { - ID string `json:"id"` - Title string `json:"title"` - Description string `json:"description"` - Status Status `json:"status"` - Verdict Verdict `json:"verdict"` + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Status Status `json:"status"` + Verdict Verdict `json:"verdict"` + CheckIDs []string `json:"check_ids,omitempty"` + EvidenceIDs []string `json:"evidence_ids,omitempty"` } // InstanceReport is one reported leg of a check. diff --git a/internal/assurance/testdata/golden/all-pass.report.json b/internal/assurance/testdata/golden/all-pass.report.json index 38b4ba22..e9c00038 100644 --- a/internal/assurance/testdata/golden/all-pass.report.json +++ b/internal/assurance/testdata/golden/all-pass.report.json @@ -82,7 +82,13 @@ "degraded": 0, "skipped": 0, "missing": 0 - } + }, + "check_ids": [ + "smoke" + ], + "evidence_ids": [ + "graph-go" + ] }, { "id": "release-integrity", @@ -97,7 +103,10 @@ "degraded": 0, "skipped": 0, "missing": 0 - } + }, + "check_ids": [ + "release-checksums" + ] }, { "id": "performance", @@ -112,7 +121,10 @@ "degraded": 0, "skipped": 0, "missing": 0 - } + }, + "check_ids": [ + "perf-samples" + ] } ], "checks": [ diff --git a/internal/assurance/testdata/golden/mixed-failure.report.json b/internal/assurance/testdata/golden/mixed-failure.report.json index 29bfb088..0c60b1a8 100644 --- a/internal/assurance/testdata/golden/mixed-failure.report.json +++ b/internal/assurance/testdata/golden/mixed-failure.report.json @@ -103,7 +103,13 @@ "missing_checks": [ "smoke" ] - } + }, + "check_ids": [ + "smoke" + ], + "evidence_ids": [ + "graph-go" + ] }, { "id": "release-integrity", @@ -121,7 +127,10 @@ "gates_failed": [ "release-checksums" ] - } + }, + "check_ids": [ + "release-checksums" + ] }, { "id": "performance", @@ -139,7 +148,10 @@ "advisories_failed": [ "perf-samples" ] - } + }, + "check_ids": [ + "perf-samples" + ] } ], "checks": [ From c3f7c5f3a160f6394aefc7e547e3e6fbc214bc9a Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Thu, 20 Aug 2026 00:14:18 -0400 Subject: [PATCH 08/13] docs(assurance): one vocabulary, and a rule for changing a schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The public document described checks and evidence as two species of thing, which is confusing: evidence is what the report *is*. It now says it once — a claim is a statement about a release, a check is what asserts it, and the report as a whole is the evidence — and drops the maintainer-facing framing of gates and stages from the reader's page. Also documents the schema compatibility rule, because two of these documents are read by bomly.dev: adding an optional field keeps the version, removing or repurposing one raises it, and the site has to learn the new shape before the version moves or reports stop appearing. TestSchemaVersionsArePinned fails on any change to those strings so raising one is always deliberate. Co-Authored-By: Claude Fable 5 --- dev-docs/RELEASE_ASSURANCE.md | 20 ++++++++++++++ docs/ASSURANCE.md | 39 ++++++++++++++++------------ internal/assurance/assurance_test.go | 25 ++++++++++++++++++ 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index 9822d643..727a5598 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -167,6 +167,26 @@ tracking issue and the published report records the failure honestly. the report without blocking a release. Use it for the first release after a framework change, then turn it back on. +## Changing a schema + +Four documents carry a schema version: the check result, the catalog, the +report, and the index. The report and index are read by bomly.dev, so their +versions are a contract with another repository. + +- Adding an optional field keeps the version. Every consumer ignores what it + does not know. +- Removing a field, renaming one, or changing what one means raises the version + — and the site has to learn the new shape *first*, or reports stop appearing. + The order is: teach `bomly-landing-page` (`SUPPORTED_REPORT_SCHEMAS` in + `lib/assurance.ts`, `REPORT_SCHEMAS` in `scripts/sync-assurance.mjs`) to + render both versions, ship that, then raise the version here. +- The site keeps rendering older reports it already mirrored, and skips reports + whose version it does not know with a warning, so a mismatch shows up as one + release missing from the page rather than a broken page. + +`TestSchemaVersionsArePinned` fails on any change to these strings, so raising +one is always deliberate. + ## What the automation needs - The Bomly Release app needs **Issues: Read and write** on `bomly-cli` for the diff --git a/docs/ASSURANCE.md b/docs/ASSURANCE.md index 3e72d6ac..a376d129 100644 --- a/docs/ASSURANCE.md +++ b/docs/ASSURANCE.md @@ -57,25 +57,32 @@ Highlights: - **Speed and stability.** The same scan is repeated with a cold and a warm cache to record timing and confirm the output does not change. -Checks are either **gates**, which stop a release, or **advisory**, which are -reported but never block. The report always says which is which, and every -count links to the job that produced it, so any number on the page is one click -from its log. +Some checks stop a release when they fail; others are recorded without +blocking. Either way, a claim that was not confirmed is shown on the page and +raises a tracking issue for maintainers, so a known problem in a published +release is never quietly dropped. Every count links to the job that produced +it, so any number on the page is one click from its log. -## Evidence claims +## Claims, checks, and evidence -Alongside the checks, the catalog carries **evidence claims**: specific -statements about Bomly's behavior, each with a pinned input — a public -repository at a recorded commit, or a checked-in fixture — the exact command to -reproduce it, the checksummed result file it is compared against, and its -stated limitations. Each claim names the check that backs it, so the report can -show whether that claim still held for the release you are looking at. +The report uses one vocabulary, and it is worth being precise about it: -A claim earns its own entry only when it adds something a check result cannot: -the pinned input and the committed file. Claims that would just restate their -check are not repeated here. +- A **claim** is one specific statement about what a release does — that it + reads an npm lockfile correctly, or that the files you download are the ones + we signed. +- A **check** is what asserts a claim: something that ran against that exact + release and produced a result you can open. +- The **report as a whole** is the evidence: every claim, whether it was + confirmed, and a link to what confirmed it. -Reproduce any claim from a repository checkout: +Some claims are asserted directly by a check. Others are asserted by comparing +a pinned input — a public repository at a recorded commit, or a checked-in +fixture — against a checksummed result file, and name the check that performed +that comparison. Both appear as claims on the page, in the section they belong +to. A claim earns a separate entry only when it adds something the check cannot +say on its own. + +Check any claim yourself from a repository checkout: ```sh make assurance-catalog @@ -89,7 +96,7 @@ The first command validates the whole catalog, including the checksums of every expected-result file it names. The second prints one claim with its reproduction command. -## What a green report does not mean +## What a verified claim does not mean - A passing report describes the checks listed in the catalog. Software can still fail in ways nobody has written a check for. diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go index 285dec9b..e2a452c2 100644 --- a/internal/assurance/assurance_test.go +++ b/internal/assurance/assurance_test.go @@ -411,3 +411,28 @@ func TestMatchJobURLPicksTheRunningJobOnThisRunner(t *testing.T) { t.Fatal("expected malformed payload to be reported") } } + +// TestSchemaVersionsArePinned makes a schema change a deliberate act. The +// published reports are read by bomly.dev, which renders a known list of +// versions: adding an optional field keeps the version, while removing or +// repurposing one has to raise it and be taught to the site first. +func TestSchemaVersionsArePinned(t *testing.T) { + for name, actual := range map[string]string{ + "check": CheckSchema, + "catalog": CatalogSchema, + "report": ReportSchema, + "index": IndexSchema, + } { + expected := map[string]string{ + "check": "bomly.assurance-check/v1", + "catalog": "bomly.assurance-catalog/v1", + "report": "bomly.assurance-report/v1", + "index": "bomly.assurance-index/v1", + }[name] + if actual != expected { + t.Errorf("%s schema is %q, want %q; raising it requires teaching bomly.dev the new shape first "+ + "(SUPPORTED_REPORT_SCHEMAS in lib/assurance.ts and REPORT_SCHEMAS in scripts/sync-assurance.mjs)", + name, actual, expected) + } + } +} From a1701cf1dcc3f63d8f41abdd9a9f6139281e0f3c Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Thu, 20 Aug 2026 00:46:21 -0400 Subject: [PATCH 09/13] feat(assurance): symmetric claims, and coverage as one stamp per ecosystem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two shapes the published report could not render uniformly. Evidence claims had no description, so they could not be laid out like the claims a check asserts directly. Both kinds now carry the same fields — title, description, proves, limitations — and the catalog requires it, which is what lets the page render one shape for every claim. Coverage was a grid of ecosystem against check. That made "The released binary scans real projects" look like it had fifteen gaps, when it deliberately re-runs a representative subset with the shipped binary and the full ecosystem matrix runs before the tag. The ecosystem was covered either way; the grid was answering a question nobody asked. Coverage is now one stamp per ecosystem, taking the worst status of every check that exercised it, so a passing stamp still cannot hide a failure. The report schema stays at v1: it has not been published yet, so v1 is still being defined rather than changed. Co-Authored-By: Claude Fable 5 --- dev-docs/RELEASE_ASSURANCE.md | 13 ++++++++ docs/assurance/catalog.json | 21 +++++++++++++ internal/assurance/aggregate.go | 30 +++++++++---------- internal/assurance/catalog.go | 7 +++-- internal/assurance/catalog_test.go | 26 ++++++++-------- internal/assurance/report.go | 19 +++++++----- internal/assurance/testdata/catalog.json | 1 + .../testdata/golden/all-pass.report.json | 17 +++++------ .../testdata/golden/mixed-failure.report.json | 17 +++++------ 9 files changed, 94 insertions(+), 57 deletions(-) diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index 727a5598..41c17c42 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -125,6 +125,9 @@ dispatch then tells the landing page a new report is available. `assurance-` artifact. 3. If the check backs a public claim, add an `evidence` entry pointing at it with `check_id` (and `instance`, when one specific leg proves the claim). + Both kinds of entry carry the same fields — title, description, proves, + limitations — because the published page renders one shape for every claim, + whichever way it is asserted. 4. Run `make assurance-catalog`, then `go test ./internal/assurance/`. Refresh goldens with `go test ./internal/assurance/ -update` when the report shape changes. @@ -167,6 +170,16 @@ tracking issue and the published report records the failure honestly. the report without blocking a release. Use it for the first release after a framework change, then turn it back on. +## Ecosystem coverage + +`expected_instances[].ecosystems` is what puts a language or package format on +the report's coverage list. Coverage is a single stamp per ecosystem, taking +the worst status of every check that exercised it — not a grid of which check +covered what. The reader's question is "was my ecosystem covered", and +answering it per check invites the false conclusion that a blank cell is a gap +when another check covered it. Adding an ecosystem to any check's instances is +enough to have it appear. + ## Changing a schema Four documents carry a schema version: the check result, the catalog, the diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json index dfd6be37..085edb0f 100644 --- a/docs/assurance/catalog.json +++ b/docs/assurance/catalog.json @@ -723,6 +723,7 @@ "id": "baseline-policy", "title": "Finding baseline lifecycle", "area": "policy", + "description": "Records a package finding in a project baseline, then scans again and compares the whole result with a recorded one.", "evidence_level": "deterministic", "check_id": "smoke", "instance": "go", @@ -770,6 +771,7 @@ "id": "container-inventory", "title": "Container package inventory", "area": "targets", + "description": "Scans a public container image and compares the operating-system packages found with a recorded inventory.", "evidence_level": "snapshot", "check_id": "smoke", "instance": "container", @@ -812,6 +814,7 @@ "id": "degraded-detector-fallback", "title": "Visible detector fallback", "area": "dependency-graph", + "description": "Forces a detector to fall back to a second technique and checks that the result says so instead of hiding it.", "evidence_level": "deterministic", "check_id": "unit-portable", "inputs": [ @@ -851,6 +854,7 @@ "id": "graph-bun", "title": "Bun lockfile graph", "area": "dependency-graph", + "description": "Scans a pinned Bun example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "node", @@ -897,6 +901,7 @@ "id": "graph-go", "title": "Go module graph", "area": "dependency-graph", + "description": "Scans a pinned Go example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "go", @@ -944,6 +949,7 @@ "id": "graph-maven", "title": "Maven dependency graph", "area": "dependency-graph", + "description": "Scans a pinned Maven example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "java", @@ -992,6 +998,7 @@ "id": "graph-npm", "title": "npm lockfile graph", "area": "dependency-graph", + "description": "Scans a pinned npm example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "node", @@ -1039,6 +1046,7 @@ "id": "graph-pnpm", "title": "pnpm lockfile graph", "area": "dependency-graph", + "description": "Scans a pinned pnpm example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "node", @@ -1086,6 +1094,7 @@ "id": "graph-python", "title": "Python requirements graph", "area": "dependency-graph", + "description": "Scans a pinned Python example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "python", @@ -1132,6 +1141,7 @@ "id": "graph-yarn", "title": "Yarn lockfile graph", "area": "dependency-graph", + "description": "Scans a pinned Yarn example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "node", @@ -1179,6 +1189,7 @@ "id": "license-policy", "title": "Complex and invalid SPDX policy", "area": "policy", + "description": "Evaluates licence policy against a matrix of SPDX expressions, including combined, nested, exception, custom, and invalid ones.", "evidence_level": "deterministic", "check_id": "unit-portable", "inputs": [ @@ -1218,6 +1229,7 @@ "id": "persisted-risk", "title": "Risk that persists across a version change", "area": "diff", + "description": "Compares two scans across a package version change and checks how a finding that survives the change is classified.", "evidence_level": "deterministic", "check_id": "unit-portable", "inputs": [ @@ -1257,6 +1269,7 @@ "id": "reachability-go", "title": "Go vulnerability reachability", "area": "reachability", + "description": "Analyses a pinned Go project and checks the reachability evidence attached to each matched vulnerability.", "evidence_level": "live-service", "check_id": "smoke", "instance": "go-reachability", @@ -1304,6 +1317,7 @@ "id": "reachability-java", "title": "Java package reachability", "area": "reachability", + "description": "Analyses a pinned Java project and checks the reachability evidence attached to each matched vulnerability.", "evidence_level": "live-service", "check_id": "smoke", "instance": "java-reachability", @@ -1352,6 +1366,7 @@ "id": "reachability-node", "title": "JavaScript package reachability", "area": "reachability", + "description": "Analyses a pinned JavaScript project and checks the reachability evidence attached to each matched vulnerability.", "evidence_level": "live-service", "check_id": "smoke", "instance": "node-reachability", @@ -1399,6 +1414,7 @@ "id": "reachability-python", "title": "Python package reachability", "area": "reachability", + "description": "Analyses a pinned Python project and checks the reachability evidence attached to each matched vulnerability.", "evidence_level": "live-service", "check_id": "smoke", "instance": "python-reachability", @@ -1445,6 +1461,7 @@ "id": "remediation-read-only", "title": "Canonical read-only remediation guidance", "area": "remediation", + "description": "Derives upgrade guidance for a set of findings and checks the fix status, versions, and per-occurrence advice it produces.", "evidence_level": "deterministic", "check_id": "unit-portable", "inputs": [ @@ -1484,6 +1501,7 @@ "id": "sbom-cyclonedx-ingest", "title": "CycloneDX 1.6 ingestion", "area": "sbom", + "description": "Reads a checked-in CycloneDX document and compares the dependency graph it produces with a recorded one.", "evidence_level": "deterministic", "check_id": "smoke", "instance": "sbom", @@ -1529,6 +1547,7 @@ "id": "sbom-spdx-ingest", "title": "SPDX 2.3 ingestion", "area": "sbom", + "description": "Reads a checked-in SPDX document and compares the dependency graph it produces with a recorded one.", "evidence_level": "deterministic", "check_id": "smoke", "instance": "sbom", @@ -1574,6 +1593,7 @@ "id": "source-change-policy", "title": "Registry-to-Git source-change review", "area": "policy", + "description": "Compares two scans of the same project where one package moved from a registry to a Git source, and checks how that is flagged.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "node-detail-policy", @@ -1627,6 +1647,7 @@ "id": "vulnerability-policy", "title": "Vulnerability policy constraints", "area": "policy", + "description": "Evaluates vulnerability policy against a matrix of severities, reachability tiers, known exploitation, and advisory allowlists.", "evidence_level": "deterministic", "check_id": "unit-portable", "inputs": [ diff --git a/internal/assurance/aggregate.go b/internal/assurance/aggregate.go index 546fa804..5b743daf 100644 --- a/internal/assurance/aggregate.go +++ b/internal/assurance/aggregate.go @@ -348,6 +348,7 @@ func buildEvidence(catalog Catalog, report Report, selected map[Stage]struct{}) } entries = append(entries, ReportEvidence{ ID: evidence.ID, Title: evidence.Title, Area: evidence.Area, + Description: evidence.Description, EvidenceLevel: evidence.EvidenceLevel, CheckID: evidence.CheckID, Instance: evidence.Instance, Status: status, Inputs: evidence.Inputs, RequiredTools: evidence.RequiredTools, Reproduce: evidence.Reproduce, @@ -358,9 +359,11 @@ func buildEvidence(catalog Catalog, report Report, selected map[Stage]struct{}) return entries } +// buildCoverage answers one question per ecosystem: was it exercised for this +// release, and did that hold? An ecosystem covered by several checks takes the +// worst of them, so a passing stamp cannot hide a failure elsewhere. func buildCoverage(catalog Catalog, report Report, selected map[Stage]struct{}) Coverage { - ecosystems := map[string]struct{}{} - var rows []CoverageRow + statuses := map[string]Status{} for _, check := range catalog.Checks { if _, wanted := selected[check.Stage]; !wanted { continue @@ -369,7 +372,6 @@ func buildCoverage(catalog Catalog, report Report, selected map[Stage]struct{}) if !found { continue } - cells := map[string]Status{} for _, expected := range check.ExpectedInstances { if len(expected.Ecosystems) == 0 { continue @@ -382,26 +384,24 @@ func buildCoverage(catalog Catalog, report Report, selected map[Stage]struct{}) } } for _, ecosystem := range expected.Ecosystems { - ecosystems[ecosystem] = struct{}{} - if existing, present := cells[ecosystem]; present { - cells[ecosystem] = Worse(existing, status) + if existing, present := statuses[ecosystem]; present { + statuses[ecosystem] = Worse(existing, status) continue } - cells[ecosystem] = status + statuses[ecosystem] = status } } - if len(cells) == 0 { - continue - } - rows = append(rows, CoverageRow{CheckID: check.ID, Title: check.Title, Cells: cells}) } - names := make([]string, 0, len(ecosystems)) - for ecosystem := range ecosystems { + names := make([]string, 0, len(statuses)) + for ecosystem := range statuses { names = append(names, ecosystem) } sort.Strings(names) - sort.Slice(rows, func(i, j int) bool { return rows[i].CheckID < rows[j].CheckID }) - return Coverage{Ecosystems: names, Rows: rows} + coverage := Coverage{} + for _, name := range names { + coverage.Ecosystems = append(coverage.Ecosystems, EcosystemCoverage{Name: name, Status: statuses[name]}) + } + return coverage } func collectRunners(results []CheckResult) []Runner { diff --git a/internal/assurance/catalog.go b/internal/assurance/catalog.go index d5cae7eb..f6e346fe 100644 --- a/internal/assurance/catalog.go +++ b/internal/assurance/catalog.go @@ -120,6 +120,7 @@ type Evidence struct { ID string `json:"id"` Title string `json:"title"` Area string `json:"area"` + Description string `json:"description"` EvidenceLevel EvidenceLevel `json:"evidence_level"` CheckID string `json:"check_id"` Instance string `json:"instance,omitempty"` @@ -260,8 +261,10 @@ func validateCheck(check Check, areas map[string]struct{}) error { } func validateEvidence(evidence Evidence, areas map[string]struct{}, checks map[string]Check) error { - if strings.TrimSpace(evidence.Title) == "" { - return errCatalog("title is required") + // Every claim carries the same fields whichever way it is asserted, so the + // published report can render one shape for all of them. + if strings.TrimSpace(evidence.Title) == "" || strings.TrimSpace(evidence.Description) == "" { + return errCatalog("title and description are required") } if _, exists := areas[evidence.Area]; !exists { return fmt.Errorf("unknown area %q", evidence.Area) diff --git a/internal/assurance/catalog_test.go b/internal/assurance/catalog_test.go index 44bf3863..d2210ce5 100644 --- a/internal/assurance/catalog_test.go +++ b/internal/assurance/catalog_test.go @@ -105,6 +105,7 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { }}, Evidence: []Evidence{{ ID: "graph-go", Title: "Go graph", Area: "end-to-end", + Description: "Scans a pinned example project and compares the result with a recorded one.", EvidenceLevel: EvidencePinnedInput, CheckID: "smoke", Inputs: []Input{{ Kind: "git", Location: "https://example.test/repo", @@ -124,18 +125,19 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { } cases := map[string]func(*Catalog){ - "bad schema": func(c *Catalog) { c.SchemaVersion = "other/v1" }, - "no areas": func(c *Catalog) { c.Areas = nil }, - "unknown check area": func(c *Catalog) { c.Checks[0].Area = "nowhere" }, - "bad stage": func(c *Catalog) { c.Checks[0].Stage = "later" }, - "bad level": func(c *Catalog) { c.Checks[0].Level = "blocking" }, - "no source": func(c *Catalog) { c.Checks[0].Source = Source{} }, - "blank claim": func(c *Catalog) { c.Checks[0].Proves = []string{" "} }, - "no limitations": func(c *Catalog) { c.Checks[0].Limitations = nil }, - "unknown backing": func(c *Catalog) { c.Evidence[0].CheckID = "nothing" }, - "undeclared instance": func(c *Catalog) { c.Evidence[0].Instance = "go" }, - "bad evidence level": func(c *Catalog) { c.Evidence[0].EvidenceLevel = "vibes" }, - "no inputs": func(c *Catalog) { c.Evidence[0].Inputs = nil }, + "bad schema": func(c *Catalog) { c.SchemaVersion = "other/v1" }, + "no areas": func(c *Catalog) { c.Areas = nil }, + "unknown check area": func(c *Catalog) { c.Checks[0].Area = "nowhere" }, + "bad stage": func(c *Catalog) { c.Checks[0].Stage = "later" }, + "bad level": func(c *Catalog) { c.Checks[0].Level = "blocking" }, + "no source": func(c *Catalog) { c.Checks[0].Source = Source{} }, + "blank claim": func(c *Catalog) { c.Checks[0].Proves = []string{" "} }, + "no limitations": func(c *Catalog) { c.Checks[0].Limitations = nil }, + "unknown backing": func(c *Catalog) { c.Evidence[0].CheckID = "nothing" }, + "evidence description": func(c *Catalog) { c.Evidence[0].Description = " " }, + "undeclared instance": func(c *Catalog) { c.Evidence[0].Instance = "go" }, + "bad evidence level": func(c *Catalog) { c.Evidence[0].EvidenceLevel = "vibes" }, + "no inputs": func(c *Catalog) { c.Evidence[0].Inputs = nil }, "git without revision": func(c *Catalog) { c.Evidence[0].Inputs = []Input{{Kind: "git", Location: "https://example.test/repo"}} }, diff --git a/internal/assurance/report.go b/internal/assurance/report.go index fbba3d80..2f7786d9 100644 --- a/internal/assurance/report.go +++ b/internal/assurance/report.go @@ -104,6 +104,7 @@ type ReportEvidence struct { ID string `json:"id"` Title string `json:"title"` Area string `json:"area"` + Description string `json:"description"` EvidenceLevel EvidenceLevel `json:"evidence_level"` CheckID string `json:"check_id"` Instance string `json:"instance,omitempty"` @@ -116,17 +117,19 @@ type ReportEvidence struct { Limitations []string `json:"limitations"` } -// CoverageRow is one check's status per ecosystem. -type CoverageRow struct { - CheckID string `json:"check_id"` - Title string `json:"title"` - Cells map[string]Status `json:"cells"` +// EcosystemCoverage is whether one language or package format was exercised +// for this release. Which check did the exercising is deliberately not part of +// it: the reader's question is "was my ecosystem covered", and answering it +// with a grid of checks invites the false conclusion that a blank cell is a +// gap when the ecosystem was covered by another check. +type EcosystemCoverage struct { + Name string `json:"name"` + Status Status `json:"status"` } -// Coverage maps ecosystems against the checks that exercise them. +// Coverage lists every ecosystem any check exercised, worst status first. type Coverage struct { - Ecosystems []string `json:"ecosystems"` - Rows []CoverageRow `json:"rows"` + Ecosystems []EcosystemCoverage `json:"ecosystems"` } // MetricTrend compares one metric against the previous release's report. diff --git a/internal/assurance/testdata/catalog.json b/internal/assurance/testdata/catalog.json index 61ca8bfe..98993f3d 100644 --- a/internal/assurance/testdata/catalog.json +++ b/internal/assurance/testdata/catalog.json @@ -64,6 +64,7 @@ "id": "graph-go", "title": "Go module graph", "area": "end-to-end", + "description": "Scans a pinned Go example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "go", diff --git a/internal/assurance/testdata/golden/all-pass.report.json b/internal/assurance/testdata/golden/all-pass.report.json index e9c00038..6b07becd 100644 --- a/internal/assurance/testdata/golden/all-pass.report.json +++ b/internal/assurance/testdata/golden/all-pass.report.json @@ -317,6 +317,7 @@ "id": "graph-go", "title": "Go module graph", "area": "end-to-end", + "description": "Scans a pinned Go example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "go", @@ -355,17 +356,13 @@ ], "coverage": { "ecosystems": [ - "Go", - "JavaScript" - ], - "rows": [ { - "check_id": "smoke", - "title": "End-to-end scans of real projects", - "cells": { - "Go": "pass", - "JavaScript": "pass" - } + "name": "Go", + "status": "pass" + }, + { + "name": "JavaScript", + "status": "pass" } ] }, diff --git a/internal/assurance/testdata/golden/mixed-failure.report.json b/internal/assurance/testdata/golden/mixed-failure.report.json index 0c60b1a8..7b7aa4e7 100644 --- a/internal/assurance/testdata/golden/mixed-failure.report.json +++ b/internal/assurance/testdata/golden/mixed-failure.report.json @@ -329,6 +329,7 @@ "id": "graph-go", "title": "Go module graph", "area": "end-to-end", + "description": "Scans a pinned Go example project and compares the whole dependency graph with a recorded one.", "evidence_level": "pinned-input", "check_id": "smoke", "instance": "go", @@ -367,17 +368,13 @@ ], "coverage": { "ecosystems": [ - "Go", - "JavaScript" - ], - "rows": [ { - "check_id": "smoke", - "title": "End-to-end scans of real projects", - "cells": { - "Go": "pass", - "JavaScript": "missing" - } + "name": "Go", + "status": "pass" + }, + { + "name": "JavaScript", + "status": "missing" } ] }, From aff590d5711f0c91f0dc930712f61428b04040d8 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Thu, 20 Aug 2026 01:25:52 -0400 Subject: [PATCH 10/13] docs(assurance): finish the agent-facing documentation Gaps found on a pass over the agent files against what the framework actually added. - The architecture package map did not list internal/assurance at all. - `make assurance-report` was undocumented. - The smoke-test rules said to keep the two workflow slice matrices in sync but not the catalog's expected_instances, which is the third place a slice has to be declared and the one that decides ecosystem coverage. They also did not mention that regenerating goldens invalidates the checksums claims pin. - The feature checklist had no assurance step, so a new user-visible feature could ship without anyone asking whether it makes a claim worth publishing. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 9 +++++++++ CLAUDE.md | 9 +++++++++ dev-docs/ARCHITECTURE.md | 1 + 3 files changed, 19 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 732b0be0..18e3a26c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,7 @@ make fuzz FUZZTIME=5s # run every registered fuzz target with a short per-tar make benchmark # run the hidden local dependency-graph benchmark make benchmark-report # analyze local benchmark artifacts with Copilot CLI make assurance-catalog # validate the release assurance catalog (docs/assurance/catalog.json) +make assurance-report # render a fixture assurance report into .assurance/ for local preview make run ARGS="scan" # go run ./cmd/bomly make generate # regenerate config reference, JSON schemas, schema docs, support matrix, and component docs (binary-driven) ``` @@ -231,6 +232,8 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai - Scan cases come from `test/smoke/testdata/scan_targets.json`; keep it in sync with `internal/benchmark/testdata/scan_targets.json` (the benchmark target list) when cases change. - Pin every scan case's detectors with `--detectors`; normalize volatile fields in `helpers_test.go::normalizeJSON` before goldens. - Register new tests in both slice matrices (`smoke.yml` and exactly one slice in `update-smoke-goldens.yml`); `go test -run` elements are unanchored regexes — use `$` anchors to keep slice ownership exact. +- A new slice also needs an entry in the `smoke` check's `expected_instances` in `docs/assurance/catalog.json` (with its `ecosystems`, which is what puts an ecosystem on the report's coverage list). `TestCatalogSmokeInstancesMatchWorkflowMatrix` fails when the two drift. +- Regenerating goldens invalidates the checksums the catalog's claims pin. `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the catalog with them; do the same when refreshing by hand. - `TestExamplePluginFixtureCompiles` runs in `make test` and must keep compiling against the pinned `bomly-dev/bomly-sdk` release; update the fixture source when the SDK contract changes. ## Release assurance @@ -316,6 +319,12 @@ If a new analyzer / matcher / detector produces deterministic output for a fixed Any new user-visible feature needs a smoke case under `test/smoke/` — follow the golden/normalizer/slice-matrix rules in the Smoke tests section above. +### Release assurance + +Ask whether the feature makes a claim worth publishing. If it does, add it to `docs/assurance/catalog.json` — a `check` when something new runs for every release, an `evidence` entry when a pinned input and a committed result file prove a specific behavior. Both carry the same fields (`title`, `description`, `proves`, `limitations`), because the public page renders one shape for every claim. + +A declared check with no result is reported as `missing` and blocks its stage, so add the catalog entry and the workflow step that emits it together. + ### Documentation - `make generate` regenerates `docs/CONFIG_REFERENCE.md`, `docs/schemas/*`, `docs/SUPPORT_MATRIX.md`, and the component docs through the built binary. Run it whenever `internal/config/config.go` or `internal/output/*` change, or when the pinned SDK version (catalog / support-matrix data) is bumped. diff --git a/CLAUDE.md b/CLAUDE.md index c3ad3a22..6daf1b9a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,6 +21,7 @@ make fuzz FUZZTIME=5s # run every registered fuzz target with a short per-tar make benchmark # run the hidden local dependency-graph benchmark make benchmark-report # analyze local benchmark artifacts with Copilot CLI make assurance-catalog # validate the release assurance catalog (docs/assurance/catalog.json) +make assurance-report # render a fixture assurance report into .assurance/ for local preview make run ARGS="scan" # go run ./cmd/bomly make generate # regenerate config reference, JSON schemas, schema docs, support matrix, and component docs (binary-driven) ``` @@ -231,6 +232,8 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai - Scan cases come from `test/smoke/testdata/scan_targets.json`; keep it in sync with `internal/benchmark/testdata/scan_targets.json` (the benchmark target list) when cases change. - Pin every scan case's detectors with `--detectors`; normalize volatile fields in `helpers_test.go::normalizeJSON` before goldens. - Register new tests in both slice matrices (`smoke.yml` and exactly one slice in `update-smoke-goldens.yml`); `go test -run` elements are unanchored regexes — use `$` anchors to keep slice ownership exact. +- A new slice also needs an entry in the `smoke` check's `expected_instances` in `docs/assurance/catalog.json` (with its `ecosystems`, which is what puts an ecosystem on the report's coverage list). `TestCatalogSmokeInstancesMatchWorkflowMatrix` fails when the two drift. +- Regenerating goldens invalidates the checksums the catalog's claims pin. `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the catalog with them; do the same when refreshing by hand. - `TestExamplePluginFixtureCompiles` runs in `make test` and must keep compiling against the pinned `bomly-dev/bomly-sdk` release; update the fixture source when the SDK contract changes. ## Release assurance @@ -316,6 +319,12 @@ If a new analyzer / matcher / detector produces deterministic output for a fixed Any new user-visible feature needs a smoke case under `test/smoke/` — follow the golden/normalizer/slice-matrix rules in the Smoke tests section above. +### Release assurance + +Ask whether the feature makes a claim worth publishing. If it does, add it to `docs/assurance/catalog.json` — a `check` when something new runs for every release, an `evidence` entry when a pinned input and a committed result file prove a specific behavior. Both carry the same fields (`title`, `description`, `proves`, `limitations`), because the public page renders one shape for every claim. + +A declared check with no result is reported as `missing` and blocks its stage, so add the catalog entry and the workflow step that emits it together. + ### Documentation - `make generate` regenerates `docs/CONFIG_REFERENCE.md`, `docs/schemas/*`, `docs/SUPPORT_MATRIX.md`, and the component docs through the built binary. Run it whenever `internal/config/config.go` or `internal/output/*` change, or when the pinned SDK version (catalog / support-matrix data) is bumped. diff --git a/dev-docs/ARCHITECTURE.md b/dev-docs/ARCHITECTURE.md index b48a6812..864a5325 100644 --- a/dev-docs/ARCHITECTURE.md +++ b/dev-docs/ARCHITECTURE.md @@ -237,6 +237,7 @@ Cache failures are non-fatal. The command should warn and continue rather than f | `internal/engine/scan` | Scan command pipeline API | | `internal/output` | Text, JSON, SARIF rendering, plus structured response payloads and schema generation | | `internal/sbom` | SPDX and CycloneDX codecs | +| `internal/assurance` | Release assurance framework: check-result contract, catalog, report and index generation, release-asset verification, plus the `sbominterop` and `perfrun` check tools and the `cmd` entry point | | `internal/benchmark` | Hidden local dependency-graph benchmark, baseline comparison, scoring, and embedded presets | | `sdk` | Shared domain types | | `internal/plugin` | Managed plugin manifests, installation, verification, store state, adapters, and runtime glue | From 9965c6fa974353f19df6386c892d57c5c9c720f0 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Mon, 24 Aug 2026 23:23:38 -0700 Subject: [PATCH 11/13] fix(assurance): act on the review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six review comments, all still valid against the rebased branch. The one that mattered: `verify-release` emitted the checksum result and then extracted and ran the binaries regardless, so a release whose archive did not match SHA256SUMS still had its binary executed by the tool whose job is to decide whether that file can be trusted. The probe now stops when this platform's archives are mismatched or absent from the checksum list, and records release-binaries as failed with that reason. `loadContext` discarded the error from LoadCatalog and returned a nil catalog, so a catalog that existed but did not parse was indistinguishable from a check that was not declared — and a gate check could be emitted without the level the verdict depends on. It now reports the parse failure; a missing catalog still leaves the context empty, so emitting from outside a checkout with an explicit --stage and --level keeps working. Also: workflow inputs reach `run` blocks through `env` rather than template expansion, in the two places that interpolated them; the fuzz step passes --details-jsonl only when the file exists, since a fuzz run that dies early never writes it and the check still has to report; the coverage comment described an ordering the code does not produce; and the public page said every unconfirmed claim raises a tracking issue, when only a failure found after publication does — earlier ones stop the release instead. Co-Authored-By: Claude Fable 5 --- .github/workflows/assurance-prerequisites.yml | 2 +- .github/workflows/fuzz.yml | 12 ++++- docs/ASSURANCE.md | 8 +-- internal/assurance/assurance_test.go | 32 ++++++++++++ internal/assurance/cmd/commands.go | 50 ++++++++++++++++--- internal/assurance/cmd/main.go | 21 ++++++-- internal/assurance/releaseassets.go | 27 ++++++++++ internal/assurance/report.go | 3 +- 8 files changed, 135 insertions(+), 20 deletions(-) diff --git a/.github/workflows/assurance-prerequisites.yml b/.github/workflows/assurance-prerequisites.yml index f39a1aed..af64c556 100644 --- a/.github/workflows/assurance-prerequisites.yml +++ b/.github/workflows/assurance-prerequisites.yml @@ -157,7 +157,7 @@ jobs: run: | go run ./internal/assurance/cmd verdict \ --results assurance-results --stage prerequisites \ - --tag '${{ inputs.assurance_tag }}' \ + --tag "${BOMLY_ASSURANCE_TAG}" \ --json assurance-results/stage-prerequisites.json --step-summary - name: Upload the stage report diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index aa9c32e2..4fe2c268 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -61,7 +61,8 @@ jobs: continue-on-error: true env: FUZZ_RESULTS_JSONL: ${{ runner.temp }}/fuzz-details.jsonl - run: make fuzz FUZZTIME=${{ inputs.fuzztime || '2m' }} + FUZZTIME: ${{ inputs.fuzztime || '2m' }} + run: make fuzz FUZZTIME="${FUZZTIME}" - name: Record the fuzz result if: always() @@ -84,10 +85,17 @@ jobs: exit_code=1 summary="${failed} of ${targets} fuzz targets found a failing input. Reproducers are attached to this run." fi + # The details file is absent when the fuzz step died before writing + # it; the check still has to report, so the flag is conditional. + details_arg="" + if [ -f "${details}" ]; then + details_arg="--details-jsonl ${details}" + fi + # shellcheck disable=SC2086 # details_arg is a flag pair or empty go run ./internal/assurance/cmd emit \ --id fuzz --exit-code "${exit_code}" --summary "${summary}" \ --metric targets="${targets}" --metric targets_failed="${failed}" \ - --details-jsonl "${details}" \ + ${details_arg} \ --out assurance-results --step-summary - name: Upload check result diff --git a/docs/ASSURANCE.md b/docs/ASSURANCE.md index a376d129..7c5dcbf7 100644 --- a/docs/ASSURANCE.md +++ b/docs/ASSURANCE.md @@ -58,9 +58,11 @@ Highlights: cache to record timing and confirm the output does not change. Some checks stop a release when they fail; others are recorded without -blocking. Either way, a claim that was not confirmed is shown on the page and -raises a tracking issue for maintainers, so a known problem in a published -release is never quietly dropped. Every count links to the job that produced +blocking. A claim that was not confirmed is always shown on the page. When the +problem is found after the release is already published, it also raises a +tracking issue for maintainers, so a known problem in a released version is +never quietly dropped; problems found earlier stop the release instead, and are +fixed before anything is published. Every count links to the job that produced it, so any number on the page is one click from its log. ## Claims, checks, and evidence diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go index e2a452c2..3c57dcd5 100644 --- a/internal/assurance/assurance_test.go +++ b/internal/assurance/assurance_test.go @@ -5,6 +5,7 @@ import ( "flag" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -436,3 +437,34 @@ func TestSchemaVersionsArePinned(t *testing.T) { } } } + +// TestUntrustedNativeArchivesGuardsTheProbe covers the rule that a release +// verifier must never execute a file its own checksum check did not vouch for. +func TestUntrustedNativeArchivesGuardsTheProbe(t *testing.T) { + full := ArchiveName("bomly", "9.9.9", runtime.GOOS, runtime.GOARCH) + lite := ArchiveName("bomly-lite", "9.9.9", runtime.GOOS, runtime.GOARCH) + + for name, testCase := range map[string]struct { + outcome ChecksumOutcome + want []string + }{ + "both verified": {ChecksumOutcome{Verified: []string{full, lite}}, nil}, + "one mismatched": {ChecksumOutcome{Verified: []string{lite}, Mismatched: []string{full}}, []string{full}}, + // The result is sorted, and "bomly-lite_" sorts before "bomly_". + "unlisted entirely": {ChecksumOutcome{Verified: []string{"unrelated.tar.gz"}}, + []string{lite, full}}, + "nothing verified": {ChecksumOutcome{}, []string{lite, full}}, + } { + t.Run(name, func(t *testing.T) { + got := UntrustedNativeArchives(testCase.outcome, "9.9.9") + if len(got) != len(testCase.want) { + t.Fatalf("untrusted = %v, want %v", got, testCase.want) + } + for index, archive := range testCase.want { + if got[index] != archive { + t.Fatalf("untrusted = %v, want %v", got, testCase.want) + } + } + }) + } +} diff --git a/internal/assurance/cmd/commands.go b/internal/assurance/cmd/commands.go index 9187a3a1..c636eac0 100644 --- a/internal/assurance/cmd/commands.go +++ b/internal/assurance/cmd/commands.go @@ -64,7 +64,10 @@ func runEmit(args []string) error { if *status != "" { result.Status = assurance.Status(*status) } - catalogCtx := loadContext(*catalogPath) + catalogCtx, err := loadContext(*catalogPath) + if err != nil { + return err + } if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { return err } @@ -231,7 +234,10 @@ func runGoTest(args []string) error { } result := baseResult(*id, *instance) result.FinishedAt = time.Now().UTC().Format(time.RFC3339) - catalogCtx := loadContext(*catalogPath) + catalogCtx, err := loadContext(*catalogPath) + if err != nil { + return err + } if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { return err } @@ -284,7 +290,10 @@ func runConvert(args []string) error { } result := baseResult(*id, *instance) result.FinishedAt = time.Now().UTC().Format(time.RFC3339) - catalogCtx := loadContext(*catalogPath) + catalogCtx, err := loadContext(*catalogPath) + if err != nil { + return err + } if err := applyCatalog(&result, catalogCtx.catalog, *stage, *level); err != nil { return err } @@ -321,7 +330,10 @@ func runVerifyRelease(args []string) error { return fmt.Errorf("verify-release requires --dir and --version") } trimmed := strings.TrimPrefix(*version, "v") - catalogCtx := loadContext(*catalogPath) + catalogCtx, err := loadContext(*catalogPath) + if err != nil { + return err + } work := *workDir if work == "" { created, err := os.MkdirTemp("", "assurance-release-") @@ -432,15 +444,37 @@ func runVerifyRelease(args []string) error { return err } - probes, err := assurance.ProbeNativeBinaries(context.Background(), *dir, trimmed, work) - if err != nil { - return err - } binaries := baseResult("release-binaries", platform) binaries.FinishedAt = time.Now().UTC().Format(time.RFC3339) if err := applyCatalog(&binaries, catalogCtx.catalog, "", ""); err != nil { return err } + + // Running a binary whose checksum did not match would invert the point of + // checking it: the checksum is what decides whether the file may be + // trusted, so a mismatch stops the probe rather than preceding it. + if untrusted := assurance.UntrustedNativeArchives(outcome, trimmed); len(untrusted) > 0 { + binaries.Status = assurance.StatusFail + binaries.Summary = fmt.Sprintf( + "Not run: %s did not match SHA256SUMS, so the binaries inside were not trusted enough to execute.", + strings.Join(untrusted, ", ")) + for _, archive := range untrusted { + binaries.Details = append(binaries.Details, assurance.Detail{ + Name: archive, Status: assurance.StatusFail, Note: "checksum mismatch, binary not run", + }) + } + if err := emit(binaries); err != nil { + return err + } + return blockingError{ + message: fmt.Sprintf("%d release verification check(s) failed", failures+1), code: 1, + } + } + + probes, err := assurance.ProbeNativeBinaries(context.Background(), *dir, trimmed, work) + if err != nil { + return err + } binaries.Status = assurance.StatusPass passed := 0 for _, probe := range probes { diff --git a/internal/assurance/cmd/main.go b/internal/assurance/cmd/main.go index e3eabb5a..e0f0d6a6 100644 --- a/internal/assurance/cmd/main.go +++ b/internal/assurance/cmd/main.go @@ -85,30 +85,41 @@ type resultContext struct { root string } -func loadContext(catalogPath string) resultContext { +// loadContext resolves the catalog a result is described by. A catalog that +// cannot be read at all leaves the context empty, so a check running outside a +// checkout can still emit with an explicit --stage and --level; a catalog that +// exists but does not parse is reported, because silently treating every check +// as undeclared would drop the stage and level a gate depends on. +func loadContext(catalogPath string) (resultContext, error) { ctx := resultContext{catalogPath: catalogPath} root, err := repositoryRoot() if err != nil { - return ctx + return ctx, nil } ctx.root = root path := catalogPath if path == "" { path = filepath.Join(root, filepath.FromSlash(assurance.DefaultCatalogPath)) + if _, statErr := os.Stat(path); statErr != nil { + return ctx, nil + } } else if !filepath.IsAbs(path) { path = filepath.Join(root, filepath.FromSlash(path)) } catalog, err := assurance.LoadCatalog(path) if err != nil { - return ctx + return ctx, err } ctx.catalog = &catalog ctx.catalogPath = path - return ctx + return ctx, nil } func mustLoadCatalog(catalogPath string) (assurance.Catalog, string, error) { - ctx := loadContext(catalogPath) + ctx, err := loadContext(catalogPath) + if err != nil { + return assurance.Catalog{}, "", err + } if ctx.catalog == nil { path := catalogPath if path == "" { diff --git a/internal/assurance/releaseassets.go b/internal/assurance/releaseassets.go index 643aa548..56882e59 100644 --- a/internal/assurance/releaseassets.go +++ b/internal/assurance/releaseassets.go @@ -211,6 +211,33 @@ type BinaryProbe struct { Note string } +// UntrustedNativeArchives returns the archives for the running platform that +// this release's checksum verification did not vouch for — mismatched, or +// absent from SHA256SUMS entirely. Their contents must not be executed. +func UntrustedNativeArchives(outcome ChecksumOutcome, version string) []string { + suspect := map[string]struct{}{} + for _, name := range outcome.Mismatched { + suspect[name] = struct{}{} + } + verified := map[string]struct{}{} + for _, name := range outcome.Verified { + verified[name] = struct{}{} + } + var untrusted []string + for _, binary := range []string{"bomly", "bomly-lite"} { + archive := ArchiveName(binary, version, runtime.GOOS, runtime.GOARCH) + if _, bad := suspect[archive]; bad { + untrusted = append(untrusted, archive) + continue + } + if _, ok := verified[archive]; !ok { + untrusted = append(untrusted, archive) + } + } + sort.Strings(untrusted) + return untrusted +} + // ProbeNativeBinaries extracts the archives built for the host platform and // checks that each binary reports the released version. func ProbeNativeBinaries(ctx context.Context, dir, version, workDir string) ([]BinaryProbe, error) { diff --git a/internal/assurance/report.go b/internal/assurance/report.go index 2f7786d9..4ce46d04 100644 --- a/internal/assurance/report.go +++ b/internal/assurance/report.go @@ -127,7 +127,8 @@ type EcosystemCoverage struct { Status Status `json:"status"` } -// Coverage lists every ecosystem any check exercised, worst status first. +// Coverage lists every ecosystem any check exercised, in name order. Each +// carries the worst status among the checks that covered it. type Coverage struct { Ecosystems []EcosystemCoverage `json:"ecosystems"` } From 56dd97bf1a6c4f58eba090df00e904d2aae087fc Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Mon, 24 Aug 2026 23:27:25 -0700 Subject: [PATCH 12/13] fix(assurance): keep the goldens workflow from running the ref it refreshes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalog refresh I added to `open-pr` ran `go run ./internal/assurance/cmd` from the checkout of `source_ref` — an arbitrary dispatch input — in the one job that holds a push-capable token. That is arbitrary code execution with write access, which is not a trade worth making for a checksum refresh. The tooling is now checked out separately from the workflow's own ref, and the command reads the regenerated files through a new `--root`, so the untrusted branch supplies data and never code. That job also stops writing Go caches, matching what the regenerate jobs already do, and the smoke checkouts no longer persist the checkout token — nothing in them uses git auth, and the dispatch path accepts any ref. Also dates ADR-0035 the day it was recorded, so the numbering still follows the dates as the index claims. Co-Authored-By: Claude Fable 5 --- .github/workflows/smoke.yml | 2 ++ .github/workflows/update-smoke-goldens.yml | 36 ++++++++++++++----- ...ance-is-a-catalog-and-a-result-contract.md | 2 +- dev-docs/adr/README.md | 2 +- internal/assurance/cmd/main.go | 22 ++++++++++++ internal/assurance/cmd/report.go | 9 ++--- 6 files changed, 59 insertions(+), 14 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 46db33bf..b00fed7e 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -48,6 +48,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs.ref || github.ref }} + persist-credentials: false - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -146,6 +147,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs.ref || github.ref }} + persist-credentials: false - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 diff --git a/.github/workflows/update-smoke-goldens.yml b/.github/workflows/update-smoke-goldens.yml index 972980ee..aa7dce2b 100644 --- a/.github/workflows/update-smoke-goldens.yml +++ b/.github/workflows/update-smoke-goldens.yml @@ -327,12 +327,23 @@ jobs: with: ref: ${{ inputs.source_ref }} + - name: Check out the assurance tooling from this workflow's own ref + # Not from source_ref: this job can push, so it must not execute code + # from the branch whose goldens it is refreshing. The tooling only + # reads the regenerated files and rewrites recorded checksums. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + path: .assurance-tooling + persist-credentials: false + - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version-file: go.mod - cache: true - cache-dependency-path: go.sum + go-version-file: .assurance-tooling/go.mod + # No cache writes: this job runs with write permissions, and a + # poisoned cache would reach the jobs that trust it. + cache: false - name: Download regenerated goldens uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -364,6 +375,20 @@ jobs: --jq '[.jobs[] | select(.name | startswith("Regenerate (")) | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out") | .name] | join(", ")')" echo "failed=${failed}" >> "$GITHUB_OUTPUT" + - name: Refresh the catalog checksums + # Public evidence claims record the checksum of the golden file that + # backs them. Regenerating a golden without refreshing the catalog + # would fail the catalog-valid gate and block the next release. + if: steps.changes.outputs.changed == 'true' + working-directory: .assurance-tooling + shell: bash + run: | + set -euo pipefail + go run ./internal/assurance/cmd catalog-validate \ + --root "${GITHUB_WORKSPACE}" \ + --catalog "${GITHUB_WORKSPACE}/docs/assurance/catalog.json" \ + --refresh + - name: Create pull request with updated goldens if: steps.changes.outputs.changed == 'true' env: @@ -384,11 +409,6 @@ jobs: git checkout -b "${branch_name}" - # Public evidence claims record the checksum of the golden file that - # backs them. Regenerating a golden without refreshing the catalog - # would fail the catalog-valid gate and block the next release. - go run ./internal/assurance/cmd catalog-validate --refresh - git add test/smoke/testdata/golden docs/assurance/catalog.json git commit -m "test: update smoke golden files" git push --set-upstream origin "${branch_name}" diff --git a/dev-docs/adr/0035-release-assurance-is-a-catalog-and-a-result-contract.md b/dev-docs/adr/0035-release-assurance-is-a-catalog-and-a-result-contract.md index 40115b43..d6d92f3e 100644 --- a/dev-docs/adr/0035-release-assurance-is-a-catalog-and-a-result-contract.md +++ b/dev-docs/adr/0035-release-assurance-is-a-catalog-and-a-result-contract.md @@ -1,6 +1,6 @@ # ADR-0035: Release assurance is a declarative catalog plus a per-check result contract -- **Date:** 2026-08-20 +- **Date:** 2026-08-25 - **Status:** Accepted ## Context diff --git a/dev-docs/adr/README.md b/dev-docs/adr/README.md index 95cecdd2..0e6f6a47 100644 --- a/dev-docs/adr/README.md +++ b/dev-docs/adr/README.md @@ -55,4 +55,4 @@ status to `Superseded by [ADR-NNNN](NNNN-slug.md)`; do not rewrite history | ADR-0032 | 2026-08-14 | [SBOM exports carry a synthesized primary component and shared document identity](0032-sbom-exports-carry-synthesized-primary-component.md) | Accepted | | ADR-0033 | 2026-08-24 | [Package origin is detector-asserted; SBOM export only projects it](0033-package-origin-is-detector-asserted.md) | Accepted | | ADR-0034 | 2026-08-24 | [Decisions are recorded as individual ADRs](0034-decisions-are-recorded-as-individual-adrs.md) | Accepted | -| ADR-0035 | 2026-08-20 | [Release assurance is a declarative catalog plus a per-check result contract](0035-release-assurance-is-a-catalog-and-a-result-contract.md) | Accepted | +| ADR-0035 | 2026-08-25 | [Release assurance is a declarative catalog plus a per-check result contract](0035-release-assurance-is-a-catalog-and-a-result-contract.md) | Accepted | diff --git a/internal/assurance/cmd/main.go b/internal/assurance/cmd/main.go index e0f0d6a6..2591dc10 100644 --- a/internal/assurance/cmd/main.go +++ b/internal/assurance/cmd/main.go @@ -138,6 +138,28 @@ func mustLoadCatalog(catalogPath string) (assurance.Catalog, string, error) { return *ctx.catalog, ctx.catalogPath, nil } +// catalogRoot resolves the repository a catalog's paths are relative to. An +// explicit root lets the tool run from one checkout against another, which is +// how the goldens workflow refreshes checksums without executing code from the +// branch it is refreshing. +func catalogRoot(explicit string) (string, error) { + if explicit == "" { + return repositoryRoot() + } + absolute, err := filepath.Abs(explicit) + if err != nil { + return "", fmt.Errorf("resolve --root: %w", err) + } + info, err := os.Stat(absolute) + if err != nil { + return "", fmt.Errorf("resolve --root: %w", err) + } + if !info.IsDir() { + return "", fmt.Errorf("--root %q is not a directory", explicit) + } + return absolute, nil +} + func repositoryRoot() (string, error) { current, err := os.Getwd() if err != nil { diff --git a/internal/assurance/cmd/report.go b/internal/assurance/cmd/report.go index 7a36a233..9ebb4c88 100644 --- a/internal/assurance/cmd/report.go +++ b/internal/assurance/cmd/report.go @@ -239,6 +239,7 @@ func runCatalogValidate(args []string) error { evidenceID := flags.String("evidence", "", "print one evidence claim") skipArtifacts := flags.Bool("skip-artifacts", false, "skip repository artifact hash verification") refresh := flags.Bool("refresh", false, "rewrite recorded checksums from the files they name") + root := flags.String("root", "", "repository the catalog's paths resolve against (default: the working directory's repository)") if err := flags.Parse(args); err != nil { return err } @@ -247,11 +248,11 @@ func runCatalogValidate(args []string) error { return err } if *refresh { - root, rootErr := repositoryRoot() + resolvedRoot, rootErr := catalogRoot(*root) if rootErr != nil { return rootErr } - changed, refreshErr := catalog.RefreshArtifacts(root) + changed, refreshErr := catalog.RefreshArtifacts(resolvedRoot) if refreshErr != nil { return refreshErr } @@ -268,11 +269,11 @@ func runCatalogValidate(args []string) error { return nil } if !*skipArtifacts { - root, rootErr := repositoryRoot() + resolvedRoot, rootErr := catalogRoot(*root) if rootErr != nil { return rootErr } - if err := catalog.VerifyArtifacts(root); err != nil { + if err := catalog.VerifyArtifacts(resolvedRoot); err != nil { return err } } From 58593a15aed9c1e30545140b7823b310574edec6 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Mon, 24 Aug 2026 23:30:32 -0700 Subject: [PATCH 13/13] fix(assurance): make the public-download claim true, not smaller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The claim said every published file is "byte-identical to what was signed before publication", but the check only compared the public assets against the SHA256SUMS downloaded beside them. That proves the two agree with each other, not that either is ours: replacing both would still pass. The job now verifies the Sigstore signature on the published checksum list before checking anything against it, using the same identity the installation docs tell users to check. The bundle was already being downloaded. Narrowing the wording was the other option, but the claim is worth making — it is the one a reader most wants — so it is now backed. Also: ParseReport rejects a claim with no description, since claims carry the same fields whichever way they are asserted and the page would otherwise render an empty card; the "upload it as an assurance- artifact" instruction now describes what the workflows actually do, which is any assurance-* name, some grouped per job; and ADR, cosign, and SLSA are spelled out where they first appear in the agent files. Co-Authored-By: Claude Fable 5 --- .github/workflows/assurance-assessment.yml | 18 +++++++++++--- AGENTS.md | 4 +-- CLAUDE.md | 4 +-- dev-docs/RELEASE_ASSURANCE.md | 5 +++- docs/assurance/catalog.json | 5 ++-- internal/assurance/assurance_test.go | 29 ++++++++++++++++++++++ internal/assurance/report.go | 14 +++++++++++ 7 files changed, 69 insertions(+), 10 deletions(-) diff --git a/.github/workflows/assurance-assessment.yml b/.github/workflows/assurance-assessment.yml index 5f52fb81..c908acfe 100644 --- a/.github/workflows/assurance-assessment.yml +++ b/.github/workflows/assurance-assessment.yml @@ -179,15 +179,27 @@ jobs: curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/SHA256SUMS.sigstore.json" "${base}/SHA256SUMS.sigstore.json" curl -fsSL --retry 3 -o "${RUNNER_TEMP}/assets/multiple.intoto.jsonl" "${base}/multiple.intoto.jsonl" + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + # Only the checksums are re-checked here: the pre-release stage already # extracted and ran the binaries, so what is new after publication is # whether the public copies are complete and unaltered. - - name: Check the public copies against the published checksums + # + # The signature is verified first. Checking assets against a checksum + # list downloaded beside them only proves the two agree; the signature is + # what ties that list back to this repository's release workflow, so a + # replacement of both files cannot pass. + - name: Check the public copies against the signed checksum list id: verify continue-on-error: true run: | set -euo pipefail cd "${RUNNER_TEMP}/assets" + cosign verify-blob SHA256SUMS \ + --bundle SHA256SUMS.sigstore.json \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + --certificate-identity-regexp '^https://github.com/bomly-dev/bomly-cli/\.github/workflows/release\.yml@refs/tags/' sha256sum --check --strict SHA256SUMS - name: Record the public download result @@ -200,13 +212,13 @@ jobs: set -euo pipefail exit_code=0 count="$(find "${RUNNER_TEMP}/assets" -type f | wc -l | tr -d ' ')" - summary="All ${count} published files downloaded from their public URLs without credentials and match the published checksum list." + summary="All ${count} published files downloaded from their public URLs without credentials and match the signed checksum list." if [ "${DOWNLOAD_OUTCOME}" != "success" ]; then exit_code=1 summary="At least one published file could not be downloaded from its public URL." elif [ "${VERIFY_OUTCOME}" != "success" ]; then exit_code=1 - summary="The published files downloaded, but at least one did not match the published checksum list." + summary="The published files downloaded, but the checksum list signature or at least one file did not verify." fi go run ./internal/assurance/cmd emit \ --id public-download --exit-code "${exit_code}" --summary "${summary}" \ diff --git a/AGENTS.md b/AGENTS.md index 18e3a26c..c6144c0b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,7 +106,7 @@ Runtime preparation is owned by `internal/engine`: build the filtered registry o - **Plugin protocol is versioned `v1`.** External plugins use the SDK/HashiCorp gRPC `Metadata` and role descriptor contract. - **No secrets or credentials in logs.** Ever. - **Matcher network calls require explicit enrichment.** Built-in matchers may contact OSV (`https://api.osv.dev`), CISA KEV, deps.dev (`https://api.deps.dev`), OpenSSF Scorecard (`https://api.scorecard.dev`), and Grype's database service (`https://grype.anchore.io/databases`, plus the archive URL it returns) only during `--enrich`. Installed external matcher plugins such as ClearlyDefined and endoflife.date may contact their documented services during `--enrich`. `--audit` evaluates existing package data and must not trigger matcher calls. Remote Git targets and build-tool detectors have separate, explicit network behavior. -- **Record architecture decisions as ADRs in [`dev-docs/adr/`](dev-docs/adr/README.md).** Copy [`dev-docs/adr/TEMPLATE.md`](dev-docs/adr/TEMPLATE.md), take the next number, and add a row to the index. `dev-docs/ARCHITECTURE.md` stays the architecture narrative; `docs/ARCHITECTURE.md` is the public, user-facing overview. +- **Record architecture decisions as architecture decision records (ADRs) in [`dev-docs/adr/`](dev-docs/adr/README.md).** Copy [`dev-docs/adr/TEMPLATE.md`](dev-docs/adr/TEMPLATE.md), take the next number, and add a row to the index. `dev-docs/ARCHITECTURE.md` stays the architecture narrative; `docs/ARCHITECTURE.md` is the public, user-facing overview. - **Prefer `internal/`.** Add new packages inside `internal/` unless there is a clear public API need; genuinely public contract surface belongs in the SDK module. - **Standard library + Cobra + existing deps only.** Do not add new dependencies without discussion. @@ -241,7 +241,7 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai Every quality check belongs to one of three release stages and is declared in `docs/assurance/catalog.json` (schema `bomly.assurance-catalog/v1`): - **prerequisites** — run on the source tree before a tag exists (smoke, portable stability, cross-builds, fuzz, catalog validation). -- **pre-release** — run inside `release.yml` against the still-draft release (asset completeness, checksums, cosign signature, SLSA provenance, released binaries). +- **pre-release** — run inside `release.yml` against the still-draft release (asset completeness, checksums, the Sigstore/cosign signature over the checksum list, SLSA build provenance, released binaries). - **post-release** — run against the shipped binaries after publication (install scripts, public download, released-binary scans, SBOM interoperability, performance samples). Rules: diff --git a/CLAUDE.md b/CLAUDE.md index 6daf1b9a..aca96953 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,7 +106,7 @@ Runtime preparation is owned by `internal/engine`: build the filtered registry o - **Plugin protocol is versioned `v1`.** External plugins use the SDK/HashiCorp gRPC `Metadata` and role descriptor contract. - **No secrets or credentials in logs.** Ever. - **Matcher network calls require explicit enrichment.** Built-in matchers may contact OSV (`https://api.osv.dev`), CISA KEV, deps.dev (`https://api.deps.dev`), OpenSSF Scorecard (`https://api.scorecard.dev`), and Grype's database service (`https://grype.anchore.io/databases`, plus the archive URL it returns) only during `--enrich`. Installed external matcher plugins such as ClearlyDefined and endoflife.date may contact their documented services during `--enrich`. `--audit` evaluates existing package data and must not trigger matcher calls. Remote Git targets and build-tool detectors have separate, explicit network behavior. -- **Record architecture decisions as ADRs in [`dev-docs/adr/`](dev-docs/adr/README.md).** Copy [`dev-docs/adr/TEMPLATE.md`](dev-docs/adr/TEMPLATE.md), take the next number, and add a row to the index. `dev-docs/ARCHITECTURE.md` stays the architecture narrative; `docs/ARCHITECTURE.md` is the public, user-facing overview. +- **Record architecture decisions as architecture decision records (ADRs) in [`dev-docs/adr/`](dev-docs/adr/README.md).** Copy [`dev-docs/adr/TEMPLATE.md`](dev-docs/adr/TEMPLATE.md), take the next number, and add a row to the index. `dev-docs/ARCHITECTURE.md` stays the architecture narrative; `docs/ARCHITECTURE.md` is the public, user-facing overview. - **Prefer `internal/`.** Add new packages inside `internal/` unless there is a clear public API need; genuinely public contract surface belongs in the SDK module. - **Standard library + Cobra + existing deps only.** Do not add new dependencies without discussion. @@ -241,7 +241,7 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai Every quality check belongs to one of three release stages and is declared in `docs/assurance/catalog.json` (schema `bomly.assurance-catalog/v1`): - **prerequisites** — run on the source tree before a tag exists (smoke, portable stability, cross-builds, fuzz, catalog validation). -- **pre-release** — run inside `release.yml` against the still-draft release (asset completeness, checksums, cosign signature, SLSA provenance, released binaries). +- **pre-release** — run inside `release.yml` against the still-draft release (asset completeness, checksums, the Sigstore/cosign signature over the checksum list, SLSA build provenance, released binaries). - **post-release** — run against the shipped binaries after publication (install scripts, public download, released-binary scans, SBOM interoperability, performance samples). Rules: diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index 41c17c42..82a59bab 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -122,7 +122,10 @@ dispatch then tells the landing page a new report is available. order those sections are read in, so put a new area where it belongs in the narrative rather than at the end. 2. Emit a result from the workflow named in `source`, and upload it as an - `assurance-` artifact. + `assurance-*` artifact. The name only has to start with `assurance-`, since + the stage jobs download them all with `merge-multiple`; group results by job + when that is simpler, as the release workflow does with + `assurance-release-`. 3. If the check backs a public claim, add an `evidence` entry pointing at it with `check_id` (and `instance`, when one specific leg proves the claim). Both kinds of entry carry the same fields — title, description, proves, diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json index 085edb0f..2abf8850 100644 --- a/docs/assurance/catalog.json +++ b/docs/assurance/catalog.json @@ -220,7 +220,7 @@ "area": "release-integrity", "stage": "post-release", "level": "gate", - "description": "Downloads every published file from its public URL, without credentials, and checks each one against the published checksum list.", + "description": "Downloads every published file from its public URL, without credentials, verifies the signature on the published checksum list, and checks each file against it.", "source": { "workflow": "assurance-assessment.yml", "job": "public-download" @@ -236,10 +236,11 @@ ] ], "proves": [ - "Every file a user downloads from the public release page is present, reachable without credentials, and byte-identical to what was signed before publication." + "Every file a user downloads from the public release page is present, reachable without credentials, and matches a checksum list signed by this repository's release workflow." ], "limitations": [ "This checks availability and integrity at one moment from one network location.", + "It confirms the published files match the signed list; it does not re-check the provenance attestation, which is verified before publication.", "Whether each archive contains a working binary is checked separately, against the draft release and by the install scripts." ] }, diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go index 3c57dcd5..b213235d 100644 --- a/internal/assurance/assurance_test.go +++ b/internal/assurance/assurance_test.go @@ -468,3 +468,32 @@ func TestUntrustedNativeArchivesGuardsTheProbe(t *testing.T) { }) } } + +// TestParseReportRequiresClaimDescriptions keeps a published report from +// carrying a claim the page would render as an empty card. +func TestParseReportRequiresClaimDescriptions(t *testing.T) { + golden, err := os.ReadFile(filepath.Join("testdata", "golden", "all-pass.report.json")) + if err != nil { + t.Fatalf("read golden report: %v", err) + } + if _, err := ParseReport(golden); err != nil { + t.Fatalf("golden report rejected: %v", err) + } + + var report map[string]any + if err := json.Unmarshal(golden, &report); err != nil { + t.Fatalf("decode golden report: %v", err) + } + evidence, ok := report["evidence"].([]any) + if !ok || len(evidence) == 0 { + t.Fatal("golden report has no evidence to blank out") + } + evidence[0].(map[string]any)["description"] = " " + blanked, err := json.Marshal(report) + if err != nil { + t.Fatalf("encode report: %v", err) + } + if _, err := ParseReport(blanked); err == nil { + t.Fatal("expected a claim with a blank description to be rejected") + } +} diff --git a/internal/assurance/report.go b/internal/assurance/report.go index 4ce46d04..4e299a41 100644 --- a/internal/assurance/report.go +++ b/internal/assurance/report.go @@ -238,6 +238,20 @@ func ParseReport(data []byte) (Report, error) { return Report{}, fmt.Errorf("check %q has unsupported status %q", check.ID, check.Status) } } + // Claims carry the same fields whichever way they are asserted, because the + // published page renders one shape for all of them; a claim with no + // description would render as an empty card. + for _, evidence := range report.Evidence { + if !idPattern.MatchString(evidence.ID) { + return Report{}, fmt.Errorf("assurance report has invalid evidence id %q", evidence.ID) + } + if !evidence.Status.Valid() { + return Report{}, fmt.Errorf("evidence %q has unsupported status %q", evidence.ID, evidence.Status) + } + if strings.TrimSpace(evidence.Description) == "" { + return Report{}, fmt.Errorf("evidence %q is missing its description", evidence.ID) + } + } return report, nil }