diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c617f7..932569c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,18 @@ jobs: - name: go test run: go test -race -count=1 ./... + # The Lane A exit gate prints a chain ledger naming which of its twelve + # links it PROVED and which it could not, and it is the deliverable of + # that packet -- "an UNPROVEN link is not a passing link". Go only shows + # test log output on failure, so on a green run the ledger was invisible + # and a reader saw "ok" with no idea seven of twelve links were proven. + # + # This step exists so the ledger is READ on the machine where it says the + # most: ubuntu-latest has dpkg, so host.Collect runs for real here and the + # host-inventory link is PROVEN, where on the Windows dev host it is not. + - name: Lane A chain ledger (printed, not just passed) + run: go test -count=1 -v -run TestLaneAChain ./test/conformance/lanea/ + # plan/00-SPINE.md S9-AMENDED splits Anvil into two distribution artifacts: # anvil (core, no network-probing capability compiled in) and anvil-dast. # plan/IMPLEMENTATION-PLAN.md 2.2 rules that a config flag inside a single diff --git a/internal/ingest/bootstrap/bootstrap.go b/internal/ingest/bootstrap/bootstrap.go index 5e1cf7c..2564ab8 100644 --- a/internal/ingest/bootstrap/bootstrap.go +++ b/internal/ingest/bootstrap/bootstrap.go @@ -135,7 +135,6 @@ import ( "context" "crypto/sha256" "database/sql" - "encoding/csv" "encoding/hex" "encoding/json" "errors" @@ -146,13 +145,13 @@ import ( "os" "path" "path/filepath" - "sort" "strconv" "strings" "time" "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" "github.com/Susquehanna-Syntax/Anvil/internal/ingest/config" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/decode" "github.com/Susquehanna-Syntax/Anvil/internal/ingest/license" "github.com/Susquehanna-Syntax/Anvil/internal/ingest/sanitize" ) @@ -885,7 +884,7 @@ func (b *Bootstrapper) bulkArchive( lastName: resume.Cursor, } - dc := &decodeCtx{feedID: feed.ID} + dc := newDecodeCtx(feed.ID) err = walkArchive(staged, resume.Entries, resume.Cursor, func(index int, name string, r io.Reader) error { res.EntriesRead++ n, derr := dc.decodeEntry(name, r, func(rec advisoryRecord) error { @@ -899,7 +898,7 @@ func (b *Bootstrapper) bulkArchive( } return w.entryDone(ctx, index, name) }) - res.Sanitizer = dc.stats + res.Sanitizer = dc.stats() res.PeakReadBytes, res.BytesRead = staged.maxRead, staged.readSeen if err != nil { return res, err @@ -1601,8 +1600,28 @@ func writeAdvisory(ctx context.Context, tx *sql.Tx, w *writer, rec advisoryRecor w.res.Degraded++ } - if _, err := tx.ExecContext(ctx, cache.UpsertAdvisoryFTSSQL, rowid, rec.Description, rec.ReferencesText()); err != nil { - return fmt.Errorf("bootstrap: indexing %s/%s: %w", rec.Source, rec.SourceID, err) + // A TOMBSTONED ADVISORY LEAVES THE SEARCH INDEX AND KEEPS ITS ROW. + // + // The row is never deleted (exit criterion 22: a finding that depended on + // it must become INVALIDATED rather than vanish), but its text must stop + // matching, because anything that retrieves it retrieves it as live + // advice. + // + // A.14's writer has always done this and this one had not, which is the + // divergence A.21's end-to-end harness found. It was invisible from inside + // either package: each writer's own tests were internally consistent, and + // the A.14 cross-producer conformance test compared `advisory`, `affected` + // and `cve_alias` but not `advisory_fts`. It is the exact failure ruling + // G11 describes, one layer down from the decoders: A.15's weekly baseline + // self-heal runs THIS path, so every advisory the delta path had correctly + // de-indexed would be re-indexed once a week, forever, with nothing + // surfacing why. The conformance test now compares the index too. + if rec.State == cache.AdvisoryPublished { + if _, err := tx.ExecContext(ctx, cache.UpsertAdvisoryFTSSQL, rowid, rec.Description, rec.ReferencesText()); err != nil { + return fmt.Errorf("bootstrap: indexing %s/%s: %w", rec.Source, rec.SourceID, err) + } + } else if _, err := tx.ExecContext(ctx, cache.DeleteAdvisoryFTSSQL, rowid); err != nil { + return fmt.Errorf("bootstrap: de-indexing tombstoned %s/%s: %w", rec.Source, rec.SourceID, err) } // Replace, never append. `affected` has an autoincrement primary key and @@ -1666,44 +1685,19 @@ func boolInt(v bool) int { // The record model // --------------------------------------------------------------------------- -// affectedRange is one row of `affected`. -type affectedRange struct { - Ecosystem string - Package string - PURL string - Introduced string - Fixed string - DistroBackport bool -} - -// advisoryRecord is one decoded advisory, sanitized, ready to bind. -type advisoryRecord struct { - Source string - SourceID string - CVEID string - Published string - Modified string - State string - TombstonedAt string - Severity string - CVSSVector string - CVSSScore any - EPSSScore any - EPSSAsOf string - KEV bool - Description string - References []string - Aliases []string - Affected []affectedRange - DataVersion string - ParseDegraded bool - StalenessSeconds int - Raw []byte -} - -// ReferencesText is the references column of advisory_fts: one string, newline -// separated, already sanitized element by element. -func (r advisoryRecord) ReferencesText() string { return strings.Join(r.References, "\n") } +// affectedRange and advisoryRecord are the row shapes internal/ingest/decode +// defines, named here so this file reads as it always did. +// +// THEY ARE GO TYPE ALIASES AND NOT CONVERSIONS. This package and +// internal/ingest/delta write ONE table from ONE wire format (orchestrator +// ruling G11), and a converter between two structurally identical record types +// is precisely where a field gets carried on one side and dropped on the other +// — which A.15's weekly self-heal would then restore forever, each importer +// undoing the other, with nothing surfacing why. +type ( + affectedRange = decode.AffectedRange + advisoryRecord = decode.Record +) // --------------------------------------------------------------------------- // Decoding — driven by what the bytes say, not by which feed asked @@ -1773,13 +1767,13 @@ func (dc *decodeCtx) decodeJSON(name string, head []byte, br *bufio.Reader, emit return dc.decodeAlpineSecdb(br, emit) case bytes.Contains(head, []byte(`"csaf_version"`)), bytes.Contains(head, []byte(`"csaf_vex"`)): - return dc.decodeSingle(name, br, dc.decodeCSAF, emit) + return dc.decodeSingle(name, br, dc.dec.CSAF, emit) case bytes.Contains(head, []byte(`"CVE_RECORD"`)): - return dc.decodeSingle(name, br, dc.decodeCVE5, emit) + return dc.decodeSingle(name, br, dc.dec.CVE5, emit) default: - return dc.decodeSingle(name, br, dc.decodeOSV, emit) + return dc.decodeSingle(name, br, dc.dec.OSV, emit) } } @@ -1815,345 +1809,77 @@ func (dc *decodeCtx) decodeSingle( return 1, nil } -// --- OSV, and therefore GHSA: github/advisory-database is OSV format --- - -type osvDoc struct { - SchemaVersion string `json:"schema_version"` - ID string `json:"id"` - Withdrawn string `json:"withdrawn"` - Published string `json:"published"` - Modified string `json:"modified"` - Summary string `json:"summary"` - Details string `json:"details"` - Aliases []string `json:"aliases"` - Related []string `json:"related"` - Severity []osvSeverity `json:"severity"` - References []osvReference `json:"references"` - Affected []osvAffected `json:"affected"` - DatabaseSpec json.RawMessage `json:"database_specific"` -} - -type osvSeverity struct { - Type string `json:"type"` - Score string `json:"score"` -} - -type osvReference struct { - Type string `json:"type"` - URL string `json:"url"` -} - -type osvAffected struct { - Package struct { - Ecosystem string `json:"ecosystem"` - Name string `json:"name"` - PURL string `json:"purl"` - } `json:"package"` - Ranges []struct { - Type string `json:"type"` - Events []struct { - Introduced string `json:"introduced"` - Fixed string `json:"fixed"` - LastAffect string `json:"last_affected"` - } `json:"events"` - } `json:"ranges"` - Versions []string `json:"versions"` -} - -func (dc *decodeCtx) decodeOSV(raw []byte) (advisoryRecord, bool, error) { - var d osvDoc - if err := json.Unmarshal(raw, &d); err != nil { - return advisoryRecord{}, false, err - } - if d.ID == "" { - return advisoryRecord{}, false, nil - } - - rec := advisoryRecord{ - Source: dc.feedID, - SourceID: dc.s(d.ID), - State: cache.AdvisoryPublished, - Raw: raw, - } - if d.Withdrawn != "" { - rec.State = cache.AdvisoryWithdrawn - rec.TombstonedAt = dc.s(d.Withdrawn) - } - rec.Published = dc.s(d.Published) - rec.Modified = dc.s(d.Modified) - rec.Description = dc.s(strings.TrimSpace(d.Summary + "\n\n" + d.Details)) - for _, s := range d.Severity { - if strings.HasPrefix(strings.ToUpper(s.Type), "CVSS") { - rec.CVSSVector = dc.s(s.Score) - break - } - } - for _, r := range d.References { - if r.URL != "" { - rec.References = append(rec.References, dc.s(r.URL)) - } - } - - // A CVE alias is a nullable, indexed alias and never the primary key - // (research/06 Risk #2): GHSA advisories frequently carry none. - for _, a := range append(append([]string{}, d.Aliases...), d.Related...) { - if isCVEID(a) { - rec.Aliases = appendUnique(rec.Aliases, dc.s(a)) - } - } - if isCVEID(d.ID) { - rec.CVEID = rec.SourceID - rec.Aliases = appendUnique(rec.Aliases, rec.SourceID) - } else if len(rec.Aliases) > 0 { - rec.CVEID = rec.Aliases[0] - } - - for _, a := range d.Affected { - eco := dc.s(a.Package.Ecosystem) - pkg := dc.s(a.Package.Name) - if eco == "" || pkg == "" { - continue - } - purl := dc.s(a.Package.PURL) - // A distro's OSV export carries a backported fix: the version number - // does not move upstream, so an upstream range would call a patched - // package vulnerable. research/12 §3, the CVE-2023-32681 / - // RHSA-2023:4520 class of false positive. - backport := isDistroEcosystem(eco) - emitted := false - for _, rg := range a.Ranges { - var introduced string - for _, ev := range rg.Events { - switch { - case ev.Introduced != "": - introduced = dc.s(ev.Introduced) - case ev.Fixed != "": - rec.Affected = append(rec.Affected, affectedRange{ - Ecosystem: eco, Package: pkg, PURL: purl, - Introduced: introduced, Fixed: dc.s(ev.Fixed), DistroBackport: backport, - }) - emitted = true - case ev.LastAffect != "": - rec.Affected = append(rec.Affected, affectedRange{ - Ecosystem: eco, Package: pkg, PURL: purl, - Introduced: introduced, DistroBackport: backport, - }) - emitted = true - } - } - } - if !emitted { - rec.Affected = append(rec.Affected, affectedRange{ - Ecosystem: eco, Package: pkg, PURL: purl, DistroBackport: backport, - }) - } - } - return rec, true, nil -} - -func isDistroEcosystem(eco string) bool { - lower := strings.ToLower(eco) - for _, p := range []string{"ubuntu", "debian", "alpine", "red hat", "redhat", "rocky", "almalinux", "suse", "photon", "chainguard", "wolfi", "mageia"} { - if strings.HasPrefix(lower, p) { - return true - } - } - return false -} - -// --- CVE 5.x, the cvelistV5 baseline --- - -type cve5Doc struct { - DataType string `json:"dataType"` - DataVersion string `json:"dataVersion"` - CVEMetadata struct { - CVEID string `json:"cveId"` - State string `json:"state"` - Published string `json:"datePublished"` - Updated string `json:"dateUpdated"` - Rejected string `json:"dateRejected"` - } `json:"cveMetadata"` - Containers struct { - CNA cve5Container `json:"cna"` - ADP []cve5Container `json:"adp"` - } `json:"containers"` -} - -type cve5Container struct { - Descriptions []struct { - Lang string `json:"lang"` - Value string `json:"value"` - } `json:"descriptions"` - References []struct { - URL string `json:"url"` - } `json:"references"` - Metrics []struct { - CVSSv31 *cve5CVSS `json:"cvssV3_1"` - CVSSv30 *cve5CVSS `json:"cvssV3_0"` - CVSSv40 *cve5CVSS `json:"cvssV4_0"` - } `json:"metrics"` - Affected []struct { - Vendor string `json:"vendor"` - Product string `json:"product"` - PackageN string `json:"packageName"` - Repo string `json:"repo"` - CPEs []string `json:"cpes"` - Versions []struct { - Version string `json:"version"` - LessThan string `json:"lessThan"` - LessOrEqual string `json:"lessThanOrEqual"` - Status string `json:"status"` - } `json:"versions"` - } `json:"affected"` -} +// --------------------------------------------------------------------------- +// The decoding context +// --------------------------------------------------------------------------- -type cve5CVSS struct { - VectorString string `json:"vectorString"` - BaseScore float64 `json:"baseScore"` - BaseSeverity string `json:"baseSeverity"` +// decodeCtx is this import's binding to internal/ingest/decode: which feed the +// rows belong to, and the running report of everything A.3 removed. +// +// EVERY WIRE FORMAT IS DECODED IN internal/ingest/decode AND NOWHERE ELSE. +// This package used to hold its own OSV, CVE 5.x and KEV decoders, unexported, +// which forced internal/ingest/delta to re-derive them — two producers writing +// one table from one wire format. A.14 guarded the duplication with a +// conformance test, but two implementations agreeing on a fixture is a smoke +// alarm and not a fix, so A.21 extracted the shared package (ruling G11). +// +// WHAT STAYS HERE IS DISPATCH, and it stays here because it is genuinely +// different from delta's. This importer walks 300,000 archive members written +// by strangers: a member it cannot read is a README or a directory entry and is +// SKIPPED, so that one bad member does not cost the other 299,999. A delta +// document was fetched BECAUSE SOMETHING SAID IT CHANGED, so the same condition +// there means a change was dropped and must be an error. Sharing the format and +// not the policy is the whole seam. +// +// The size bounds stay here too. MaxRecordBytes is a statement about an archive +// member being streamed out of a 570 MB artifact; delta's MaxDocumentBytes is a +// statement about a body the poller already read into memory. They are +// different numbers for different reasons and neither belongs in the shared +// package. +type decodeCtx struct { + feedID string + dec *decode.Decoder } -// knownCVEDataVersions are the record schema versions this decoder has been -// written against. An UNKNOWN one is PERSISTED with parse_degraded = 1 and -// never dropped: A.2's exit criterion 23 and spine S6's parse_degraded field -// exist because silently discarding a record from a newer schema is how a -// vulnerability disappears from a security tool with no error anywhere. -var knownCVEDataVersions = map[string]bool{"5.0": true, "5.1": true, "5.2": true} - -func (dc *decodeCtx) decodeCVE5(raw []byte) (advisoryRecord, bool, error) { - var d cve5Doc - if err := json.Unmarshal(raw, &d); err != nil { - return advisoryRecord{}, false, err - } - if d.CVEMetadata.CVEID == "" { - return advisoryRecord{}, false, nil - } - - rec := advisoryRecord{ - Source: dc.feedID, - SourceID: dc.s(d.CVEMetadata.CVEID), - CVEID: dc.s(d.CVEMetadata.CVEID), - Published: dc.s(d.CVEMetadata.Published), - Modified: dc.s(d.CVEMetadata.Updated), - State: cache.AdvisoryPublished, - DataVersion: dc.s(d.DataVersion), - ParseDegraded: !knownCVEDataVersions[strings.TrimSpace(d.DataVersion)], - Raw: raw, - } - rec.Aliases = append(rec.Aliases, rec.CVEID) - if strings.EqualFold(d.CVEMetadata.State, "REJECTED") { - rec.State = cache.AdvisoryRejected - rec.TombstonedAt = dc.s(firstNonEmpty(d.CVEMetadata.Rejected, d.CVEMetadata.Updated)) - } - - containers := append([]cve5Container{d.Containers.CNA}, d.Containers.ADP...) - for _, c := range containers { - for _, desc := range c.Descriptions { - if rec.Description == "" && (desc.Lang == "" || strings.HasPrefix(strings.ToLower(desc.Lang), "en")) { - rec.Description = dc.s(desc.Value) - } - } - for _, ref := range c.References { - if ref.URL != "" { - rec.References = appendUnique(rec.References, dc.s(ref.URL)) - } - } - for _, m := range c.Metrics { - for _, v := range []*cve5CVSS{m.CVSSv40, m.CVSSv31, m.CVSSv30} { - if v == nil || v.VectorString == "" || rec.CVSSVector != "" { - continue - } - rec.CVSSVector = dc.s(v.VectorString) - rec.Severity = dc.s(v.BaseSeverity) - score := v.BaseScore - rec.CVSSScore = score - } - } - for _, a := range c.Affected { - pkg := dc.s(firstNonEmpty(a.PackageN, a.Product)) - if pkg == "" { - continue - } - eco := dc.s(firstNonEmpty(a.Vendor, "cpe")) - for _, v := range a.Versions { - if strings.EqualFold(v.Status, "unaffected") { - continue - } - rec.Affected = append(rec.Affected, affectedRange{ - Ecosystem: eco, - Package: pkg, - Introduced: dc.s(v.Version), - Fixed: dc.s(firstNonEmpty(v.LessThan, v.LessOrEqual)), - }) - } - } - } - return rec, true, nil +func newDecodeCtx(feedID string) *decodeCtx { + return &decodeCtx{feedID: feedID, dec: decode.New(feedID)} } -// --- CISA KEV: one document, tens of thousands of records, streamed --- - -type kevEntry struct { - CVEID string `json:"cveID"` - VendorProject string `json:"vendorProject"` - Product string `json:"product"` - VulnerabilityName string `json:"vulnerabilityName"` - DateAdded string `json:"dateAdded"` - ShortDescription string `json:"shortDescription"` - RequiredAction string `json:"requiredAction"` - DueDate string `json:"dueDate"` - Notes string `json:"notes"` -} +// stats is everything the sanitizer removed across this import. +// +// A feed that ships zero-width joiners, bidi overrides or HTML comments inside +// an advisory description is not a curiosity — spine S7 puts prompt injection +// at ingest, and the counts are the only place the fact is visible after the +// bytes are clean. BootstrapResult.Sanitizer carries the merged total. +func (dc *decodeCtx) stats() sanitize.SanitizeStats { return dc.dec.Stats() } +// decodeKEV streams the CISA KEV catalogue element by element. +// +// The traversal is this package's — a 570 MB archive member is never held in +// memory — and the per-entry mapping is the shared one, which is exactly the +// split ruling G11 asked for: delta reads the same catalogue out of a body the +// poller already buffered, and both write the same row. func (dc *decodeCtx) decodeKEV(br *bufio.Reader, emit func(advisoryRecord) error) (int, error) { n := 0 - err := streamArrayField(br, "vulnerabilities", func(raw json.RawMessage) error { - var e kevEntry - if err := json.Unmarshal(raw, &e); err != nil || e.CVEID == "" { + err := decode.StreamArrayField(br, "vulnerabilities", MaxRecordBytes, func(raw json.RawMessage) error { + rec, ok, err := dc.dec.KEVEntry(raw) + if err != nil || !ok { + // A single malformed catalogue entry is skipped, not fatal: KEV is + // one document holding every entry, and one bad entry must not + // cost the rest. It is bounded to a single element. return nil } - rec := advisoryRecord{ - Source: dc.feedID, - SourceID: dc.s(e.CVEID), - CVEID: dc.s(e.CVEID), - Published: dc.s(e.DateAdded), - State: cache.AdvisoryPublished, - KEV: true, - Description: dc.s(strings.TrimSpace(e.VulnerabilityName + "\n\n" + e.ShortDescription + "\n\n" + e.RequiredAction)), - Raw: append([]byte(nil), raw...), - } - rec.Aliases = append(rec.Aliases, rec.CVEID) - if pkg := dc.s(e.Product); pkg != "" { - rec.Affected = append(rec.Affected, affectedRange{ - Ecosystem: dc.s(firstNonEmpty(e.VendorProject, "vendor")), - Package: pkg, - }) - } - if e.Notes != "" { - rec.References = append(rec.References, dc.s(e.Notes)) - } n++ return emit(rec) }) + if errors.Is(err, decode.ErrElementTooLarge) { + return 0, refuse(ErrRecordTooLarge, "an element of %q exceeded %d bytes", "vulnerabilities", MaxRecordBytes) + } return n, err } -// --- Alpine secdb: per-branch JSON, package -> secfixes -> CVE list --- - -type alpineDoc struct { - APKURL string `json:"apkurl"` - Archs []any `json:"archs"` - Reponame string `json:"reponame"` - URLPrefix string `json:"urlprefix"` - DistroVersion string `json:"distroversion"` - Packages []struct { - Pkg struct { - Name string `json:"name"` - Secfixes map[string][]string `json:"secfixes"` - } `json:"pkg"` - } `json:"packages"` -} - +// decodeAlpineSecdb bounds the branch file, then hands it to the shared +// decoder. The bound is enforced on what ARRIVED, never on a declared size. func (dc *decodeCtx) decodeAlpineSecdb(br *bufio.Reader, emit func(advisoryRecord) error) (int, error) { raw, err := io.ReadAll(io.LimitReader(br, MaxRecordBytes+1)) if err != nil { @@ -2162,442 +1888,11 @@ func (dc *decodeCtx) decodeAlpineSecdb(br *bufio.Reader, emit func(advisoryRecor if len(raw) > MaxRecordBytes { return 0, refuse(ErrRecordTooLarge, "an alpine secdb branch file exceeded %d bytes", MaxRecordBytes) } - var d alpineDoc - if err := json.Unmarshal(raw, &d); err != nil { - return 0, nil - } - - // One advisory row per (branch, CVE), carrying every package whose secfix - // names it. The branch is part of the id because the same CVE is fixed at - // different versions on different Alpine branches, and collapsing them - // would lose the version that actually matters to a host. - branch := dc.s(firstNonEmpty(d.DistroVersion, "alpine")) - byCVE := map[string]*advisoryRecord{} - var order []string - for _, p := range d.Packages { - pkg := dc.s(p.Pkg.Name) - if pkg == "" { - continue - } - for version, cves := range p.Pkg.Secfixes { - fixed := dc.s(version) - for _, cve := range cves { - id := dc.s(strings.TrimSpace(cve)) - if id == "" { - continue - } - key := branch + "/" + id - rec, ok := byCVE[key] - if !ok { - rec = &advisoryRecord{ - Source: dc.feedID, - SourceID: key, - State: cache.AdvisoryPublished, - Raw: raw, - } - if isCVEID(id) { - rec.CVEID = id - rec.Aliases = append(rec.Aliases, id) - } - byCVE[key] = rec - order = append(order, key) - } - rec.Affected = append(rec.Affected, affectedRange{ - Ecosystem: "apk", - Package: pkg, - Fixed: fixed, - // Alpine ships backported fixes with an -rN suffix, which - // is precisely the case an upstream range gets wrong. - DistroBackport: true, - }) - } - } - } - // Deterministic order: the map iteration above is not, and a bootstrap - // whose row order varies between runs is a resume cursor that means - // different things on different days. - sort.Strings(order) - for _, key := range order { - if err := emit(*byCVE[key]); err != nil { - return 0, err - } - } - return len(order), nil + return dc.dec.AlpineSecdb(raw, emit) } -// --- Red Hat CSAF/VEX --- -// -// OVAL v2 is deprecated and must not be ingested: since 2024-07-10 Red Hat -// publishes CSAF for every RHSA and VEX for every CVE touching the portfolio -// (research/06 S19). This decoder reads the shape both share. - -type csafDoc struct { - Document struct { - Category string `json:"category"` - Title string `json:"title"` - Tracking struct { - ID string `json:"id"` - InitialReleaseDate string `json:"initial_release_date"` - CurrentReleaseDate string `json:"current_release_date"` - Status string `json:"status"` - } `json:"tracking"` - Notes []struct { - Category string `json:"category"` - Text string `json:"text"` - } `json:"notes"` - References []struct { - URL string `json:"url"` - } `json:"references"` - } `json:"document"` - Vulnerabilities []struct { - CVE string `json:"cve"` - Title string `json:"title"` - Notes []struct { - Category string `json:"category"` - Text string `json:"text"` - } `json:"notes"` - ProductStatus struct { - Fixed []string `json:"fixed"` - KnownAffected []string `json:"known_affected"` - } `json:"product_status"` - Scores []struct { - CVSSv3 struct { - VectorString string `json:"vectorString"` - BaseScore float64 `json:"baseScore"` - BaseSeverity string `json:"baseSeverity"` - } `json:"cvss_v3"` - } `json:"scores"` - } `json:"vulnerabilities"` -} - -func (dc *decodeCtx) decodeCSAF(raw []byte) (advisoryRecord, bool, error) { - var d csafDoc - if err := json.Unmarshal(raw, &d); err != nil { - return advisoryRecord{}, false, err - } - if d.Document.Tracking.ID == "" { - return advisoryRecord{}, false, nil - } - - rec := advisoryRecord{ - Source: dc.feedID, - SourceID: dc.s(d.Document.Tracking.ID), - Published: dc.s(d.Document.Tracking.InitialReleaseDate), - Modified: dc.s(d.Document.Tracking.CurrentReleaseDate), - State: cache.AdvisoryPublished, - Raw: raw, - } - rec.Description = dc.s(d.Document.Title) - for _, n := range d.Document.Notes { - if n.Category == "description" || n.Category == "summary" { - rec.Description = dc.s(strings.TrimSpace(rec.Description + "\n\n" + n.Text)) - } - } - for _, r := range d.Document.References { - if r.URL != "" { - rec.References = appendUnique(rec.References, dc.s(r.URL)) - } - } - if strings.EqualFold(d.Document.Tracking.Status, "withdrawn") { - rec.State = cache.AdvisoryWithdrawn - rec.TombstonedAt = rec.Modified - } - - for _, v := range d.Vulnerabilities { - if isCVEID(v.CVE) { - id := dc.s(v.CVE) - rec.Aliases = appendUnique(rec.Aliases, id) - if rec.CVEID == "" { - rec.CVEID = id - } - } - for _, s := range v.Scores { - if s.CVSSv3.VectorString != "" && rec.CVSSVector == "" { - rec.CVSSVector = dc.s(s.CVSSv3.VectorString) - rec.Severity = dc.s(s.CVSSv3.BaseSeverity) - score := s.CVSSv3.BaseScore - rec.CVSSScore = score - } - } - // A VEX "fixed" product id is an RPM NEVRA. The fix is BACKPORTED — - // Red Hat patches without bumping the upstream version — which is the - // column that defeats the CVE-2023-32681 / RHSA-2023:4520 false - // positive class (research/12 §3). - for _, p := range v.ProductStatus.Fixed { - name, version := splitNEVRA(p) - if name == "" { - continue - } - rec.Affected = append(rec.Affected, affectedRange{ - Ecosystem: "rpm", Package: dc.s(name), - Fixed: dc.s(version), DistroBackport: true, - }) - } - for _, p := range v.ProductStatus.KnownAffected { - name, version := splitNEVRA(p) - if name == "" { - continue - } - rec.Affected = append(rec.Affected, affectedRange{ - Ecosystem: "rpm", Package: dc.s(name), - Introduced: dc.s(version), DistroBackport: true, - }) - } - } - return rec, true, nil -} - -// splitNEVRA pulls a package name and version out of a CSAF product id such as -// "Red Hat Enterprise Linux 9:openssl-1:3.0.7-24.el9.x86_64". It is deliberately -// conservative: a shape it does not recognise yields an empty name and is -// dropped, because a wrong package name in `affected` is a false positive -// against a package nobody has. -func splitNEVRA(product string) (string, string) { - p := strings.TrimSpace(product) - - // A product id is ":", and only the FIRST - // colon separates the two — the second colon, if there is one, is the - // RPM epoch inside the NEVRA. Splitting on the last colon (the obvious - // first attempt) strips the package name and leaves the version, which - // then reaches `affected.package` as a version string. - if i := strings.Index(p, ":"); i >= 0 { - if candidate := strings.TrimSpace(p[i+1:]); looksLikeNEVRA(candidate) { - p = candidate - } - } - - // name-[epoch:]version-release.arch. The name ends at the first hyphen - // followed by a digit, which is where RPM's version field begins. - idx := -1 - for k := 0; k+1 < len(p); k++ { - if p[k] == '-' && p[k+1] >= '0' && p[k+1] <= '9' { - idx = k - break - } - } - if idx <= 0 { - return "", "" - } - name, rest := p[:idx], p[idx+1:] - if j := strings.Index(rest, ":"); j >= 0 { - rest = rest[j+1:] - } - return name, rest -} - -func looksLikeNEVRA(s string) bool { - for k := 0; k+1 < len(s); k++ { - if s[k] == '-' && s[k+1] >= '0' && s[k+1] <= '9' { - return true - } - } - return false -} - -// --- EPSS: a CSV with comment lines, streamed row by row --- - +// decodeEPSS streams the EPSS CSV. The whole file is never buffered, so the +// per-record bound above does not apply and none is invented here. func (dc *decodeCtx) decodeEPSS(br *bufio.Reader, emit func(advisoryRecord) error) (int, error) { - asOf := "" - // Skip and remember the leading '#' comment lines; the model date lives - // there and it is what as_of should say about an EPSS score. - for { - peek, err := br.Peek(1) - if err != nil || len(peek) == 0 || peek[0] != '#' { - break - } - line, err := br.ReadString('\n') - if err != nil && !errors.Is(err, io.EOF) { - return 0, fmt.Errorf("bootstrap: reading EPSS header: %w", err) - } - if i := strings.Index(line, "score_date:"); i >= 0 { - asOf = strings.TrimSpace(strings.TrimSuffix(line[i+len("score_date:"):], "\n")) - } - if err != nil { - return 0, nil - } - } - - r := csv.NewReader(br) - r.FieldsPerRecord = -1 - r.ReuseRecord = true - header, err := r.Read() - if err != nil { - return 0, nil - } - col := map[string]int{} - for i, h := range header { - col[strings.ToLower(strings.TrimSpace(h))] = i - } - cveCol, ok := col["cve"] - if !ok { - return 0, nil - } - epssCol, hasEPSS := col["epss"] - - n := 0 - for { - row, err := r.Read() - if errors.Is(err, io.EOF) { - return n, nil - } - if err != nil { - // A malformed line ends the file rather than the import: an EPSS - // score is never load-bearing for a verdict (it is Tier 3, opt-in, - // risk-accepted) and what has been read is still usable. - return n, nil - } - if cveCol >= len(row) || !isCVEID(row[cveCol]) { - continue - } - rec := advisoryRecord{ - Source: dc.feedID, - SourceID: dc.s(row[cveCol]), - CVEID: dc.s(row[cveCol]), - State: cache.AdvisoryPublished, - EPSSAsOf: dc.s(asOf), - Raw: []byte(strings.Join(row, ",")), - } - rec.Aliases = append(rec.Aliases, rec.CVEID) - if hasEPSS && epssCol < len(row) { - if f, err := strconv.ParseFloat(strings.TrimSpace(row[epssCol]), 64); err == nil { - rec.EPSSScore = f - } - } - n++ - if err := emit(rec); err != nil { - return n, err - } - } -} - -// --------------------------------------------------------------------------- -// Streaming helpers -// --------------------------------------------------------------------------- - -// streamArrayField walks a top-level JSON object and hands every element of the -// named array to fn, ONE AT A TIME. -// -// This is the function that makes "never loads the whole archive into memory" -// true for the multi-record feeds. CISA KEV is a single JSON document holding -// every known-exploited vulnerability; json.Unmarshal of it would allocate the -// entire file plus its object graph, which for the merged OSV export would be -// gigabytes. json.Decoder buffers one VALUE. -func streamArrayField(r io.Reader, field string, fn func(json.RawMessage) error) error { - dec := json.NewDecoder(r) - tok, err := dec.Token() - if err != nil { - return nil - } - if d, ok := tok.(json.Delim); !ok || d != '{' { - return nil - } - for dec.More() { - keyTok, err := dec.Token() - if err != nil { - return nil - } - key, _ := keyTok.(string) - if key != field { - var skip json.RawMessage - if err := dec.Decode(&skip); err != nil { - return nil - } - continue - } - open, err := dec.Token() - if err != nil { - return nil - } - if d, ok := open.(json.Delim); !ok || d != '[' { - return nil - } - for dec.More() { - var raw json.RawMessage - if err := dec.Decode(&raw); err != nil { - return nil - } - if len(raw) > MaxRecordBytes { - return refuse(ErrRecordTooLarge, "an element of %q exceeded %d bytes", field, MaxRecordBytes) - } - if err := fn(raw); err != nil { - return err - } - } - return nil - } - return nil -} - -// decodeCtx is the per-import decoding context: which feed the rows belong to, -// and the running report of everything A.3 removed. -// -// It exists so the sanitizer's findings are not thrown away. A feed that ships -// zero-width joiners, bidi overrides or HTML comments inside an advisory -// description is not a curiosity — spine S7 puts prompt injection at ingest, -// and the counts are the only place the fact is visible after the bytes are -// clean. BootstrapResult.Sanitizer carries the merged total. -type decodeCtx struct { - feedID string - stats sanitize.SanitizeStats -} - -// s is A.3 applied to one field, accumulating what it removed. -// -// It is a named method rather than an inline call at every site so that -// internal/ingest/sanitize's writer guard — which resolves the package-local -// call graph by NAME — sees the decoders reaching the sanitizer, and so that -// there is exactly one place where a field can be bound without passing -// through it (there is none). -func (dc *decodeCtx) s(raw string) string { - clean, st := sanitize.Sanitize(raw) - dc.stats.Merge(st) - return clean -} - -func isCVEID(s string) bool { - t := strings.TrimSpace(s) - if !strings.HasPrefix(t, "CVE-") || len(t) < 8 { - return false - } - rest := t[4:] - dash := strings.Index(rest, "-") - if dash < 4 { - return false - } - for _, r := range rest[:dash] { - if r < '0' || r > '9' { - return false - } - } - tail := rest[dash+1:] - if tail == "" { - return false - } - for _, r := range tail { - if r < '0' || r > '9' { - return false - } - } - return true -} - -func appendUnique(list []string, v string) []string { - if v == "" { - return list - } - for _, e := range list { - if e == v { - return list - } - } - return append(list, v) -} - -func firstNonEmpty(vals ...string) string { - for _, v := range vals { - if strings.TrimSpace(v) != "" { - return v - } - } - return "" + return dc.dec.EPSS(br, emit) } diff --git a/internal/ingest/bootstrap/ghsa_clone.go b/internal/ingest/bootstrap/ghsa_clone.go index 1ad04fe..cc24f82 100644 --- a/internal/ingest/bootstrap/ghsa_clone.go +++ b/internal/ingest/bootstrap/ghsa_clone.go @@ -292,7 +292,7 @@ func (b *Bootstrapper) bloblessClone( start = 0 } - dc := &decodeCtx{feedID: feed.ID} + dc := newDecodeCtx(feed.ID) meter := &readMeter{} for i := start; i < len(files); i++ { rel := files[i] @@ -300,7 +300,7 @@ func (b *Bootstrapper) bloblessClone( return w.add(ctx, rec) }) if err != nil { - res.Sanitizer, res.PeakReadBytes, res.BytesRead = dc.stats, meter.MaxRead, meter.Total + res.Sanitizer, res.PeakReadBytes, res.BytesRead = dc.stats(), meter.MaxRead, meter.Total return res, err } res.EntriesRead++ @@ -308,11 +308,11 @@ func (b *Bootstrapper) bloblessClone( res.EntriesSkipped++ } if err := w.entryDone(ctx, i, rel); err != nil { - res.Sanitizer, res.PeakReadBytes, res.BytesRead = dc.stats, meter.MaxRead, meter.Total + res.Sanitizer, res.PeakReadBytes, res.BytesRead = dc.stats(), meter.MaxRead, meter.Total return res, err } } - res.Sanitizer, res.PeakReadBytes, res.BytesRead = dc.stats, meter.MaxRead, meter.Total + res.Sanitizer, res.PeakReadBytes, res.BytesRead = dc.stats(), meter.MaxRead, meter.Total if err := w.finish(ctx); err != nil { return res, err diff --git a/internal/ingest/decode/decode.go b/internal/ingest/decode/decode.go new file mode 100644 index 0000000..6be13a2 --- /dev/null +++ b/internal/ingest/decode/decode.go @@ -0,0 +1,419 @@ +// Package decode turns a publisher's advisory document into the row shape +// internal/ingest/cache stores. It is the ONE implementation of every wire +// format Lane A reads. +// +// =========================================================================== +// WHY THIS PACKAGE EXISTS — ORCHESTRATOR RULING G11 +// =========================================================================== +// +// It did not exist until A.21. internal/ingest/bootstrap's decoders were +// unexported, so internal/ingest/delta RE-DERIVED CVE 5.x, OSV and KEV +// decoding, and the cache had two producers writing one table from one wire +// format. +// +// That is worse than ordinary duplication. If the two drifted, A.15's weekly +// self-heal would RESTORE THE SAME ROWS FOREVER and nothing would surface it: +// the baseline importer would rewrite what the delta importer wrote, each +// convinced the other was wrong, and the cache would settle on whichever ran +// last. A self-healing system healing toward the wrong answer is quieter than +// one that breaks. +// +// A.14 guarded the duplication with a conformance test that ran both importers +// over identical fixtures — the right move available to it — but two +// implementations agreeing today is a smoke alarm, not a fix. This package is +// the fix. There is now one decodeOSV, one decodeCVE5, one KEV entry mapping, +// one distro-ecosystem list and one known-dataVersion set, and a divergence +// between the two importers is no longer expressible. +// +// =========================================================================== +// WHAT THIS PACKAGE DOES NOT OWN — THE SEAM IS DELIBERATE +// =========================================================================== +// +// DISPATCH IS NOT SHARED, because the two callers legitimately disagree about +// what an unrecognised document MEANS, and collapsing that disagreement would +// be a second bug wearing the first one's clothes: +// +// - A.8's bulk importer walks 300,000 archive members written by strangers. +// A member it does not recognise is a README, a directory entry or a CWE +// catalog, and it is SKIPPED — one bad member must not cost the other +// 299,999. +// - A.14's delta path fetched a document BECAUSE SOMETHING SAID IT CHANGED. +// A document it does not recognise means a change was dropped, so it is an +// ERROR that routes the feed to a path that does understand it. +// +// Each caller therefore keeps its own dispatch and its own size bounds (a +// streamed archive member and a body already read into memory are bounded by +// different numbers for different reasons), and calls in here for the format +// itself. What a record IS, is shared; what a caller does about a record it +// cannot read, is not. +// +// SIZE BOUNDS ARE PARAMETERS, never constants of this package. A limit stated +// twice is a limit that can diverge, which is the exact failure this package +// was created to end. +// +// =========================================================================== +// SANITISATION +// =========================================================================== +// +// Every string this package binds into a Record goes through Decoder.s, which +// is internal/ingest/sanitize applied field by field with the removal counts +// accumulated (A.3 forbids dropping characters without a count; spine S7 puts +// prompt-injection defence at ingest, not at prompt time). Record.Raw is the +// one exception and it is deliberate: `advisory.raw_json` stores the +// PUBLISHER'S BYTES VERBATIM, because CVE-TOU requires records be stored +// unedited, and because two importers that re-render a document store two +// different digests of the same advisory. +// +// The callers re-prove it at the write site with sanitize.AssertAllSanitized +// on the exact values about to be bound. This package's guarantee is not +// trusted there; it is checked. +package decode + +import ( + "encoding/json" + "io" + "strings" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/sanitize" +) + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +// ErrElementTooLarge is returned by StreamArrayField when one element of a +// streamed array exceeds the caller's bound. +// +// It is the only error this package originates that is not simply a JSON parse +// failure handed back. Callers wrap it in their own refusal vocabulary — the +// bound is theirs, so the sentence about it is theirs too. +var ErrElementTooLarge = errorString("decode: an array element exceeds the caller's size bound") + +type errorString string + +func (e errorString) Error() string { return string(e) } + +// --------------------------------------------------------------------------- +// The record model +// --------------------------------------------------------------------------- + +// AffectedRange is one row of the cache's `affected` table: a package, an +// ecosystem, and the version window a comparator answers against. +// +// plan/00-SPINE.md S1 is why these rows are the point of Lane A at all: +// "CVE/OSV/GHSA describe vulnerable PACKAGE VERSIONS, and a version comparator +// answers that exactly and for free." +type AffectedRange struct { + Ecosystem string + Package string + PURL string + + // Introduced and Fixed bound the vulnerable window. Either may be empty: + // an OSV entry with a `last_affected` event and no `fixed` has no fixed + // version, and that is a fact about the advisory, not a parse failure. + Introduced string + Fixed string + + // DistroBackport marks a range that came from a vendor or distro advisory + // rather than from upstream. research/12 §3: a distro backports a fix + // without moving the upstream version, so an upstream range calls a + // patched package vulnerable. This column is what A.17 needs to not do + // that. + DistroBackport bool +} + +// Record is one decoded advisory, already sanitized, ready to bind. +// +// Both importers write THIS type. internal/ingest/delta.Record and +// internal/ingest/bootstrap's advisoryRecord are Go type ALIASES for it, so +// there is one field set, one meaning per field, and no conversion function +// anywhere that could quietly drop one. +type Record struct { + // Source is the feed id and SourceID the native id within it. Together + // they are the primary key, and it is NEVER the CVE id: research/06 Risk + // #2 requires a cvelistV5 outage to be survivable by swapping sources, + // and GHSA advisories frequently carry no CVE at all. + Source string + SourceID string + + // CVEID is the nullable, indexed alias. Aliases carries the one-to-many. + CVEID string + Aliases []string + + Published string + Modified string + + // State is one of cache.AdvisoryPublished / AdvisoryWithdrawn / + // AdvisoryRejected. A non-published state MUST carry TombstonedAt, and + // the schema's advisory_tombstone_paired CHECK refuses the row otherwise + // — withdrawn advisories are tombstoned, never deleted, so that findings + // that depended on them become invalidated rather than vanishing. + State string + TombstonedAt string + + Severity string + CVSSVector string + + // CVSSScore and EPSSScore are `any` so that "no score" is SQL NULL rather + // than 0.0. A zero CVSS base score is a real value, and a comparator that + // cannot tell it from "absent" ranks an unscored advisory as harmless. + CVSSScore any + EPSSScore any + EPSSAsOf string + + KEV bool + + Description string + References []string + + Affected []AffectedRange + + // DataVersion is the record schema version the document declared, and + // ParseDegraded is spine S6's field for "this was persisted anyway". + // A.2 exit criterion 23: an unknown CVE dataVersion is PERSISTED with + // parse_degraded = 1, never dropped, because silently discarding a record + // from a newer schema is how a vulnerability disappears from a security + // tool with no error anywhere. + DataVersion string + ParseDegraded bool + + // StalenessSeconds overrides the batch-wide value when the record carries + // its own age. Zero means "use the batch's". + StalenessSeconds int + + // Raw is the document verbatim, stored in advisory.raw_json. It is never + // sanitized: the column is the publisher's bytes, and CVE-TOU requires + // records be stored byte-verbatim (research/06 §"License"). + Raw []byte +} + +// ReferencesText is the `references_text` column of advisory_fts: the +// references as one newline-separated string, each element already sanitized. +func (r Record) ReferencesText() string { return strings.Join(r.References, "\n") } + +// --------------------------------------------------------------------------- +// The decoder +// --------------------------------------------------------------------------- + +// Decoder is the per-import decoding context: which feed the rows belong to, +// and the running report of everything A.3 removed. +// +// The stats exist so the sanitizer's findings are not thrown away. A feed that +// ships zero-width joiners, bidi overrides or HTML comments inside an advisory +// description is not a curiosity — spine S7 puts prompt injection at ingest, +// and the counts are the only place the fact is visible after the bytes are +// clean. +type Decoder struct { + feedID string + stats sanitize.SanitizeStats +} + +// New returns a decoder that stamps feedID into every record's Source. +func New(feedID string) *Decoder { return &Decoder{feedID: feedID} } + +// FeedID is the feed every record from this decoder is attributed to. +func (dc *Decoder) FeedID() string { return dc.feedID } + +// Stats is everything the sanitizer removed across every field decoded so far. +func (dc *Decoder) Stats() sanitize.SanitizeStats { return dc.stats } + +// s is A.3 applied to one field, accumulating what it removed. +// +// It is a named method rather than an inline call at every site so that +// internal/ingest/sanitize's writer guard — which resolves the package-local +// call graph by NAME — sees the decoders reaching the sanitizer, and so that +// there is exactly one place where a field could be bound without passing +// through it (there is none). +func (dc *Decoder) s(raw string) string { + clean, st := sanitize.Sanitize(raw) + dc.stats.Merge(st) + return clean +} + +// --------------------------------------------------------------------------- +// Streaming helper +// --------------------------------------------------------------------------- + +// StreamArrayField walks a top-level JSON object and hands every element of the +// named array to fn, ONE AT A TIME. +// +// This is the function that makes "never loads the whole archive into memory" +// true for the multi-record feeds. CISA KEV is a single JSON document holding +// every known-exploited vulnerability; json.Unmarshal of it would allocate the +// entire file plus its object graph, which for the merged OSV export would be +// gigabytes. json.Decoder buffers one VALUE. +// +// maxElement bounds ONE element and is the caller's number: an archive member +// and a body already read into memory are bounded differently, for different +// reasons, and a constant here would be a third bound nobody owns. A zero or +// negative maxElement means unbounded, which is only correct when the caller +// has already bounded the whole document. +// +// A MALFORMED DOCUMENT ENDS THE WALK AND IS NOT AN ERROR. That is this +// function's contract and the callers depend on it: a bulk archive member that +// is not the shape it looked like is a skip, and the caller counts the skip. +// Only the size refusal and fn's own error propagate. +func StreamArrayField(r io.Reader, field string, maxElement int, fn func(json.RawMessage) error) error { + dec := json.NewDecoder(r) + tok, err := dec.Token() + if err != nil { + return nil + } + if d, ok := tok.(json.Delim); !ok || d != '{' { + return nil + } + for dec.More() { + keyTok, err := dec.Token() + if err != nil { + return nil + } + key, _ := keyTok.(string) + if key != field { + var skip json.RawMessage + if err := dec.Decode(&skip); err != nil { + return nil + } + continue + } + open, err := dec.Token() + if err != nil { + return nil + } + if d, ok := open.(json.Delim); !ok || d != '[' { + return nil + } + for dec.More() { + var raw json.RawMessage + if err := dec.Decode(&raw); err != nil { + return nil + } + if maxElement > 0 && len(raw) > maxElement { + return ErrElementTooLarge + } + if err := fn(raw); err != nil { + return err + } + } + return nil + } + return nil +} + +// --------------------------------------------------------------------------- +// Small shared helpers +// --------------------------------------------------------------------------- + +// IsCVEID is the ALLOWLIST that decides whether a string from a feed may be +// treated as a CVE identifier. +// +// It is not only a parsing convenience: the deltaLog route in +// internal/ingest/delta uses it as the ONLY thing that may cross from feed +// content into a fetch, so its strictness is a security property and not a +// nicety. +// +// The shape is CVE-<4+ digits>-<1+ digits> and nothing else. It is an allowlist +// of characters and structure, not a denylist of dangerous ones: this project +// has lost three guards to a symbol, a verb or a wording nobody listed, and +// `CVE-2024-0001/../../etc/passwd` is precisely the string a denylist misses. +func IsCVEID(s string) bool { + t := strings.TrimSpace(s) + if !strings.HasPrefix(t, "CVE-") || len(t) < 8 { + return false + } + rest := t[4:] + dash := strings.Index(rest, "-") + if dash < 4 { + return false + } + for _, r := range rest[:dash] { + if r < '0' || r > '9' { + return false + } + } + tail := rest[dash+1:] + if tail == "" { + return false + } + for _, r := range tail { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// distroEcosystemPrefixes are the ecosystems whose advisories carry BACKPORTED +// fixes. research/12 §3, the CVE-2023-32681 / RHSA-2023:4520 class: a distro +// patches without moving the upstream version, so an upstream range calls a +// fixed package vulnerable. +// +// THIS LIST EXISTS ONCE. It used to exist twice — identically, and guarded by a +// conformance test — and a divergence would have changed +// `affected.distro_backport` for the same bytes depending on which importer +// ran, which is the column A.17's vendor-first precedence rests on. +var distroEcosystemPrefixes = []string{ + "ubuntu", "debian", "alpine", "red hat", "redhat", "rocky", "almalinux", + "suse", "photon", "chainguard", "wolfi", "mageia", +} + +// IsDistroEcosystem reports whether an OSV ecosystem string names a +// distribution, and therefore whether its ranges are backported. +func IsDistroEcosystem(eco string) bool { + lower := strings.ToLower(eco) + for _, p := range distroEcosystemPrefixes { + if strings.HasPrefix(lower, p) { + return true + } + } + return false +} + +// knownCVEDataVersions are the record schema versions these decoders were +// written against. An UNKNOWN one is PERSISTED with parse_degraded = 1 and +// never dropped (Lane A exit criterion 23, spine S6): silently discarding a +// record from a newer schema is how a vulnerability disappears from a security +// tool with no error anywhere. +var knownCVEDataVersions = map[string]bool{"5.0": true, "5.1": true, "5.2": true} + +// KnownCVEDataVersion reports whether a CVE record's declared dataVersion is +// one these decoders understand. +func KnownCVEDataVersion(v string) bool { return knownCVEDataVersions[strings.TrimSpace(v)] } + +// KnownCVEDataVersions is the sorted set, for a caller that has to report it. +func KnownCVEDataVersions() []string { + out := make([]string, 0, len(knownCVEDataVersions)) + for v := range knownCVEDataVersions { + out = append(out, v) + } + // Three fixed values; an insertion sort keeps the dependency count at zero + // and the order stable for a report. + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j] < out[j-1]; j-- { + out[j], out[j-1] = out[j-1], out[j] + } + } + return out +} + +// AppendUnique appends v to list when it is neither empty nor already present. +func AppendUnique(list []string, v string) []string { + if v == "" { + return list + } + for _, e := range list { + if e == v { + return list + } + } + return append(list, v) +} + +// FirstNonEmpty returns the first value that is not blank after trimming. +func FirstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} diff --git a/internal/ingest/decode/distro.go b/internal/ingest/decode/distro.go new file mode 100644 index 0000000..d085742 --- /dev/null +++ b/internal/ingest/decode/distro.go @@ -0,0 +1,401 @@ +package decode + +import ( + "bufio" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + "strconv" + "strings" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" +) + +// --------------------------------------------------------------------------- +// Alpine secdb: per-branch JSON, package -> secfixes -> CVE list +// --------------------------------------------------------------------------- + +type alpineDoc struct { + APKURL string `json:"apkurl"` + Archs []any `json:"archs"` + Reponame string `json:"reponame"` + URLPrefix string `json:"urlprefix"` + DistroVersion string `json:"distroversion"` + Packages []struct { + Pkg struct { + Name string `json:"name"` + Secfixes map[string][]string `json:"secfixes"` + } `json:"pkg"` + } `json:"packages"` +} + +// AlpineSecdb decodes one Alpine secdb branch file, already read and bounded by +// the caller, and emits one advisory per (branch, CVE). +// +// The branch is part of the id because the same CVE is fixed at different +// versions on different Alpine branches, and collapsing them would lose the +// version that actually matters to a host. +// +// A document that does not parse emits nothing and is NOT an error: a secdb +// file is one member of a bulk pull, and the caller counts the skip. +func (dc *Decoder) AlpineSecdb(raw []byte, emit func(Record) error) (int, error) { + var d alpineDoc + if err := json.Unmarshal(raw, &d); err != nil { + return 0, nil + } + + branch := dc.s(FirstNonEmpty(d.DistroVersion, "alpine")) + byCVE := map[string]*Record{} + var order []string + for _, p := range d.Packages { + pkg := dc.s(p.Pkg.Name) + if pkg == "" { + continue + } + for version, cves := range p.Pkg.Secfixes { + fixed := dc.s(version) + for _, cve := range cves { + id := dc.s(strings.TrimSpace(cve)) + if id == "" { + continue + } + key := branch + "/" + id + rec, ok := byCVE[key] + if !ok { + rec = &Record{ + Source: dc.feedID, + SourceID: key, + State: cache.AdvisoryPublished, + Raw: raw, + } + if IsCVEID(id) { + rec.CVEID = id + rec.Aliases = append(rec.Aliases, id) + } + byCVE[key] = rec + order = append(order, key) + } + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: "apk", + Package: pkg, + Fixed: fixed, + // Alpine ships backported fixes with an -rN suffix, which + // is precisely the case an upstream range gets wrong. + DistroBackport: true, + }) + } + } + } + // Deterministic order: the map iteration above is not, and an importer + // whose row order varies between runs is a resume cursor that means + // different things on different days. + sort.Strings(order) + for _, key := range order { + if err := emit(*byCVE[key]); err != nil { + return 0, err + } + } + return len(order), nil +} + +// --------------------------------------------------------------------------- +// Red Hat CSAF/VEX +// --------------------------------------------------------------------------- +// +// OVAL v2 is deprecated and must not be ingested: since 2024-07-10 Red Hat +// publishes CSAF for every RHSA and VEX for every CVE touching the portfolio +// (research/06 S19). This decoder reads the shape both share. + +type csafDoc struct { + Document struct { + Category string `json:"category"` + Title string `json:"title"` + Tracking struct { + ID string `json:"id"` + InitialReleaseDate string `json:"initial_release_date"` + CurrentReleaseDate string `json:"current_release_date"` + Status string `json:"status"` + } `json:"tracking"` + Notes []struct { + Category string `json:"category"` + Text string `json:"text"` + } `json:"notes"` + References []struct { + URL string `json:"url"` + } `json:"references"` + } `json:"document"` + Vulnerabilities []struct { + CVE string `json:"cve"` + Title string `json:"title"` + Notes []struct { + Category string `json:"category"` + Text string `json:"text"` + } `json:"notes"` + ProductStatus struct { + Fixed []string `json:"fixed"` + KnownAffected []string `json:"known_affected"` + } `json:"product_status"` + Scores []struct { + CVSSv3 struct { + VectorString string `json:"vectorString"` + BaseScore float64 `json:"baseScore"` + BaseSeverity string `json:"baseSeverity"` + } `json:"cvss_v3"` + } `json:"scores"` + } `json:"vulnerabilities"` +} + +// CSAF decodes one Red Hat CSAF or VEX document. +func (dc *Decoder) CSAF(raw []byte) (Record, bool, error) { + var d csafDoc + if err := json.Unmarshal(raw, &d); err != nil { + return Record{}, false, err + } + if d.Document.Tracking.ID == "" { + return Record{}, false, nil + } + + rec := Record{ + Source: dc.feedID, + SourceID: dc.s(d.Document.Tracking.ID), + Published: dc.s(d.Document.Tracking.InitialReleaseDate), + Modified: dc.s(d.Document.Tracking.CurrentReleaseDate), + State: cache.AdvisoryPublished, + Raw: raw, + } + rec.Description = dc.s(d.Document.Title) + for _, n := range d.Document.Notes { + if n.Category == "description" || n.Category == "summary" { + rec.Description = dc.s(strings.TrimSpace(rec.Description + "\n\n" + n.Text)) + } + } + for _, r := range d.Document.References { + if r.URL != "" { + rec.References = AppendUnique(rec.References, dc.s(r.URL)) + } + } + if strings.EqualFold(d.Document.Tracking.Status, "withdrawn") { + rec.State = cache.AdvisoryWithdrawn + rec.TombstonedAt = rec.Modified + } + + for _, v := range d.Vulnerabilities { + if IsCVEID(v.CVE) { + id := dc.s(v.CVE) + rec.Aliases = AppendUnique(rec.Aliases, id) + if rec.CVEID == "" { + rec.CVEID = id + } + } + for _, s := range v.Scores { + if s.CVSSv3.VectorString != "" && rec.CVSSVector == "" { + rec.CVSSVector = dc.s(s.CVSSv3.VectorString) + rec.Severity = dc.s(s.CVSSv3.BaseSeverity) + score := s.CVSSv3.BaseScore + rec.CVSSScore = score + } + } + // A VEX "fixed" product id is an RPM NEVRA. The fix is BACKPORTED — + // Red Hat patches without bumping the upstream version — which is the + // column that defeats the CVE-2023-32681 / RHSA-2023:4520 false + // positive class (research/12 §3). + for _, p := range v.ProductStatus.Fixed { + name, version := SplitNEVRA(p) + if name == "" { + continue + } + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: "rpm", Package: dc.s(name), + Fixed: dc.s(version), DistroBackport: true, + }) + } + for _, p := range v.ProductStatus.KnownAffected { + name, version := SplitNEVRA(p) + if name == "" { + continue + } + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: "rpm", Package: dc.s(name), + Introduced: dc.s(version), DistroBackport: true, + }) + } + } + return rec, true, nil +} + +// SplitNEVRA pulls a package name and version out of a CSAF product id such as +// "Red Hat Enterprise Linux 9:openssl-1:3.0.7-24.el9.x86_64". It is deliberately +// conservative: a shape it does not recognise yields an empty name and is +// dropped, because a wrong package name in `affected` is a false positive +// against a package nobody has. +func SplitNEVRA(product string) (string, string) { + p := strings.TrimSpace(product) + + // A product id is ":", and only the FIRST + // colon separates the two — the second colon, if there is one, is the + // RPM epoch inside the NEVRA. Splitting on the last colon (the obvious + // first attempt) strips the package name and leaves the version, which + // then reaches `affected.package` as a version string. + if i := strings.Index(p, ":"); i >= 0 { + if candidate := strings.TrimSpace(p[i+1:]); looksLikeNEVRA(candidate) { + p = candidate + } + } + + // name-[epoch:]version-release.arch. The name ends at the first hyphen + // followed by a digit, which is where RPM's version field begins. + idx := -1 + for k := 0; k+1 < len(p); k++ { + if p[k] == '-' && p[k+1] >= '0' && p[k+1] <= '9' { + idx = k + break + } + } + if idx <= 0 { + return "", "" + } + name, rest := p[:idx], p[idx+1:] + if j := strings.Index(rest, ":"); j >= 0 { + rest = rest[j+1:] + } + return name, stripArchSuffix(rest) +} + +// rpmArchSuffixes is the ALLOWLIST of architecture suffixes a NEVRA may end +// with. It is an allowlist because the alternative — "drop whatever follows +// the last dot" — eats `.el9`, and a release qualifier removed from a version +// is a comparison against a different package. +// +// A suffix not on this list is LEFT IN PLACE. That is the conservative +// direction: an unknown arch stays part of the version string, which can only +// make the comparator refuse or over-report, never silently under-report. +var rpmArchSuffixes = map[string]bool{ + "noarch": true, "src": true, "nosrc": true, + "x86_64": true, "i386": true, "i486": true, "i586": true, "i686": true, "athlon": true, + "aarch64": true, "armv5tel": true, "armv6hl": true, "armv7hl": true, "armv7hnl": true, + "ppc": true, "ppc64": true, "ppc64le": true, "ppc64p7": true, + "s390": true, "s390x": true, "riscv64": true, "loongarch64": true, + "sparc": true, "sparc64": true, "sparcv9": true, "ia64": true, "alpha": true, + "mips": true, "mipsel": true, "mips64": true, "mips64el": true, +} + +// stripArchSuffix removes the trailing "." of an RPM NEVRA. +// +// WHY THIS EXISTS. Without it, a Red Hat VEX "fixed" product id such as +// +// Red Hat Enterprise Linux 9:python3-requests-0:2.25.1-3.el9.noarch +// +// lands in `affected.fixed` as "2.25.1-3.el9.noarch". `fixed` is an EXCLUSIVE +// upper bound, and under RPM version comparison "2.25.1-3.el9" sorts BELOW +// "2.25.1-3.el9.noarch", so a host running EXACTLY the fixed package is +// reported vulnerable — by the vendor advisory whose whole purpose is to say +// it is not. That is the CVE-2023-32681 / RHSA-2023:4520 false-positive class +// (research/12 §3) reintroduced by the decoder, on the one path Lane A relies +// on to defeat it. +// +// Found by A.21's end-to-end harness, and invisible from inside this package: +// A.8's own CSAF test asserts that a backported rpm range EXISTS and never +// asserts what its `fixed` value is, so nothing compared the string to a real +// installed version until a comparator was put on the other end of it. +func stripArchSuffix(version string) string { + dot := strings.LastIndex(version, ".") + if dot <= 0 { + return version + } + if rpmArchSuffixes[version[dot+1:]] { + return version[:dot] + } + return version +} + +func looksLikeNEVRA(s string) bool { + for k := 0; k+1 < len(s); k++ { + if s[k] == '-' && s[k+1] >= '0' && s[k+1] <= '9' { + return true + } + } + return false +} + +// --------------------------------------------------------------------------- +// EPSS: a CSV with comment lines, streamed row by row +// --------------------------------------------------------------------------- + +// EPSS decodes the EPSS daily CSV, emitting one record per scored CVE. +// +// A malformed line ENDS the file rather than the import: an EPSS score is never +// load-bearing for a verdict (it is Tier 3, opt-in, risk-accepted) and what has +// been read is still usable. +func (dc *Decoder) EPSS(br *bufio.Reader, emit func(Record) error) (int, error) { + asOf := "" + // Skip and remember the leading '#' comment lines; the model date lives + // there and it is what as_of should say about an EPSS score. + for { + peek, err := br.Peek(1) + if err != nil || len(peek) == 0 || peek[0] != '#' { + break + } + line, err := br.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return 0, fmt.Errorf("decode: reading EPSS header: %w", err) + } + if i := strings.Index(line, "score_date:"); i >= 0 { + asOf = strings.TrimSpace(strings.TrimSuffix(line[i+len("score_date:"):], "\n")) + } + if err != nil { + return 0, nil + } + } + + r := csv.NewReader(br) + r.FieldsPerRecord = -1 + r.ReuseRecord = true + header, err := r.Read() + if err != nil { + return 0, nil + } + col := map[string]int{} + for i, h := range header { + col[strings.ToLower(strings.TrimSpace(h))] = i + } + cveCol, ok := col["cve"] + if !ok { + return 0, nil + } + epssCol, hasEPSS := col["epss"] + + n := 0 + for { + row, err := r.Read() + if errors.Is(err, io.EOF) { + return n, nil + } + if err != nil { + return n, nil + } + if cveCol >= len(row) || !IsCVEID(row[cveCol]) { + continue + } + rec := Record{ + Source: dc.feedID, + SourceID: dc.s(row[cveCol]), + CVEID: dc.s(row[cveCol]), + State: cache.AdvisoryPublished, + EPSSAsOf: dc.s(asOf), + Raw: []byte(strings.Join(row, ",")), + } + rec.Aliases = append(rec.Aliases, rec.CVEID) + if hasEPSS && epssCol < len(row) { + if f, err := strconv.ParseFloat(strings.TrimSpace(row[epssCol]), 64); err == nil { + rec.EPSSScore = f + } + } + n++ + if err := emit(rec); err != nil { + return n, err + } + } +} diff --git a/internal/ingest/decode/formats.go b/internal/ingest/decode/formats.go new file mode 100644 index 0000000..0df2850 --- /dev/null +++ b/internal/ingest/decode/formats.go @@ -0,0 +1,374 @@ +package decode + +import ( + "encoding/json" + "strings" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" +) + +// --------------------------------------------------------------------------- +// OSV, and therefore GHSA: github/advisory-database is OSV format +// --------------------------------------------------------------------------- + +type osvDoc struct { + SchemaVersion string `json:"schema_version"` + ID string `json:"id"` + Withdrawn string `json:"withdrawn"` + Published string `json:"published"` + Modified string `json:"modified"` + Summary string `json:"summary"` + Details string `json:"details"` + Aliases []string `json:"aliases"` + Related []string `json:"related"` + Severity []osvSeverity `json:"severity"` + References []osvReference `json:"references"` + Affected []osvAffected `json:"affected"` + DatabaseSpec json.RawMessage `json:"database_specific"` +} + +type osvSeverity struct { + Type string `json:"type"` + Score string `json:"score"` +} + +type osvReference struct { + Type string `json:"type"` + URL string `json:"url"` +} + +type osvAffected struct { + Package struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + PURL string `json:"purl"` + } `json:"package"` + Ranges []struct { + Type string `json:"type"` + Events []struct { + Introduced string `json:"introduced"` + Fixed string `json:"fixed"` + LastAffect string `json:"last_affected"` + } `json:"events"` + } `json:"ranges"` + Versions []string `json:"versions"` +} + +// OSV decodes one OSV-schema advisory document. +// +// The bool is false for a document that parsed as JSON but carries no `id`, +// which is not an error: it is how "this JSON object is not an OSV advisory" +// is reported to a dispatch that has to decide between skipping and refusing. +func (dc *Decoder) OSV(raw []byte) (Record, bool, error) { + var d osvDoc + if err := json.Unmarshal(raw, &d); err != nil { + return Record{}, false, err + } + if d.ID == "" { + return Record{}, false, nil + } + + rec := Record{ + Source: dc.feedID, + SourceID: dc.s(d.ID), + State: cache.AdvisoryPublished, + Raw: raw, + } + if d.Withdrawn != "" { + rec.State = cache.AdvisoryWithdrawn + rec.TombstonedAt = dc.s(d.Withdrawn) + } + rec.Published = dc.s(d.Published) + rec.Modified = dc.s(d.Modified) + rec.Description = dc.s(strings.TrimSpace(d.Summary + "\n\n" + d.Details)) + for _, s := range d.Severity { + if strings.HasPrefix(strings.ToUpper(s.Type), "CVSS") { + rec.CVSSVector = dc.s(s.Score) + break + } + } + for _, r := range d.References { + if r.URL != "" { + rec.References = append(rec.References, dc.s(r.URL)) + } + } + + // A CVE alias is a nullable, indexed alias and never the primary key + // (research/06 Risk #2): GHSA advisories frequently carry none. + for _, a := range append(append([]string{}, d.Aliases...), d.Related...) { + if IsCVEID(a) { + rec.Aliases = AppendUnique(rec.Aliases, dc.s(a)) + } + } + if IsCVEID(d.ID) { + rec.CVEID = rec.SourceID + rec.Aliases = AppendUnique(rec.Aliases, rec.SourceID) + } else if len(rec.Aliases) > 0 { + rec.CVEID = rec.Aliases[0] + } + + for _, a := range d.Affected { + eco := dc.s(a.Package.Ecosystem) + pkg := dc.s(a.Package.Name) + if eco == "" || pkg == "" { + continue + } + purl := dc.s(a.Package.PURL) + // A distro's OSV export carries a backported fix: the version number + // does not move upstream, so an upstream range would call a patched + // package vulnerable. research/12 §3, the CVE-2023-32681 / + // RHSA-2023:4520 class of false positive. + backport := IsDistroEcosystem(eco) + emitted := false + for _, rg := range a.Ranges { + var introduced string + for _, ev := range rg.Events { + switch { + case ev.Introduced != "": + introduced = dc.s(ev.Introduced) + case ev.Fixed != "": + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: eco, Package: pkg, PURL: purl, + Introduced: introduced, Fixed: dc.s(ev.Fixed), DistroBackport: backport, + }) + emitted = true + case ev.LastAffect != "": + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: eco, Package: pkg, PURL: purl, + Introduced: introduced, DistroBackport: backport, + }) + emitted = true + } + } + } + if !emitted { + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: eco, Package: pkg, PURL: purl, DistroBackport: backport, + }) + } + } + return rec, true, nil +} + +// --------------------------------------------------------------------------- +// CVE 5.x — the cvelistV5 baseline and the shape the deltaLog names +// --------------------------------------------------------------------------- + +type cve5Doc struct { + DataType string `json:"dataType"` + DataVersion string `json:"dataVersion"` + CVEMetadata struct { + CVEID string `json:"cveId"` + State string `json:"state"` + Published string `json:"datePublished"` + Updated string `json:"dateUpdated"` + Rejected string `json:"dateRejected"` + } `json:"cveMetadata"` + Containers struct { + CNA cve5Container `json:"cna"` + ADP []cve5Container `json:"adp"` + } `json:"containers"` +} + +type cve5Container struct { + Descriptions []struct { + Lang string `json:"lang"` + Value string `json:"value"` + } `json:"descriptions"` + References []struct { + URL string `json:"url"` + } `json:"references"` + Metrics []struct { + CVSSv31 *cve5CVSS `json:"cvssV3_1"` + CVSSv30 *cve5CVSS `json:"cvssV3_0"` + CVSSv40 *cve5CVSS `json:"cvssV4_0"` + } `json:"metrics"` + Affected []struct { + Vendor string `json:"vendor"` + Product string `json:"product"` + PackageN string `json:"packageName"` + Repo string `json:"repo"` + CPEs []string `json:"cpes"` + Versions []struct { + Version string `json:"version"` + LessThan string `json:"lessThan"` + LessOrEqual string `json:"lessThanOrEqual"` + Status string `json:"status"` + } `json:"versions"` + } `json:"affected"` +} + +type cve5CVSS struct { + VectorString string `json:"vectorString"` + BaseScore float64 `json:"baseScore"` + BaseSeverity string `json:"baseSeverity"` +} + +// CVE5 decodes one CVE Record Format 5.x document. +// +// An unknown dataVersion sets ParseDegraded and the record is still returned. +// Lane A exit criterion 23 and spine S6: a record from a newer schema is +// persisted and flagged, never dropped. +func (dc *Decoder) CVE5(raw []byte) (Record, bool, error) { + var d cve5Doc + if err := json.Unmarshal(raw, &d); err != nil { + return Record{}, false, err + } + if d.CVEMetadata.CVEID == "" { + return Record{}, false, nil + } + + rec := Record{ + Source: dc.feedID, + SourceID: dc.s(d.CVEMetadata.CVEID), + CVEID: dc.s(d.CVEMetadata.CVEID), + Published: dc.s(d.CVEMetadata.Published), + Modified: dc.s(d.CVEMetadata.Updated), + State: cache.AdvisoryPublished, + DataVersion: dc.s(d.DataVersion), + ParseDegraded: !KnownCVEDataVersion(d.DataVersion), + Raw: raw, + } + rec.Aliases = append(rec.Aliases, rec.CVEID) + if strings.EqualFold(d.CVEMetadata.State, "REJECTED") { + rec.State = cache.AdvisoryRejected + rec.TombstonedAt = dc.s(FirstNonEmpty(d.CVEMetadata.Rejected, d.CVEMetadata.Updated)) + } + + containers := append([]cve5Container{d.Containers.CNA}, d.Containers.ADP...) + for _, c := range containers { + for _, desc := range c.Descriptions { + if rec.Description == "" && (desc.Lang == "" || strings.HasPrefix(strings.ToLower(desc.Lang), "en")) { + rec.Description = dc.s(desc.Value) + } + } + for _, ref := range c.References { + if ref.URL != "" { + rec.References = AppendUnique(rec.References, dc.s(ref.URL)) + } + } + for _, m := range c.Metrics { + for _, v := range []*cve5CVSS{m.CVSSv40, m.CVSSv31, m.CVSSv30} { + if v == nil || v.VectorString == "" || rec.CVSSVector != "" { + continue + } + rec.CVSSVector = dc.s(v.VectorString) + rec.Severity = dc.s(v.BaseSeverity) + score := v.BaseScore + rec.CVSSScore = score + } + } + for _, a := range c.Affected { + pkg := dc.s(FirstNonEmpty(a.PackageN, a.Product)) + if pkg == "" { + continue + } + eco := dc.s(FirstNonEmpty(a.Vendor, "cpe")) + for _, v := range a.Versions { + if strings.EqualFold(v.Status, "unaffected") { + continue + } + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: eco, + Package: pkg, + Introduced: dc.s(v.Version), + Fixed: dc.s(FirstNonEmpty(v.LessThan, v.LessOrEqual)), + }) + } + } + } + return rec, true, nil +} + +// --------------------------------------------------------------------------- +// CISA KEV +// --------------------------------------------------------------------------- + +// kevDoc keeps its entries as RAW MESSAGES rather than as decoded structs. +// +// That is not a style choice: `advisory.raw_json` stores the publisher's bytes +// verbatim, and re-marshalling a decoded struct would store Anvil's rendering +// of the entry instead — different key order, dropped unknown fields, and a +// different digest from the one the streaming importer writes for the same +// catalogue entry. +type kevDoc struct { + CatalogVersion string `json:"catalogVersion"` + Vulnerabilities []json.RawMessage `json:"vulnerabilities"` +} + +type kevEntry struct { + CVEID string `json:"cveID"` + VendorProject string `json:"vendorProject"` + Product string `json:"product"` + VulnerabilityName string `json:"vulnerabilityName"` + DateAdded string `json:"dateAdded"` + ShortDescription string `json:"shortDescription"` + RequiredAction string `json:"requiredAction"` + DueDate string `json:"dueDate"` + Notes string `json:"notes"` +} + +// KEVEntry decodes ONE element of the KEV catalogue's `vulnerabilities` array. +// +// It is the shared unit deliberately: the bulk importer streams the catalogue +// element by element (it is walking a 570 MB archive and will not hold a file +// in memory) while the delta path is handed a body the poller already read +// under its own cap. Those are different traversals of the same array, and +// before this package they were also two different mappings of the same entry. +// +// The bool is false for an element that is not a KEV entry — malformed, or +// carrying no cveID. The caller decides what that means; both callers today +// skip the element, which is the one place a skip is right because the KEV +// catalogue is a single document holding every entry and one bad element must +// not cost the rest. +func (dc *Decoder) KEVEntry(entryRaw []byte) (Record, bool, error) { + var e kevEntry + if err := json.Unmarshal(entryRaw, &e); err != nil { + return Record{}, false, err + } + if e.CVEID == "" { + return Record{}, false, nil + } + rec := Record{ + Source: dc.feedID, + SourceID: dc.s(e.CVEID), + CVEID: dc.s(e.CVEID), + Published: dc.s(e.DateAdded), + State: cache.AdvisoryPublished, + KEV: true, + Description: dc.s(strings.TrimSpace(e.VulnerabilityName + "\n\n" + e.ShortDescription + "\n\n" + e.RequiredAction)), + Raw: append([]byte(nil), entryRaw...), + } + rec.Aliases = append(rec.Aliases, rec.CVEID) + if pkg := dc.s(e.Product); pkg != "" { + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: dc.s(FirstNonEmpty(e.VendorProject, "vendor")), + Package: pkg, + }) + } + if e.Notes != "" { + rec.References = append(rec.References, dc.s(e.Notes)) + } + return rec, true, nil +} + +// KEVDocument decodes a whole KEV catalogue held in memory. +// +// A malformed catalogue is an error — the caller asked for a document it had +// already read and bounded — while a malformed ENTRY inside a well-formed +// catalogue is skipped by KEVEntry's contract. +func (dc *Decoder) KEVDocument(raw []byte) ([]Record, error) { + var d kevDoc + if err := json.Unmarshal(raw, &d); err != nil { + return nil, err + } + out := make([]Record, 0, len(d.Vulnerabilities)) + for _, entryRaw := range d.Vulnerabilities { + rec, ok, err := dc.KEVEntry(entryRaw) + if err != nil || !ok { + continue + } + out = append(out, rec) + } + return out, nil +} diff --git a/internal/ingest/decode/nevra_test.go b/internal/ingest/decode/nevra_test.go new file mode 100644 index 0000000..1c3a4f4 --- /dev/null +++ b/internal/ingest/decode/nevra_test.go @@ -0,0 +1,96 @@ +package decode + +import "testing" + +// TestSplitNEVRADropsTheArchAndNothingElse is the direct test of the fix A.21's +// end-to-end harness forced. +// +// WHY IT MATTERS. `affected.fixed` is an EXCLUSIVE upper bound. A Red Hat VEX +// "fixed" product id that leaves its architecture on the version — +// "2.25.1-3.el9.noarch" — sorts ABOVE the installed "2.25.1-3.el9" under RPM +// version comparison, so a host running EXACTLY the fixed package is reported +// vulnerable by the advisory whose purpose is to say it is not. That is the +// CVE-2023-32681 / RHSA-2023:4520 false-positive class (research/12 §3) +// reintroduced by the decoder, on the one path Lane A relies on to defeat it. +// +// THE CORPUS IS HAND-WRITTEN. Every product id below is typed from the shape +// Red Hat's CSAF product tree documents, not captured from this package's own +// output — a test whose corpus comes from the implementation is not a test. +// +// THE ARCH LIST IS AN ALLOWLIST and the last two cases are why: a suffix that +// is not a known architecture STAYS, because "drop whatever follows the last +// dot" eats `.el9`, and a release qualifier removed from a version is a +// comparison against a different package. +func TestSplitNEVRADropsTheArchAndNothingElse(t *testing.T) { + cases := []struct { + product string + wantName string + wantVersion string + why string + }{ + { + product: "Red Hat Enterprise Linux 9:python3-requests-0:2.25.1-3.el9.noarch", + wantName: "python3-requests", wantVersion: "2.25.1-3.el9", + why: "the backport case: a host at 2.25.1-3.el9 must not be flagged by this range", + }, + { + product: "Red Hat Enterprise Linux 9:openssl-1:3.0.7-24.el9.x86_64", + wantName: "openssl", wantVersion: "3.0.7-24.el9", + why: "an epoch AND an arch, both removed", + }, + { + product: "Red Hat Enterprise Linux 8:kernel-0:4.18.0-513.5.1.el8_9.aarch64", + wantName: "kernel", wantVersion: "4.18.0-513.5.1.el8_9", + why: "a release with its own dots keeps every one of them", + }, + { + product: "AppStream-9.3.0.GA:nodejs-1:18.18.2-2.module+el9.3.0+20334+3dd35b3f.ppc64le", + wantName: "nodejs", wantVersion: "18.18.2-2.module+el9.3.0+20334+3dd35b3f", + why: "a modular release: only the trailing arch goes", + }, + { + product: "Red Hat Enterprise Linux 9:python3-requests-0:2.25.1-3.el9", + wantName: "python3-requests", wantVersion: "2.25.1-3.el9", + why: "NO arch at all — the release must survive untouched, which is what an " + + "allowlist buys over trimming the last dotted segment", + }, + { + product: "Red Hat Enterprise Linux 9:widget-0:1.2.3-4.el9.sparc128", + wantName: "widget", wantVersion: "1.2.3-4.el9.sparc128", + why: "an UNKNOWN arch stays in place. Leaving it can make the comparator " + + "over-report or refuse; guessing it away can silently under-report, and " + + "under-reporting is the failure a security tool must not choose", + }, + { + product: "Red Hat Enterprise Linux 9", + wantName: "", wantVersion: "", + why: "a product tree branch with no NEVRA yields nothing rather than a " + + "package name nobody has", + }, + } + + for _, tc := range cases { + name, version := SplitNEVRA(tc.product) + if name != tc.wantName || version != tc.wantVersion { + t.Errorf("SplitNEVRA(%q) = (%q, %q), want (%q, %q)\n %s", + tc.product, name, version, tc.wantName, tc.wantVersion, tc.why) + } + } +} + +// TestTheArchAllowlistIsNotEmpty keeps the guard from going inert: an empty +// map would make stripArchSuffix a no-op and every case above would still +// "pass" for the no-arch rows. +func TestTheArchAllowlistIsNotEmpty(t *testing.T) { + if len(rpmArchSuffixes) == 0 { + t.Fatal("the arch allowlist is empty, so stripArchSuffix strips nothing and the NEVRA " + + "tests above pass over a function that does not run") + } + if !rpmArchSuffixes["noarch"] || !rpmArchSuffixes["x86_64"] { + t.Error("the two architectures Red Hat's own advisories use most are not on the allowlist") + } + if rpmArchSuffixes["el9"] { + t.Error("a release qualifier is on the ARCHITECTURE allowlist; it would be stripped off " + + "every version and every comparison would be against a different package") + } +} diff --git a/internal/ingest/delta/delta_test.go b/internal/ingest/delta/delta_test.go index 4204748..b64bab2 100644 --- a/internal/ingest/delta/delta_test.go +++ b/internal/ingest/delta/delta_test.go @@ -1164,21 +1164,46 @@ func TestTheDeltaLogsOwnLinksAreNeverFetched(t *testing.T) { // --------------------------------------------------------------------------- // TestDeltaAndBootstrapDecodeTheSameBytesIntoTheSameRows is the guard on the -// duplication this package was forced into. +// duplication between the two importers. // -// internal/ingest/bootstrap's decoders are unexported, so A.14 re-derives them, -// and two producers writing one table from one document is precisely the drift -// spine S6 names for the fingerprint. If they diverge, A.15's weekly self-heal -// "restores" the same rows forever and nothing surfaces why. +// WHAT IT GUARDED, AND WHAT IT GUARDS NOW. It was written when +// internal/ingest/bootstrap's decoders were unexported and this package had +// re-derived them: two DECODERS for one wire format, which A.21 ended by +// extracting internal/ingest/decode (orchestrator ruling G11). Decoding is now +// one implementation and a decoder divergence is no longer expressible, so +// this test could have become tautological — a test that cannot fail is worse +// than no test. +// +// IT DID NOT. The two WRITE PATHS are still two: bootstrap's writer binds +// cache.UpsertAdvisorySQL from its own staleness, tombstone and licence-column +// derivation, and delta's writeRecord binds the same statement from its own. +// Those are the halves that can still drift, and a divergence in either still +// makes A.15's weekly self-heal restore the same rows forever with nothing +// surfacing why. Verified RED at extraction time: perturbing one bound column +// in bootstrap's writer fails this test on every row. // // The fixture documents are written in THIS file and handed to both importers. -// Neither decoder's output is the other's expectation, and neither is compared +// Neither writer's output is the other's expectation, and neither is compared // against a table derived from itself. func TestDeltaAndBootstrapDecodeTheSameBytesIntoTheSameRows(t *testing.T) { const feedID = "conformance" + // A REJECTED record is in the corpus deliberately. Without one, every + // comparison below runs over published rows only, and the two writers' + // handling of a TOMBSTONED advisory — which is where they actually + // diverged — is never exercised. A cross-producer fixture that contains no + // instance of the state the producers treat differently is a fixture that + // cannot fail. + rejected := bytes.Replace(cve5Record("CVE-2026-0003", "2026-08-09T00:00:00Z"), + []byte(`"state":"PUBLISHED"`), []byte(`"state":"REJECTED"`), 1) + if bytes.Contains(rejected, []byte(`"state":"PUBLISHED"`)) { + t.Fatal("the rejected fixture still says PUBLISHED; the tombstone half of this test would " + + "prove nothing") + } + members := []zipMember{ {"cves/CVE-2026-0001.json", cve5Record("CVE-2026-0001", "2026-08-09T00:00:00Z")}, + {"cves/CVE-2026-0003.json", rejected}, {"osv/GHSA-test-000001.json", osvRecord(1)}, {"osv/USN-000002-1.json", ubuntuOSVRecord(2)}, {"kev/known_exploited_vulnerabilities.json", kevCatalogue("CVE-2026-9001", "CVE-2026-9002")}, @@ -1271,6 +1296,28 @@ FROM affected ORDER BY source, source_id, ecosystem, package, ifnull(introduced, t.Errorf("the two importers disagree about `affected`:\nA.8: %v\nA.14: %v", la, ra) } + // advisory_fts IS COMPARED, and it was not until A.21. + // + // The omission cost a real divergence: A.8's writer indexed every record + // including tombstoned ones while A.14's de-indexed them, so a REJECTED + // advisory stayed searchable on one path and not the other — and A.15's + // weekly baseline re-runs A.8's path, which would have re-indexed it every + // week forever. Three columns compared and one not is how a cross-producer + // guard passes over the producer's actual difference. + // The query asks whether an index ROW EXISTS, not what it contains. + // advisory_fts is an EXTERNAL-CONTENT (contentless) FTS5 table: selecting + // its columns returns NULL by design, so a comparison of column values + // compares NULL to NULL and passes over any divergence at all. + const ftsQ = ` +SELECT a.source, a.source_id, a.state, + CASE WHEN f.rowid IS NULL THEN 'not-indexed' ELSE 'indexed' END +FROM advisory a LEFT JOIN advisory_fts f ON f.rowid = a.rowid +ORDER BY a.source, a.source_id` + lf, rf := dumpRows(t, a, ftsQ), dumpRows(t, b, ftsQ) + if !reflect.DeepEqual(lf, rf) { + t.Errorf("the two importers disagree about `advisory_fts`:\nA.8: %v\nA.14: %v", lf, rf) + } + const aliasQ = `SELECT cve_id, source, source_id FROM cve_alias ORDER BY cve_id, source, source_id` lc, rc := dumpRows(t, a, aliasQ), dumpRows(t, b, aliasQ) if !reflect.DeepEqual(lc, rc) { diff --git a/internal/ingest/delta/upsert.go b/internal/ingest/delta/upsert.go index 4d648a3..9a08d09 100644 --- a/internal/ingest/delta/upsert.go +++ b/internal/ingest/delta/upsert.go @@ -69,6 +69,7 @@ import ( "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" "github.com/Susquehanna-Syntax/Anvil/internal/ingest/config" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/decode" "github.com/Susquehanna-Syntax/Anvil/internal/ingest/license" "github.com/Susquehanna-Syntax/Anvil/internal/ingest/sanitize" ) @@ -195,98 +196,23 @@ func queryRowDB(ctx context.Context, db *sql.DB, q string, args ...any) (*sql.Ro // The record model // --------------------------------------------------------------------------- -// AffectedRange is one row of the cache's `affected` table: a package, an -// ecosystem, and the version window a comparator answers against. +// AffectedRange and Record are the row shapes internal/ingest/decode defines. // -// plan/00-SPINE.md S1 is why these rows are the point of Lane A at all: -// "CVE/OSV/GHSA describe vulnerable PACKAGE VERSIONS, and a version comparator -// answers that exactly and for free." -type AffectedRange struct { - Ecosystem string - Package string - PURL string - - // Introduced and Fixed bound the vulnerable window. Either may be empty: - // an OSV entry with a `last_affected` event and no `fixed` has no fixed - // version, and that is a fact about the advisory, not a parse failure. - Introduced string - Fixed string - - // DistroBackport marks a range that came from a vendor or distro advisory - // rather than from upstream. research/12 §3: a distro backports a fix - // without moving the upstream version, so an upstream range calls a - // patched package vulnerable. This column is what A.17 needs to not do - // that. - DistroBackport bool -} - -// Record is one decoded advisory, already sanitized, ready to bind. -// -// It is EXPORTED because A.15's reconciliation writes the same rows through -// the same path. A second write path for the same table is how a schema +// THEY ARE GO TYPE ALIASES AND NOT CONVERSIONS, and they are EXPORTED because +// A.15's reconciliation and A.16's drift handler build the same rows and reach +// the same statements — a second write path for one table is how a schema // invariant survives in one writer and not the other. -type Record struct { - // Source is the feed id and SourceID the native id within it. Together - // they are the primary key, and it is NEVER the CVE id: research/06 Risk - // #2 requires a cvelistV5 outage to be survivable by swapping sources, - // and GHSA advisories frequently carry no CVE at all. - Source string - SourceID string - - // CVEID is the nullable, indexed alias. Aliases carries the one-to-many. - CVEID string - Aliases []string - - Published string - Modified string - - // State is one of cache.AdvisoryPublished / AdvisoryWithdrawn / - // AdvisoryRejected. A non-published state MUST carry TombstonedAt, and - // the schema's advisory_tombstone_paired CHECK refuses the row otherwise - // — withdrawn advisories are tombstoned, never deleted, so that findings - // that depended on them become invalidated rather than vanishing. - State string - TombstonedAt string - - Severity string - CVSSVector string - - // CVSSScore and EPSSScore are `any` so that "no score" is SQL NULL rather - // than 0.0. A zero CVSS base score is a real value, and a comparator that - // cannot tell it from "absent" ranks an unscored advisory as harmless. - CVSSScore any - EPSSScore any - EPSSAsOf string - - KEV bool - - Description string - References []string - - Affected []AffectedRange - - // DataVersion is the record schema version the document declared, and - // ParseDegraded is spine S6's field for "this was persisted anyway". - // A.2 exit criterion 23: an unknown CVE dataVersion is PERSISTED with - // parse_degraded = 1, never dropped, because silently discarding a record - // from a newer schema is how a vulnerability disappears from a security - // tool with no error anywhere. - DataVersion string - ParseDegraded bool - - // StalenessSeconds overrides the batch-wide value when the record carries - // its own age. Zero means "use the batch's". - StalenessSeconds int - - // Raw is the document verbatim, stored in advisory.raw_json. It is never - // sanitized: the column is the publisher's bytes, and CVE-TOU requires - // records be stored byte-verbatim (research/06 §"License"). - Raw []byte -} - -// ReferencesText is the `references_text` column of advisory_fts: the -// references as one newline-separated string, each element already sanitized. -func (r Record) ReferencesText() string { return strings.Join(r.References, "\n") } +// +// The types moved out of this package under orchestrator ruling G11. Until +// A.21, internal/ingest/bootstrap's decoders were unexported, so this package +// re-derived CVE 5.x, OSV and KEV decoding, and the cache had two producers +// writing one table from one wire format. If they had drifted, A.15's weekly +// self-heal would have restored the same rows forever with nothing surfacing +// it. There is now one decoder; see internal/ingest/decode. +type ( + AffectedRange = decode.AffectedRange + Record = decode.Record +) // BatchStats counts what one Apply wrote. Every field counts WRITES, not net // growth: `affected` and `cve_alias` are replaced per advisory, so a re-upsert @@ -651,22 +577,33 @@ func parseTimestamp(s string) (time.Time, bool) { // claimed. const MaxDocumentBytes = 16 << 20 -// decoder is A.3 applied field by field, accumulating what was removed. +// decoder is this package's binding to internal/ingest/decode. // -// The `s` method is named rather than inlined for the same reason A.8 names -// its own: internal/ingest/sanitize's writer guard resolves the package-local -// call graph by NAME, so a decoder that reaches the sanitizer through one -// method is visible to the guard, and there is exactly one place a field could -// be bound without passing through it — which is none. +// EVERY WIRE FORMAT IS DECODED THERE AND NOWHERE ELSE (ruling G11). What stays +// here is the DISPATCH below, and it stays because this package's answer to an +// unreadable document is genuinely different from A.8's: see Decode. type decoder struct { feedID string - stats sanitize.SanitizeStats + dec *decode.Decoder } -func (dc *decoder) s(raw string) string { - clean, st := sanitize.Sanitize(raw) - dc.stats.Merge(st) - return clean +// decodeKEV reads the whole catalogue into memory, which is the one place this +// path deliberately differs from A.8's traversal of the same array. +// +// The reason is the caller. A.8 streams because it walks a 570 MB archive whose +// members it has not seen; this path is handed a body A.7 already read into +// memory under Options.MaxBodyBytes, so a streaming parse here would buy +// nothing and would need a second bounded reader to enforce a bound that has +// already been enforced. MaxDocumentBytes is the backstop. The per-entry +// mapping is the SHARED one, so both importers write the same row for the same +// catalogue entry — including raw_json byte for byte, because neither +// re-marshals a decoded struct. +func (dc *decoder) decodeKEV(raw []byte) ([]Record, error) { + recs, err := dc.dec.KEVDocument(raw) + if err != nil { + return nil, refuse(ErrUnrecognisedShape, "feed %q: a KEV-shaped document did not parse: %v", dc.feedID, err) + } + return recs, nil } // Decode turns one fetched document into advisory records. @@ -685,9 +622,9 @@ func (dc *decoder) s(raw string) string { // change was dropped, and the caller needs to route the feed to a path that // does understand it. SyncDelta does exactly that. func Decode(feedID string, raw []byte) ([]Record, sanitize.SanitizeStats, error) { - dc := &decoder{feedID: feedID} + dc := &decoder{feedID: feedID, dec: decode.New(feedID)} recs, err := dc.decode(raw, 0) - return recs, dc.stats, err + return recs, dc.dec.Stats(), err } // maxDecodeDepth bounds the array-of-documents recursion. One level of nesting @@ -750,14 +687,14 @@ func (dc *decoder) decode(raw []byte, depth int) ([]Record, error) { return dc.decodeKEV(trimmed) case bytes.Contains(head, []byte(`"CVE_RECORD"`)): - rec, ok, err := dc.decodeCVE5(trimmed) + rec, ok, err := dc.dec.CVE5(trimmed) if err != nil || !ok { return nil, refuse(ErrUnrecognisedShape, "feed %q: a CVE_RECORD document did not decode: %v", dc.feedID, err) } return []Record{rec}, nil default: - rec, ok, err := dc.decodeOSV(trimmed) + rec, ok, err := dc.dec.OSV(trimmed) if err != nil || !ok { return nil, refuse(ErrUnrecognisedShape, "feed %q: the document is a JSON object in no shape this decoder recognises "+ @@ -767,362 +704,6 @@ func (dc *decoder) decode(raw []byte, depth int) ([]Record, error) { } } -// --- OSV, and therefore GHSA: github/advisory-database is OSV format --- - -type osvDoc struct { - SchemaVersion string `json:"schema_version"` - ID string `json:"id"` - Withdrawn string `json:"withdrawn"` - Published string `json:"published"` - Modified string `json:"modified"` - Summary string `json:"summary"` - Details string `json:"details"` - Aliases []string `json:"aliases"` - Related []string `json:"related"` - Severity []osvSeverity `json:"severity"` - References []osvReference `json:"references"` - Affected []osvAffected `json:"affected"` -} - -type osvSeverity struct { - Type string `json:"type"` - Score string `json:"score"` -} - -type osvReference struct { - Type string `json:"type"` - URL string `json:"url"` -} - -type osvAffected struct { - Package struct { - Ecosystem string `json:"ecosystem"` - Name string `json:"name"` - PURL string `json:"purl"` - } `json:"package"` - Ranges []struct { - Type string `json:"type"` - Events []struct { - Introduced string `json:"introduced"` - Fixed string `json:"fixed"` - LastAffect string `json:"last_affected"` - } `json:"events"` - } `json:"ranges"` - Versions []string `json:"versions"` -} - -func (dc *decoder) decodeOSV(raw []byte) (Record, bool, error) { - var d osvDoc - if err := json.Unmarshal(raw, &d); err != nil { - return Record{}, false, err - } - if d.ID == "" { - return Record{}, false, nil - } - - rec := Record{ - Source: dc.feedID, - SourceID: dc.s(d.ID), - State: cache.AdvisoryPublished, - Raw: raw, - } - if d.Withdrawn != "" { - rec.State = cache.AdvisoryWithdrawn - rec.TombstonedAt = dc.s(d.Withdrawn) - } - rec.Published = dc.s(d.Published) - rec.Modified = dc.s(d.Modified) - rec.Description = dc.s(strings.TrimSpace(d.Summary + "\n\n" + d.Details)) - for _, s := range d.Severity { - if strings.HasPrefix(strings.ToUpper(s.Type), "CVSS") { - rec.CVSSVector = dc.s(s.Score) - break - } - } - for _, r := range d.References { - if r.URL != "" { - rec.References = append(rec.References, dc.s(r.URL)) - } - } - - for _, a := range append(append([]string{}, d.Aliases...), d.Related...) { - if IsCVEID(a) { - rec.Aliases = appendUnique(rec.Aliases, dc.s(a)) - } - } - if IsCVEID(d.ID) { - rec.CVEID = rec.SourceID - rec.Aliases = appendUnique(rec.Aliases, rec.SourceID) - } else if len(rec.Aliases) > 0 { - rec.CVEID = rec.Aliases[0] - } - - for _, a := range d.Affected { - eco := dc.s(a.Package.Ecosystem) - pkg := dc.s(a.Package.Name) - if eco == "" || pkg == "" { - continue - } - purl := dc.s(a.Package.PURL) - backport := isDistroEcosystem(eco) - emitted := false - for _, rg := range a.Ranges { - var introduced string - for _, ev := range rg.Events { - switch { - case ev.Introduced != "": - introduced = dc.s(ev.Introduced) - case ev.Fixed != "": - rec.Affected = append(rec.Affected, AffectedRange{ - Ecosystem: eco, Package: pkg, PURL: purl, - Introduced: introduced, Fixed: dc.s(ev.Fixed), DistroBackport: backport, - }) - emitted = true - case ev.LastAffect != "": - rec.Affected = append(rec.Affected, AffectedRange{ - Ecosystem: eco, Package: pkg, PURL: purl, - Introduced: introduced, DistroBackport: backport, - }) - emitted = true - } - } - } - if !emitted { - rec.Affected = append(rec.Affected, AffectedRange{ - Ecosystem: eco, Package: pkg, PURL: purl, DistroBackport: backport, - }) - } - } - return rec, true, nil -} - -// isDistroEcosystem marks the ecosystems whose advisories carry BACKPORTED -// fixes. research/12 §3, the CVE-2023-32681 / RHSA-2023:4520 class: a distro -// patches without moving the upstream version, so an upstream range calls a -// fixed package vulnerable. -// -// The list matches A.8's exactly. It is duplicated for the same reason the -// decoders are, and the same conformance test covers it: a divergence changes -// `affected.distro_backport` for the same bytes depending on which importer ran. -func isDistroEcosystem(eco string) bool { - lower := strings.ToLower(eco) - for _, p := range []string{ - "ubuntu", "debian", "alpine", "red hat", "redhat", "rocky", "almalinux", - "suse", "photon", "chainguard", "wolfi", "mageia", - } { - if strings.HasPrefix(lower, p) { - return true - } - } - return false -} - -// --- CVE 5.x, the shape the deltaLog names --- - -type cve5Doc struct { - DataType string `json:"dataType"` - DataVersion string `json:"dataVersion"` - CVEMetadata struct { - CVEID string `json:"cveId"` - State string `json:"state"` - Published string `json:"datePublished"` - Updated string `json:"dateUpdated"` - Rejected string `json:"dateRejected"` - } `json:"cveMetadata"` - Containers struct { - CNA cve5Container `json:"cna"` - ADP []cve5Container `json:"adp"` - } `json:"containers"` -} - -type cve5Container struct { - Descriptions []struct { - Lang string `json:"lang"` - Value string `json:"value"` - } `json:"descriptions"` - References []struct { - URL string `json:"url"` - } `json:"references"` - Metrics []struct { - CVSSv31 *cve5CVSS `json:"cvssV3_1"` - CVSSv30 *cve5CVSS `json:"cvssV3_0"` - CVSSv40 *cve5CVSS `json:"cvssV4_0"` - } `json:"metrics"` - Affected []struct { - Vendor string `json:"vendor"` - Product string `json:"product"` - PackageN string `json:"packageName"` - Repo string `json:"repo"` - CPEs []string `json:"cpes"` - Versions []struct { - Version string `json:"version"` - LessThan string `json:"lessThan"` - LessOrEqual string `json:"lessThanOrEqual"` - Status string `json:"status"` - } `json:"versions"` - } `json:"affected"` -} - -type cve5CVSS struct { - VectorString string `json:"vectorString"` - BaseScore float64 `json:"baseScore"` - BaseSeverity string `json:"baseSeverity"` -} - -// knownCVEDataVersions are the record schema versions this decoder was written -// against. An UNKNOWN one is PERSISTED with parse_degraded = 1 and never -// dropped (A.2 exit criterion 23, spine S6): silently discarding a record from -// a newer schema is how a vulnerability disappears from a security tool with no -// error anywhere. -// -// It matches A.8's list, and the conformance test is what keeps it matching. -var knownCVEDataVersions = map[string]bool{"5.0": true, "5.1": true, "5.2": true} - -func (dc *decoder) decodeCVE5(raw []byte) (Record, bool, error) { - var d cve5Doc - if err := json.Unmarshal(raw, &d); err != nil { - return Record{}, false, err - } - if d.CVEMetadata.CVEID == "" { - return Record{}, false, nil - } - - rec := Record{ - Source: dc.feedID, - SourceID: dc.s(d.CVEMetadata.CVEID), - CVEID: dc.s(d.CVEMetadata.CVEID), - Published: dc.s(d.CVEMetadata.Published), - Modified: dc.s(d.CVEMetadata.Updated), - State: cache.AdvisoryPublished, - DataVersion: dc.s(d.DataVersion), - ParseDegraded: !knownCVEDataVersions[strings.TrimSpace(d.DataVersion)], - Raw: raw, - } - rec.Aliases = append(rec.Aliases, rec.CVEID) - if strings.EqualFold(d.CVEMetadata.State, "REJECTED") { - rec.State = cache.AdvisoryRejected - rec.TombstonedAt = dc.s(firstNonEmpty(d.CVEMetadata.Rejected, d.CVEMetadata.Updated)) - } - - containers := append([]cve5Container{d.Containers.CNA}, d.Containers.ADP...) - for _, c := range containers { - for _, desc := range c.Descriptions { - if rec.Description == "" && (desc.Lang == "" || strings.HasPrefix(strings.ToLower(desc.Lang), "en")) { - rec.Description = dc.s(desc.Value) - } - } - for _, ref := range c.References { - if ref.URL != "" { - rec.References = appendUnique(rec.References, dc.s(ref.URL)) - } - } - for _, m := range c.Metrics { - for _, v := range []*cve5CVSS{m.CVSSv40, m.CVSSv31, m.CVSSv30} { - if v == nil || v.VectorString == "" || rec.CVSSVector != "" { - continue - } - rec.CVSSVector = dc.s(v.VectorString) - rec.Severity = dc.s(v.BaseSeverity) - score := v.BaseScore - rec.CVSSScore = score - } - } - for _, a := range c.Affected { - pkg := dc.s(firstNonEmpty(a.PackageN, a.Product)) - if pkg == "" { - continue - } - eco := dc.s(firstNonEmpty(a.Vendor, "cpe")) - for _, v := range a.Versions { - if strings.EqualFold(v.Status, "unaffected") { - continue - } - rec.Affected = append(rec.Affected, AffectedRange{ - Ecosystem: eco, - Package: pkg, - Introduced: dc.s(v.Version), - Fixed: dc.s(firstNonEmpty(v.LessThan, v.LessOrEqual)), - }) - } - } - } - return rec, true, nil -} - -// --- CISA KEV --- - -// kevDoc keeps its entries as RAW MESSAGES rather than as decoded structs. -// -// That is not a style choice: `advisory.raw_json` stores the publisher's bytes -// verbatim, and re-marshalling a decoded struct would store Anvil's rendering -// of the entry instead — different key order, dropped unknown fields, and a -// different digest from the one A.8 writes for the same catalogue entry. The -// conformance test compares those bytes. -type kevDoc struct { - CatalogVersion string `json:"catalogVersion"` - Vulnerabilities []json.RawMessage `json:"vulnerabilities"` -} - -type kevEntry struct { - CVEID string `json:"cveID"` - VendorProject string `json:"vendorProject"` - Product string `json:"product"` - VulnerabilityName string `json:"vulnerabilityName"` - DateAdded string `json:"dateAdded"` - ShortDescription string `json:"shortDescription"` - RequiredAction string `json:"requiredAction"` - DueDate string `json:"dueDate"` - Notes string `json:"notes"` -} - -// decodeKEV reads the whole catalogue into memory rather than streaming it, -// which is the one place this decoder deliberately differs from A.8's. -// -// The reason is the caller. A.8 streams because it walks a 570 MB archive whose -// members it has not seen; this decoder is handed a body A.7 already read into -// memory under Options.MaxBodyBytes, so a streaming parse here would buy -// nothing and would need a second bounded reader to enforce a bound that has -// already been enforced. MaxDocumentBytes is the backstop. -func (dc *decoder) decodeKEV(raw []byte) ([]Record, error) { - var d kevDoc - if err := json.Unmarshal(raw, &d); err != nil { - return nil, refuse(ErrUnrecognisedShape, "feed %q: a KEV-shaped document did not parse: %v", dc.feedID, err) - } - out := make([]Record, 0, len(d.Vulnerabilities)) - for _, entryRaw := range d.Vulnerabilities { - var e kevEntry - if err := json.Unmarshal(entryRaw, &e); err != nil || e.CVEID == "" { - // A single malformed catalogue entry is skipped, not fatal: the - // KEV catalogue is one document holding every entry, and one bad - // entry must not cost the rest. This is the one place a skip is - // right, and it is bounded to a single element. - continue - } - rec := Record{ - Source: dc.feedID, - SourceID: dc.s(e.CVEID), - CVEID: dc.s(e.CVEID), - Published: dc.s(e.DateAdded), - State: cache.AdvisoryPublished, - KEV: true, - Description: dc.s(strings.TrimSpace(e.VulnerabilityName + "\n\n" + e.ShortDescription + "\n\n" + e.RequiredAction)), - Raw: append([]byte(nil), entryRaw...), - } - rec.Aliases = append(rec.Aliases, rec.CVEID) - if pkg := dc.s(e.Product); pkg != "" { - rec.Affected = append(rec.Affected, AffectedRange{ - Ecosystem: dc.s(firstNonEmpty(e.VendorProject, "vendor")), - Package: pkg, - }) - } - if e.Notes != "" { - rec.References = append(rec.References, dc.s(e.Notes)) - } - out = append(out, rec) - } - return out, nil -} - // --------------------------------------------------------------------------- // Small shared helpers // --------------------------------------------------------------------------- @@ -1130,59 +711,15 @@ func (dc *decoder) decodeKEV(raw []byte) ([]Record, error) { // IsCVEID is the ALLOWLIST that decides whether a string from a feed may be // treated as a CVE identifier. // -// It is exported because it is not only a parsing convenience: the deltaLog -// route in delta.go uses it as the ONLY thing that may cross from feed content -// into a fetch, so its strictness is a security property and not a nicety. See -// checkRecordName. +// It is re-exported from internal/ingest/decode rather than re-implemented, +// because it is not only a parsing convenience: the deltaLog route in delta.go +// uses it as the ONLY thing that may cross from feed content into a fetch, so +// its strictness is a security property and not a nicety. See checkRecordName. +// Two copies of a security allowlist is two allowlists, and the second one is +// always the one nobody re-reads. // -// The shape is CVE-<4+ digits>-<1+ digits> and nothing else. It is an allowlist -// of characters and structure, not a denylist of dangerous ones: this project -// has lost three guards to a symbol, a verb or a wording nobody listed, and +// The shape is CVE-<4+ digits>-<1+ digits> and nothing else: an allowlist of +// characters and structure, not a denylist of dangerous ones. This project has +// lost three guards to a symbol, a verb or a wording nobody listed, and // `CVE-2024-0001/../../etc/passwd` is precisely the string a denylist misses. -func IsCVEID(s string) bool { - t := strings.TrimSpace(s) - if !strings.HasPrefix(t, "CVE-") || len(t) < 8 { - return false - } - rest := t[4:] - dash := strings.Index(rest, "-") - if dash < 4 { - return false - } - for _, r := range rest[:dash] { - if r < '0' || r > '9' { - return false - } - } - tail := rest[dash+1:] - if tail == "" { - return false - } - for _, r := range tail { - if r < '0' || r > '9' { - return false - } - } - return true -} - -func appendUnique(list []string, v string) []string { - if v == "" { - return list - } - for _, e := range list { - if e == v { - return list - } - } - return append(list, v) -} - -func firstNonEmpty(vals ...string) string { - for _, v := range vals { - if strings.TrimSpace(v) != "" { - return v - } - } - return "" -} +func IsCVEID(s string) bool { return decode.IsCVEID(s) } diff --git a/internal/match/comparator.go b/internal/match/comparator.go index 62d4dba..f4a6473 100644 --- a/internal/match/comparator.go +++ b/internal/match/comparator.go @@ -325,10 +325,47 @@ type Refusal struct { Ecosystem string Package string Purl string - Version string - Source string - SourceID string - Detail string + // PurlType is the purl `type` segment, in its canonical lowercase form, + // when THAT is the token this comparator does not implement. + // + // IT IS A SEPARATE FIELD FROM Ecosystem ON PURPOSE. A record may carry an + // ecosystem this comparator DOES implement next to a purl whose type it + // does not — `{Ecosystem: "deb", Purl: "pkg:npm/lodash@4.17.20"}` is a + // real shape when a collector fills the ecosystem column from the host and + // the purl from a lockfile — and the refusal has to name the token that + // was actually refused. Writing the purl type into Ecosystem would put + // "deb" on the operator's implement-next list, which is worse than leaving + // it empty: it would name a scheme that IS implemented. + // + // CoverageReport.EcosystemsRefused reads this field for the purl route and + // Ecosystem for the ecosystem route. See refusedIdentityToken. + PurlType string + Version string + Source string + SourceID string + Detail string +} + +// refusedIdentityToken returns the vocabulary token whose absence from this +// comparator's implementation caused the refusal, or "" when the refusal was +// not about an unimplemented scheme. +// +// It is the ONE place that decides what feeds +// CoverageReport.EcosystemsRefused, because that list answers a question — "what +// must Anvil implement next?" — whose answer does not depend on which of the +// two identity routes the input happened to arrive by. A refusal that arrives +// through the purl route is the same fact about coverage as one that arrives +// through the ecosystem route, and counting only the second makes the list +// quietly wrong in the direction that looks better: empty exactly when the +// input was WELL-FORMED enough to carry a purl. +func (r Refusal) refusedIdentityToken() string { + switch r.Reason { + case RefusalUnsupportedEcosystem: + return r.Ecosystem + case RefusalUnsupportedPurlType: + return r.PurlType + } + return "" } // Error renders the refusal. It always names the reason first, so grepping a @@ -357,6 +394,9 @@ func (r *Refusal) Error() string { if r.Purl != "" { b.WriteString(" purl=" + strconv.Quote(r.Purl)) } + if r.PurlType != "" { + b.WriteString(" purlType=" + strconv.Quote(r.PurlType)) + } if r.Source != "" || r.SourceID != "" { b.WriteString(" advisory=" + strconv.Quote(r.Source+"/"+r.SourceID)) } @@ -371,7 +411,7 @@ func (r *Refusal) Error() string { func (r Refusal) sortKey() string { return strings.Join([]string{ string(r.Reason), string(r.Scheme), r.Ecosystem, r.Package, - r.Purl, r.Version, r.Source, r.SourceID, r.Detail, + r.Purl, r.PurlType, r.Version, r.Source, r.SourceID, r.Detail, }, "\x00") } @@ -1215,9 +1255,23 @@ type CoverageReport struct { // consumer reading a stored CoverageReport knows what the producing // build could compare without having to guess from its version. SchemesImplemented []Scheme - // EcosystemsRefused is the distinct, sorted set of ecosystem strings that - // were refused. It is the list an operator uses to decide what to - // implement next. + // EcosystemsRefused is the distinct, sorted set of identity tokens that + // were refused for an unimplemented scheme. It is the list an operator + // uses to decide what to implement next. + // + // BOTH IDENTITY ROUTES FEED IT: a record refused on its `ecosystem` + // column (RefusalUnsupportedEcosystem) contributes that column, and a + // record refused on its purl `type` (RefusalUnsupportedPurlType) + // contributes the type in its canonical lowercase form. The two routes + // are the same fact about coverage, and for deb/rpm/apk the two + // vocabularies are the same strings, so they deduplicate rather than + // double-count. + // + // IT DID NOT ALWAYS DO THIS. Only the ecosystem route fed the list + // until A.21, which meant the list was EMPTY exactly when the input was + // well-formed enough to carry a purl — every repo-SCA finding, and what + // every collector is encouraged to supply. PackagesRefusedScheme counted + // them; nothing said what they were. EcosystemsRefused []string // Refusals, Defences, UpstreamOnlyAdvisories and @@ -1499,8 +1553,8 @@ func (m *Matcher) Match(ctx context.Context, inventory []PackageRecord) ([]Match anyRefusal = true if r, ok := asRefusal(err); ok { refusalsCollect = append(refusalsCollect, *r) - if r.Reason == RefusalUnsupportedEcosystem && r.Ecosystem != "" { - refusedEcos[r.Ecosystem] = true + if tok := r.refusedIdentityToken(); tok != "" { + refusedEcos[tok] = true } return } diff --git a/internal/match/comparator_test.go b/internal/match/comparator_test.go index 6f54274..06060c6 100644 --- a/internal/match/comparator_test.go +++ b/internal/match/comparator_test.go @@ -1780,10 +1780,14 @@ func TestCoverageCountsTheFalseNegativeRiskClass(t *testing.T) { if cov.PackagesWithNoAdvisoryData != 1 { t.Errorf("PackagesWithNoAdvisoryData = %d, want 1", cov.PackagesWithNoAdvisoryData) } - want := []string{"npm"} + // BOTH refused packages appear: "npm" arrived on the ecosystem column and + // "pypi" on the purl type. Before A.21 this was []string{"npm"} alone, + // and the entry that went missing was the one whose record was BETTER + // formed. See TestBothRefusalRoutesReachEcosystemsRefused. + want := []string{"npm", "pypi"} if !reflect.DeepEqual(cov.EcosystemsRefused, want) { - t.Errorf("EcosystemsRefused = %v, want %v (the purl-typed refusal reports its type in Detail, "+ - "not as an ecosystem)", cov.EcosystemsRefused, want) + t.Errorf("EcosystemsRefused = %v, want %v (one refusal came by the ecosystem column and one "+ + "by the purl type; both are the same fact about coverage)", cov.EcosystemsRefused, want) } if cov.Complete { t.Error("Complete is true despite five refusals") @@ -3773,3 +3777,124 @@ func TestDependencyGraphGuardFiresOnAPackageThatViolatesIt(t *testing.T) { "nonStdlibDeps cannot be trusted to see one below internal/match either. Got: %v", got) } } + +// TestBothRefusalRoutesReachEcosystemsRefused is A.21's finding, asserted. +// +// EcosystemsRefused answers ONE question — "what must Anvil implement next?" — +// and the answer cannot depend on which of the two identity routes a record +// happened to arrive by. A record carrying an unimplemented `ecosystem` column +// and a record carrying an unimplemented purl `type` are the same fact about +// coverage. +// +// The Lane A conformance harness reported this as a live defect: every +// repo-SCA finding carries a purl, so every one of them was refused as +// RefusalUnsupportedPurlType, and the operator-facing list was EMPTY exactly +// when the input was well-formed. PackagesRefusedScheme said "1 refused"; the +// thing to implement next was invisible. +// +// The three cases below refuse the SAME ecosystem by each route in turn, so a +// regression that drops either one is a diff in this test rather than a +// silently shorter list. +func TestBothRefusalRoutesReachEcosystemsRefused(t *testing.T) { + cases := []struct { + name string + record PackageRecord + reason RefusalReason + }{ + { + name: "ecosystem column only", + record: PackageRecord{ + Collector: CollectorRepoSCA, Ecosystem: "npm", + Name: "lodash", Version: "4.17.20", + }, + reason: RefusalUnsupportedEcosystem, + }, + { + name: "purl only", + record: PackageRecord{ + Collector: CollectorRepoSCA, Name: "lodash", Version: "4.17.20", + Purl: "pkg:npm/lodash@4.17.20", + }, + reason: RefusalUnsupportedPurlType, + }, + { + // Both columns present and agreeing, which is what a collector + // that fills everything it knows actually produces. The purl is + // consulted FIRST, so this is the purl route too — and it is the + // exact shape that used to leave the list empty. + name: "ecosystem column and purl together", + record: PackageRecord{ + Collector: CollectorRepoSCA, Ecosystem: "npm", + Name: "lodash", Version: "4.17.20", + Purl: "pkg:npm/lodash@4.17.20", + }, + reason: RefusalUnsupportedPurlType, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m, err := NewMatcher(NewStaticSource(nil)) + if err != nil { + t.Fatal(err) + } + results, cov, err := m.Match(context.Background(), []PackageRecord{tc.record}) + if err != nil { + t.Fatal(err) + } + if len(results) != 0 { + t.Fatalf("a refused package produced findings: %+v", results) + } + if cov.PackagesRefusedScheme != 1 { + t.Errorf("PackagesRefusedScheme = %d, want 1", cov.PackagesRefusedScheme) + } + if len(cov.Refusals) != 1 || cov.Refusals[0].Reason != tc.reason { + t.Fatalf("want exactly one %s refusal, got %+v", tc.reason, cov.Refusals) + } + want := []string{"npm"} + if !reflect.DeepEqual(cov.EcosystemsRefused, want) { + t.Errorf("EcosystemsRefused = %v, want %v. The count says a package was refused "+ + "for an unimplemented scheme; the list is what tells an operator WHICH one, "+ + "and it must not depend on the route the refusal arrived by.", + cov.EcosystemsRefused, want) + } + if cov.Complete { + t.Error("a run that refused a package reported Complete") + } + }) + } +} + +// TestAnImplementedEcosystemNeverEntersEcosystemsRefused is the negative +// control for the change above, and the reason PurlType is its own field. +// +// `{Ecosystem: "deb", Purl: "pkg:npm/..."}` is refused on the PURL TYPE. If the +// purl route had been wired by writing the type into Refusal.Ecosystem — or by +// reading Refusal.Ecosystem for both routes — this record would put "deb" on +// the implement-next list: a scheme this comparator DOES implement. That is +// worse than the empty list it replaced, because it is confidently wrong. +func TestAnImplementedEcosystemNeverEntersEcosystemsRefused(t *testing.T) { + m, err := NewMatcher(NewStaticSource(nil)) + if err != nil { + t.Fatal(err) + } + _, cov, err := m.Match(context.Background(), []PackageRecord{{ + Collector: CollectorRepoSCA, Ecosystem: EcosystemDeb, + Name: "lodash", Version: "4.17.20", Purl: "pkg:npm/lodash@4.17.20", + }}) + if err != nil { + t.Fatal(err) + } + for _, got := range cov.EcosystemsRefused { + if _, implemented := ecosystemAllowlist[got]; implemented { + t.Errorf("EcosystemsRefused names %q, which IS implemented (%v). The list is what an "+ + "operator implements next; naming an implemented scheme sends them to write code "+ + "that already exists.", got, cov.EcosystemsRefused) + } + } + want := []string{"npm"} + if !reflect.DeepEqual(cov.EcosystemsRefused, want) { + t.Errorf("EcosystemsRefused = %v, want %v (the refused token is the purl type, not the "+ + "ecosystem column that sat beside it)", cov.EcosystemsRefused, want) + } +} diff --git a/internal/match/purl.go b/internal/match/purl.go index 2ea2045..25598a3 100644 --- a/internal/match/purl.go +++ b/internal/match/purl.go @@ -188,10 +188,17 @@ func SchemeForEcosystem(ecosystem string) (Scheme, error) { // lowercased first because the purl specification defines the type segment as // case-insensitive with a lowercase canonical form. func SchemeForPurlType(purlType string) (Scheme, error) { - s, ok := purlTypeAllowlist[strings.ToLower(purlType)] + canonical := strings.ToLower(purlType) + s, ok := purlTypeAllowlist[canonical] if !ok { return "", &Refusal{ Reason: RefusalUnsupportedPurlType, + // PurlType carries the CANONICAL form because + // CoverageReport.EcosystemsRefused deduplicates on it: "npm" and + // "NPM" are one thing to implement, not two. Detail keeps the + // spelling that actually arrived, because that is what an + // operator greps their collector output for. + PurlType: canonical, Detail: "no version comparator is implemented for purl type " + strconv.Quote(purlType) + "; implemented schemes are " + joinSchemes(schemeOrder), } diff --git a/internal/record/lanea/REVIEW-A.20.md b/internal/record/lanea/REVIEW-A.20.md new file mode 100644 index 0000000..3a450a2 --- /dev/null +++ b/internal/record/lanea/REVIEW-A.20.md @@ -0,0 +1,326 @@ +# REVIEW-A.20 — independent review of A.19 record emission + +**Verdict: FAIL.** One blocker, two majors, three minors. + +**This was a SAME-FAMILY critic.** The packet routes A.20 to OpenCode `openai/gpt-5.5`; that route is +withdrawn per the OWNER DECISION block atop `plan/00-ROUTING.md`. This review was performed by a Claude +subagent — the same family that wrote A.19 — and therefore carries none of the independence the +cross-family guard was written to buy. Do **not** record this as "cross-family critic: PASS". To +compensate, every claim below is backed by a probe I wrote and ran against the committed code, not by +reading A.19's own tests. Nine probes were written; six passed, three failed. The probes were deleted +after the run and the tree is unmodified apart from this file. + +Reviewed at `74dee08` + untracked `internal/record/lanea/`, Go 1.26.5, Windows. +`gofmt -l` clean · `go vet` clean · `go build ./...` clean · `go test -count=1 ./...` **all green**. +`go test -race` was not run: cgo is unavailable on this host (`cgo.exe` exit 2). Noted, not waived. + +--- + +## Priority 1 — Can a host finding ever carry `remediable_by_agent = true`? + +**On the literal question asked by the packet: no. This part is earned.** + +I probed the bytes rather than the struct, per A.9's lesson. For a host `MatchResult` whose own +`RemediableByAgent` field asserts `true`, I marshalled every shape that leaves this package and walked +the decoded JSON for any key matching `remediable`: + +| shape | JSON pointer | value | +|---|---|---| +| `Emission` | `/remediableByAgent` | `false` | +| `Emission` | `/result/properties/anvil/remediableByAgent` | `false` | +| `*Emission` | both of the above | `false` | +| `Emission.Result` | `/properties/anvil/remediableByAgent` | `false` | +| `Results()[0]` | `/properties/anvil/remediableByAgent` | `false` | +| sealed `record.SARIFLog` | `/runs/0/results/0/properties/anvil/remediableByAgent` | `false` | +| `record.TaskCard` | `/remediableByAgent` | `false` | + +No outbound shape omits the key, so the "what does a consumer default to" question does not arise on +the happy path. I checked the zero value anyway: `contract.go:1527` carries no `omitempty`, and +unmarshalling `{"properties":{}}` into a `record.Result` yields `RemediableByAgent = false` — omission +defaults to the safe answer in Go and in every JSON binding that maps absent-bool to false. + +The branch I exhaustively checked is `remediableByAgent` at `emit.go:676-693`. It is a genuine +allowlist over five inputs: `collector == match.CollectorRepoSCA` **and** `kind == DetectorKindSCA` +**and** `class == EvidenceClassSCA` **and** `fixedVersion != ""` **and** `!parseDegraded`. It takes no +options struct, reads no config, holds no clock, and is called from exactly one site (`emit.go:848`). +`detectorFor` (`emit.go:702-711`) re-derives kind and class from the collector rather than copying them +off the `MatchResult`, and `Emit` refuses a disagreement at `emit.go:752-757` — so the "reaching a gate +is not obeying it" defect from `internal/scanctl` does not recur here. `record.Result.validate` +(`contract.go:2341-2348`) refuses the combination from the other side, and I confirmed it fires: see +finding 2 below. + +**But the guard is narrower than the file claims, and the gap is reachable — see finding 2.** + +## Priority 2 — Is a second fingerprint computed anywhere? + +**No. This is clean.** I reflected over the marshalled `Emission` for both collectors and collected +every `[0-9a-f]{32,}` run in every string value, plus every field whose path contains +`digest`/`fingerprint`/`findingId`: + +``` +/result/partialFingerprints/anvilFindingId/v1 = e5c9c775…5e08 ← record.Host() +/result/properties/anvil/findingId = e5c9c775…5e08 ← same value +/result/properties/anvil/advisory/snapshotDigest = "sha256:zzsnapshotqx" ← copied from the row, not derived +``` + +Every digest-shaped value in the emitted bytes is the one `internal/record` computed. There is no hash +import, no digest arithmetic, and `record.Host` / `record.Sca` are the only producers. The golden +comparison in `emit_test.go:648` reads `testdata/fingerprint_corpus/*.golden`, which +`scripts/compute_golden_fingerprints.py` produced from `FINGERPRINT-SPEC.md` — so that test's corpus +does not come from the implementation. The named cross-area edge holds. + +## Priority 3 — Is `anvil/trust` correct on every externally-originating string? + +**Substantially yes.** I injected a distinctive marker into every caller-supplied slot and enumerated +where it surfaced in the emitted JSON: + +``` +/result/ruleId · /result/message/text · /result/locations/0/…/uri +/result/properties/anvil/detector/advisoryItemId +/result/properties/anvil/advisory/{ids/0,sourceFeed,snapshotDigest,licenseSpdx,excerpt/text} +/licenseManualNote/text +``` + +All ten are covered by `Trust.Default = untrusted`, which `emit.go:1192-1198` enforces on the emitted +value rather than trusting the contract — and that enforcement is load-bearing, because +`ValidateResultTrust`'s external-pointer loop (`contract.go:2092`) iterates +`Result.ExternalStringPointers()`, which is **empty** for an SCA/host result. A.19 states this +explicitly at `emit.go:1180-1191` instead of implying the contract covers it. That is the right call +and it is unusual to see stated rather than assumed. + +The one `anvil_generated` label is on `/properties/anvil~1reasoning`, and it is true by construction: +`composeReasoning` (`emit.go:600-619`) refuses any part outside a closed constant vocabulary or a +base-10 integer, the guard is driven RED by `TestComposeReasoningRefusesAnExternalString` including +near-misses (`"+3600"`, `"3600.0"`, Arabic-Indic digits), and my injection probe confirmed the marker +never reaches the reasoning string. `anvil_generated` on the advisory row is refused +(`emit.go:797-802`). This is not the defect area B shipped. + +Two residual gaps, both minor, at findings 4 and 5. + +## Priority 4 — Do `as_of` and `staleness_seconds` reflect the feed watermark or emission time? + +**Neither, and that is the blocker. See finding 1.** + +## Priority 5 — Bare enum string literals + +**None.** I walked `emit.go`'s AST and compared every `BasicLit` string against the full value sets of +`DetectorKind`, `EvidenceClass`, `Trust`, `Verdict` and the two collector constants. Zero hits. The +literals `"apt"`, `"rpm"`, `"apk"` in `hostPackageManagers` (`emit.go:523-527`) are anvil-fp locator +segments, not contract enum members, and their keys are `match.Ecosystem*` constants. The `"apt"` +choice is pinned against the frozen corpus rather than against the map, which is the correct direction. + +--- + +# Findings + +## 1. BLOCKER — `staleness_seconds` measures the wrong interval, and a three-week-old cache emits as fresh + +`emit.go:903-904`, `emit.go:850`, `emit.go:470-473`. + +The frozen contract defines the field it is filling (`contract.go:1631-1634`): + +> `StalenessSeconds` is S6's `staleness_seconds`: **record-assembly time minus AsOf**. Carried +> explicitly so a consumer never has to know the assembly clock. + +A.19 fills it with something else. `AdvisoryRow.StalenessSeconds` is documented at `emit.go:375-378` as +"`advisory.staleness_seconds`, the age ingestion stamped at write time … carried through unmodified", +and that column is defined by `internal/ingest/delta/delta.go:1158-1180` as *the age of the DATA at +write time* — publisher lag, measured against `Last-Modified` at the instant of the last successful +sync. Meanwhile `advisory.as_of` is the **write** timestamp (`upsert.go:352-353`, bound at +`upsert.go:494` from `Apply`'s `asOf`, which `delta.go:863` passes as `now`), not "when this advisory +data was current". + +So two producers now write one named field under two different definitions. That is the same class of +defect as two `/v1` fingerprint algorithms under one name, in the freshness dimension instead of the +identity dimension, and S6's "defined once" rule is the rule it breaks. + +The consequence is exactly the one the priority list names. A 304 or a failed poll writes nothing +(`delta.go:857-861`), so during an outage both columns freeze. I built that state and read the emitted +values: + +``` +input : as_of = now-21d staleness_seconds = 3600 freshness_slo = 86400 +output: asOf = 2026-08-01T06:24:36Z + stalenessSeconds = 3600 + beyondFreshnessSlo= false + reasoning = "… Age is measured from the ingestion cache watermark, + not from the emission clock. Staleness in seconds: 3600" + TRUE age of the data = 1 818 000 s (21.0 days) + contract's definition (assembly − asOf) = 1 814 400 s +``` + +A finding resting on twenty-one-day-old feed data reports one hour of staleness, declares itself inside +a one-day SLO, and says so in the prose a human reads. `BeyondFreshnessSLO()` — the only +machine-readable SLO statement on the crossing artifact — is wrong by a factor of 504. Both errors +point the same way: fresher than reality. + +**A.19's Validation item 2 is therefore not met.** The packet requires "a test asserting a stale +advisory produces a Record that surfaces that staleness rather than silently reporting clean". The test +that claims it, `TestStaleAdvisorySurfacesItsStalenessRatherThanReportingClean` (`emit_test.go:487`), +sets `a.StalenessSeconds = 21*24*3600` **on the input row** and asserts that value appears on the +output. It proves copy-through. It cannot fail for the outage case, because the outage case is the one +where the input row's number stays small. + +**The suite already contains the contradiction and asserts both halves of it.** `fixtureWatermark` is +frozen at `2026-07-01` (`emit_test.go:75`). At review time that is **52.3 days** ago. The same test +asserts at line 504 that `time.Since(as_of) >= 24h` — "it must be the watermark" — and then at line +530-534 asserts, on an emission carrying that same 52-day-old watermark and an 86400-second SLO, that +`beyondFreshnessSlo == false`, commenting it as "data one hour into a one-day SLO". Under the frozen +contract's definition of the field, those two assertions cannot both be correct. Probe: + +``` +PROBE5b: fixture as_of is 52.3 days old; emitted stalenessSeconds = 3600; + SLO = 86400; beyondFreshnessSlo = false +``` + +**Note for whoever fixes this:** it cannot be fixed inside the package as currently shaped. A.19's +"this package reads no clock" property (`emit.go:104-110`, pinned by `TestThisPackageReadsNoClock`) is +a good property and I would not trade it away, but it means the assembly-relative age must arrive as an +input. The minimal shape is for `AdvisoryRow` to carry the scan's `as_of` — the value O.2 already holds +— or for the caller to supply `StalenessSeconds` already computed as `scanAsOf − (row.as_of − +row.staleness_seconds)`. Either way the emitted `asOf` should be *when the data was current* +(`row.as_of − row.staleness_seconds`), not the write instant, or the two fields will keep disagreeing +with each other by exactly the publisher lag. This is a contract-semantics question that reaches +`internal/ingest` and `internal/record`, so it is above this packet's write scope and belongs to the +orchestrator. + +## 2. MAJOR — the allowlist does not cover the ecosystem, so an OS package labelled `repo-sca` is emitted as agent-remediable + +`emit.go:676-693`, against the claim at `emit.go:55-62`. + +The package header claims the guard "fails closed for any collector, detector kind or evidence class +that arrives later". The ecosystem is a fourth dimension and it is not covered. I emitted a `repo-sca` +match whose ecosystem is a host ecosystem: + +``` +PROBE8 deb: EMITTED, remediableByAgent(top)=true nested=true detectorKind=sca +PROBE8 rpm: EMITTED, remediableByAgent(top)=true nested=true detectorKind=sca +PROBE8 apk: EMITTED, remediableByAgent(top)=true nested=true detectorKind=sca +``` + +`record.Validate` accepts all three, because from the record's point of view they are ordinary SCA +findings. The coding agent is handed "bump `openssl` in `Dockerfile`" as actionable work against an +`apt` package. That is the same authorization defect S7 exists to prevent, arriving through the +collector label rather than through the collector. + +It is not reachable *today*: `internal/collector/repo/trivy.go:771-782` refuses Trivy's `os-pkgs` +class outright, counting it rather than emitting it, with a comment naming this exact laundering risk. +So the guarantee holds end-to-end — but it holds **one lane up**, not here, and this package's own doc +claims it holds here. A guard whose correctness depends on an invariant enforced in another package, +while documenting itself as self-contained, is the shape that fails the next time someone adds an SBOM +or container collector under the `repo-sca` label. + +The fix is one condition using a map this file already owns: refuse, or clamp to non-remediable, a +`repo-sca` match whose `Ecosystem` has an entry in `hostPackageManagers`. Cost is a line; it converts +the ecosystem dimension from denylist-by-omission to allowlist. + +Strictly, the packet's question — "no code path allows a **host-collector-sourced** Record to carry +`remediable_by_agent = true`" — is answered *no*, correctly. I am reporting this because the finding it +produces is a host **package**, and the authorization consequence is identical. + +## 3. MAJOR — `Emission.RemediableByAgent()`'s doc claims a re-clamp that the crossing bytes do not perform + +`emit.go:452-462`. The doc says: + +> Reading the record and clamping again also means a Result that arrived from somewhere other than +> `Emit` — hand-built, deserialised, or **mutated after emission** — cannot present a host finding as +> actionable. + +The clamp applies to the top-level mirror only. The nested canonical record inside the same JSON object +is not re-clamped, and neither is `Results()` (`emit.go:495-501`), which is the documented projection +onto the `record.Result`s a `Run` carries — i.e. the artifact that actually reaches the store and the +SARIF log. Probe, mutating a host emission after `Emit` returned: + +``` +PROBE2 Emission top-level remediableByAgent = false +PROBE2 Emission nested result anvil/remediableByAgent = true ← disagrees +PROBE2 Results()[0] anvil/remediableByAgent = true ← disagrees +``` + +The consequence is contained: `record.Validate` refuses the assembled log — + +``` +runs[0]: results[0]: anvil/remediableByAgent must be false for a host finding +(00-SPINE.md S7: the host agent is read-only) +``` + +— and `CardsFromLog` clamps again at `taskcard.go:550`. So this is not an authorization hole. It is a +false claim in the doc comment of the authorization guard, and the standard this project has already +paid for is that such a claim is deleted rather than qualified. Either narrow the sentence to the +top-level mirror, or make `Results()` clamp so the sentence becomes true. + +Related, smaller: `MarshalJSON` at `emit.go:483-490` says "the serialised form cannot disagree with the +method". It cannot disagree at the top level; the probe above shows the object as a whole disagreeing +with itself. + +## 4. MINOR — the advisory excerpt's bound and sanitisation are asserted, never checked + +`emit.go:392-396` claims the excerpt is "pre-trimmed (<= `record.MaxAdvisoryExcerptTokens`), already +sanitised by A.3 at ingest". Nothing verifies either, and `excerpt()` (`emit.go:1097-1102`) copies the +string through. Probe: + +``` +PROBE9: EMITTED an excerpt of 24034 bytes (record.MaxAdvisoryExcerptBytes = 2400); + contains U+200B = true, U+202E = true + record.Validate ACCEPTED it +``` + +Exposure is bounded downstream — `taskcard.go:893` caps the card's excerpt at +`MaxAdvisoryExcerptBytes`, so the agent-facing artifact is trimmed — but the 24 KB string with the +zero-width space and the RTL override persists in the record and the store, and **no component anywhere +in `internal/record` re-checks invisible characters** (`internal/ingest/invisible` exists and is not +reachable from here). This package refuses on fifteen lesser invariants, including two that only a +malformed caller could trip; trusting an unverified claim about the highest-risk external field on the +record is inconsistent with its own posture. Either check the byte length and reject, or delete the +"already sanitised" clause so the reader knows it is an assumption. + +## 5. MINOR — `TrustVerified` passes through with no validation step named + +`emit.go:1097-1112` copies `AdvisoryRow.Trust` onto the excerpt and the licence note. The contract says +of that value (`contract.go:636-639`): "the bytes originated outside Anvil AND passed an explicit +validation step **that is named in the record** … Never a default." A row asserting `verified` produces +a record labelling advisory prose `verified` with no validation step named anywhere: + +``` +PROBE6 with row trust=verified: excerpt trust = verified +``` + +Not live today — `internal/ingest/delta/upsert.go:494` always binds `cache.AdvisoryTrustDefault` +(`untrusted`), and the column's CHECK permits only `untrusted|verified`. So this is forward-looking: +the moment a signature-checked snapshot path lands, `verified` will reach the record without the +naming the contract requires. Either refuse `verified` here until there is a slot to name the step in, +or carry the step's identity alongside it. + +## 6. MINOR — two tests cannot fail for the reason they claim + +- `TestStaleAdvisorySurfacesItsStalenessRatherThanReportingClean` (`emit_test.go:487`) asserts an + input value reappears on the output. Its corpus is its own assertion. See finding 1. +- The "fresh" control at `emit_test.go:528-534` is time-dependent through the hard-coded + `fixtureWatermark`. It passes today only because the field it exercises ignores the interval that + makes it stale; under a corrected `staleness_seconds` it becomes a test that fails on a wall-clock + date rather than on a code change. + +--- + +# What I checked that came back clean + +Recorded so the next reviewer does not re-run it: the fifteen refusal reasons are each driven RED by +`TestEveryRefusalReasonIsReachable`; `EmitAll` refuses rather than skipping on an unresolvable advisory +row (`emit.go:952-963`) — a silent drop there would be indistinguishable from a clean scan; emission is +deterministic across repeated calls; `CvssV4Base` is populated only for a `CVSS:4.0/` vector, leaving +the slot null for a v3.1 row rather than writing a number that is wrong in a way no consumer can +detect; `Result.Level` is deliberately unset with the reason stated rather than a severity vocabulary +invented; the import allowlist is pinned by AST, not by grep; the `deb → apt` locator choice is pinned +against the independently-produced golden rather than against the map that makes it. + +The three DEVIATIONS declared in the package header (`emit.go:123-151`) are all real, all reported +rather than applied quietly, and all in the safe direction. Deviation 3 in particular — `parse_degraded` +clamping `remediable_by_agent` to `false` — can only move an authorization answer from yes to no, and +the finding is still emitted as report-only rather than dropped. I would keep all three. + +# Gate + +Finding 1 blocks A.21. The conformance harness would otherwise pin the wrong `staleness_seconds` +semantics into a two-run byte-identity assertion, at which point correcting it becomes a change to a +frozen conformance expectation rather than a change to one field. Findings 2 and 3 should land in the +same pass; 4, 5 and 6 can follow. diff --git a/internal/record/lanea/emit.go b/internal/record/lanea/emit.go new file mode 100644 index 0000000..1a4c6d2 --- /dev/null +++ b/internal/record/lanea/emit.go @@ -0,0 +1,1499 @@ +// Package lanea populates the Lane-A-owned fields of Anvil's canonical audit +// record: step A.19 of plan/20-lane-a-ingestion-sca.md. +// +// =========================================================================== +// WHAT THIS PACKAGE IS, AND THE ONE THING IT IS NOT +// =========================================================================== +// +// It takes one A.17 MatchResult — "this installed version is inside a range +// this advisory calls vulnerable" — plus the `advisory` row that decided it, +// and produces a record.Result: the SAME struct every other area produces, +// carrying the frozen `anvil/*` property bag. +// +// IT DEFINES NO SECOND FINGERPRINT, NO SECOND FINDING STRUCT AND NO SECOND +// ENUM. plan/IMPLEMENTATION-PLAN.md §1 names this the cross-area edge in +// exactly those words: "Lane A must populate the canonical record and MUST NOT +// INVENT A SECOND FINGERPRINT." So: +// +// - the digest comes from record.Sca / record.Host (anvil-fp/v1, +// internal/record/FINGERPRINT-SPEC.md is authoritative). There is no +// hash function in this file, and emit_test.go asserts the emitted digest +// against testdata/fingerprint_corpus's GOLDEN values — which were +// produced by a separate implementation of the spec, not by the Go code +// this package calls; +// - every enum value written here is a Go constant from internal/record +// (record.HalfSast, record.DetectorKindHost, record.TrustUntrusted, ...). +// There is not one re-spelled string literal; +// - the output is record.Result. Emission wraps it to carry the two +// Lane-A-owned facts the frozen contract has no slot for, and carries no +// copy of anything the Result already holds. See Emission. +// +// A.17 set the precedent this file follows: match.MatchResult carries no +// fingerprint, and match.Purl.Base delegates to record.PurlBase. +// +// =========================================================================== +// THE SEVEN LANE-A-OWNED FIELDS (plan/20 Dependency Summary) +// =========================================================================== +// +// remediable_by_agent -> Result.Properties.RemediableByAgent +// as_of -> Result.Properties.Advisory.AsOf +// staleness_seconds -> Result.Properties.Advisory.StalenessSeconds +// ** COMPUTED here, see section 3 ** +// parse_degraded -> Result.Properties.Advisory.ParseDegraded +// anvil/trust -> Result.Properties.Trust (+ the excerpt's own) +// license_spdx -> Result.Properties.Advisory.LicenseSpdx +// license_manual_note -> Emission.LicenseManualNote ** see DEVIATION 1 ** +// +// =========================================================================== +// 1. remediable_by_agent IS FALSE FOR EVERY HOST FINDING. ALWAYS. +// =========================================================================== +// +// The coding agent's write surface is the git repository (plan/00-SPINE.md +// S7), so it CANNOT fix a host package: there is no file to edit, and the host +// agent is read-only "not behind a flag". Handing it a host finding as +// actionable is an authorization defect, not a cosmetic one. +// +// THE GUARD IS AN ALLOWLIST, NOT A HOST CHECK. remediableByAgent below returns +// false unless EVERY condition for the one remediable shape holds. A denylist +// ("false when host") fails open for any collector, detector kind, evidence +// class or ECOSYSTEM that arrives later; this fails closed. It takes no +// options, reads no configuration and consults no clock, so there is no +// location an override could live in — the same construction +// internal/collector/host used when it made RemediableByAgent a method over an +// untyped constant rather than an assignable field. +// +// S7 IS ABOUT THE THING, NOT ABOUT WHO FOUND IT. An OS package is not +// agent-fixable however it was noticed, because the agent's write surface is +// the git repository either way. So the ecosystem is an arm of the allowlist in +// its own right: a `repo-sca` match whose ecosystem is deb, rpm or apk is NOT +// remediable. Without that arm the guarantee held only because +// internal/collector/repo happens to refuse Trivy's os-pkgs class one lane up — +// a guard whose correctness lives in another package while this file's doc +// claims it lives here. +// +// THE FLAG TRAVELS IN THE BYTES. A.12's review found this field present on +// internal/collector/host's own Inventory and MISSING from FindingSeed, the +// artifact that actually crossed to A.17 — the wrong way round. So every shape +// this package emits carries it in its SERIALISED form: record.Result always +// marshals `anvil/remediableByAgent` (no omitempty), and Emission.MarshalJSON +// adds a CLAMPED top-level mirror. emit_test.go asserts the bytes, not the +// struct field, for both. +// +// =========================================================================== +// 2. anvil/trust ON EVERY STRING ORIGINATING OUTSIDE ANVIL +// =========================================================================== +// +// Advisory prose, package names, versions, purls and manifest paths are +// attacker-influenceable text heading for a repo-credentialed agent. Anvil +// assembling a struct around external bytes does not make the bytes Anvil's: +// record.Trust's own doc says "the question TrustLevel answers is 'who wrote +// these bytes', never 'who assigned this field'", and area B was caught +// stamping anvil_generated on verbatim repo source. +// +// So TrustAssertion.Default is record.TrustUntrusted UNCONDITIONALLY on every +// result this package emits, exactly as record.CardTrust does, and Fields +// names only the strings Anvil itself wrote. Today that is exactly one: +// `anvil/reasoning`. +// +// AND THAT ONE IS PROVED, NOT ASSERTED. A sentence that interpolates a package +// name is not Anvil-generated however it is labelled, so the reasoning string +// is not formatted — it is JOINED FROM A CLOSED VOCABULARY of constant +// fragments plus base-10 integers Anvil computed itself (composeReasoning). A +// part outside that vocabulary is refused and no record is emitted. That makes +// "no external byte reaches an anvil_generated string" a property of the +// construction rather than a claim about the format string. +// +// The advisory EXCERPT keeps its own trust inline (record.TrustedString) and +// takes it from the `advisory` row's `anvil_trust` column, whose CHECK admits +// only untrusted|verified. A row carrying anvil_generated is refused here too. +// +// =========================================================================== +// 3. as_of IS THE CACHE WATERMARK; staleness_seconds IS THE INTERVAL SINCE IT +// =========================================================================== +// +// The two fields are DIFFERENT QUANTITIES and the frozen contract defines both: +// AdvisoryContext.AsOf is "when this advisory data was current", and +// AdvisoryContext.StalenessSeconds is "record-assembly time minus AsOf, carried +// explicitly so a consumer never has to know the assembly clock". +// +// SO staleness_seconds IS COMPUTED HERE, NOT COPIED. `advisory.staleness_seconds` +// in the cache is a THIRD quantity — the publisher lag ingestion measured at +// write time — and copying it into this field publishes two definitions under +// one name. The failure it produces is the exact one S6 exists to prevent: a +// feed outage freezes both cache columns, so a finding resting on three-week-old +// data reports the hour of publisher lag that was true at the last successful +// sync, declares itself inside a one-day SLO, and says so in the prose a human +// reads. An absence of information becomes a false assurance. +// +// AND THIS PACKAGE STILL READS NO CLOCK. There is no time.Now() in this file +// and no clock field on Emitter; TestThisPackageReadsNoClock enforces it. The +// assembly instant ARRIVES instead, on Emitter.AssembledAt, from the caller +// that owns the scan — and it is REQUIRED, refused when zero rather than +// defaulted, because a zero default computes an age against the Unix epoch or +// (worse) silently reproduces the copy-through bug. research/06 Risk #5 and +// plan/00-SPINE.md S6 both say the same thing: serve stale data with an `as_of` +// and a `staleness_seconds`, and say so. +// +// KNOWN GAP, STATED RATHER THAN PAPERED OVER: `advisory.as_of` is the cache +// WRITE instant, so the emitted age omits whatever publisher lag preceded that +// write. Closing it means either redefining `advisory.as_of` or carrying the lag +// into the record, and both reach internal/ingest and internal/record. Reported +// to the orchestrator; not decided here. The emitted number is therefore a LOWER +// BOUND on the true age of the data — it is exactly the interval the contract +// defines, and the contract's own `as_of` is the quantity that is imprecise. +// +// Staleness is SURFACED, NOT ACTED ON. A feed outage is the normal case that +// on_failure=serve_stale exists for (internal/ingest/config: there is +// deliberately no fail_scan value), so a finding decided on stale data is +// still a true positive and still actionable — it just says how old the data +// was. Refusing to act on every finding during a two-day outage is the +// over-refusal this project's own coverage reasoning says gets a control +// dismissed. +// +// parse_degraded is different and IS acted on: see remediableByAgent. +// +// =========================================================================== +// FOUR DEVIATIONS FROM THE PACKET, REPORTED RATHER THAN APPLIED QUIETLY +// =========================================================================== +// +// 1. license_manual_note HAS NO SLOT IN THE FROZEN RECORD. record's +// AdvisoryContext carries LicenseSpdx and nothing else; run-level +// AdvisorySnapshot carries no licence field either. plan/00-SPINE.md S8's +// compliance mechanics require the manual-override field carrying the +// quoted operative sentence — precisely for the sources whose SPDX id is +// NONE, NOASSERTION or a LicenseRef-, which is where the SPDX id alone +// establishes nothing. Every slot it could be smuggled into is worse than +// no slot: LicenseSpdx is an identifier field and prose corrupts it, and +// Reasoning is anvil_generated while the note is a QUOTATION from a +// publisher's LICENSE file. So it travels on Emission, typed as a +// record.TrustedString, and the gap is reported to the orchestrator. +// Dropping it silently would lose a compliance obligation. +// +// 2. Result.Level IS LEFT UNSET. Severity mapping (advisory severity or CVSS +// -> SARIF level) is a vocabulary nobody has been assigned and is not one +// of the seven fields this step owns; inventing one here would be the +// eleventh defect of the shape plan/IMPLEMENTATION-PLAN.md §6 rules on. +// Risk (CVSS/EPSS/KEV) IS populated, because the record names Lane A +// ingestion as its producer in so many words. +// +// 3. parse_degraded CLAMPS remediable_by_agent TO FALSE. A.19's sketched +// expression reads only the collector. The clamp can only ever move the +// answer from true to false, and its reason is at remediableByAgent +// condition 4: the fixed version an agent would bump to was parsed out of +// the same record A.16 says the parser did not fully understand. The +// finding is still emitted, as report-only. +// +// 4. `verified` HAS NO SLOT TO NAME ITS VALIDATION STEP IN. record.Trust +// defines TrustVerified as "the bytes originated outside Anvil AND passed +// an explicit validation step THAT IS NAMED IN THE RECORD ... Never a +// default", and the frozen AdvisoryContext has nowhere to put the step's +// name. Passing `verified` through unaccompanied would publish the label +// without the thing that makes it mean anything. So a row asserting +// `verified` without naming its step is REFUSED here +// (RefusalTrustValidationStep), and the named step travels on +// Emission.TrustValidationStep — the same treatment, and the same reported +// record-contract gap, as license_manual_note in deviation 1. +package lanea + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/match" + "github.com/Susquehanna-Syntax/Anvil/internal/record" +) + +// --------------------------------------------------------------------------- +// Refusals +// --------------------------------------------------------------------------- + +// RefusalReason names why an emission was refused. It is a closed set and a +// named type so a second reason cannot arrive as a bare string, mirroring +// match.RefusalReason. +type RefusalReason string + +const ( + // RefusalNoTargetID: the emitter carries no target id. It is field 1 of + // every anvil-fp/v1 tier and of `UNIQUE (target_id, fingerprint)`. + RefusalNoTargetID RefusalReason = "no_target_id" + + // RefusalUnrecognisedCollector: the match names a collector outside + // {host, repo-sca}. remediable_by_agent is derived from it, so a + // defaulted collector would default an authorization decision. + RefusalUnrecognisedCollector RefusalReason = "unrecognised_collector" + + // RefusalInconsistentMatch: the match's own frozen enums disagree with + // its collector — e.g. Collector=host with DetectorKind=sca. Reaching a + // gate is not obeying it: this package re-derives both from the + // collector and refuses a disagreement rather than trusting either side. + RefusalInconsistentMatch RefusalReason = "inconsistent_match" + + // RefusalAdvisoryMismatch: the supplied advisory row is not the row the + // match was decided on, by (source, source_id) or by CVE alias. + RefusalAdvisoryMismatch RefusalReason = "advisory_mismatch" + + // RefusalNoAdvisoryID: neither a CVE id nor a source id, so the finding + // cannot be identified against the advisory table. + RefusalNoAdvisoryID RefusalReason = "no_advisory_id" + + // RefusalNoPurl: no purl on either the package or the advisory range. A + // purl is field 4 of the SCA/host formula and this package will not + // synthesise one — the namespace (debian vs ubuntu, redhat vs fedora) + // comes from os-release, which a MatchResult does not carry, and a + // guessed namespace forks identity against every producer that had one. + RefusalNoPurl RefusalReason = "no_purl" + + // RefusalNoManifestPath: a repo-SCA match with no manifest relpath. It + // is the SCA tier's locator; two manifests pulling the same vulnerable + // package are two findings with two owners and two fixes. + RefusalNoManifestPath RefusalReason = "no_manifest_path" + + // RefusalUnsupportedHostEcosystem: a host match whose ecosystem has no + // entry in hostPackageManagers. The locator's manager segment is part of + // the fingerprint, so guessing it forks identity. + RefusalUnsupportedHostEcosystem RefusalReason = "unsupported_host_ecosystem" + + // RefusalNoWatermark: the advisory row carries no as_of. Emitting + // without one would leave the record's freshness unstated, which is the + // state spine S6's as_of/staleness_seconds fields exist to prevent. + RefusalNoWatermark RefusalReason = "no_watermark" + + // RefusalNoAssemblyTime: the emitter carries no record-assembly instant. + // staleness_seconds is DEFINED as assembly time minus as_of, and this + // package holds no clock to supply the first term, so the caller must. + // It is refused rather than defaulted: a zero default would measure the + // age against the Unix epoch, and any other default would be this + // package inventing the emission instant it is forbidden to read. + RefusalNoAssemblyTime RefusalReason = "no_assembly_time" + + // RefusalNegativeStaleness: a negative age — either on the row's own + // column (the cache's advisory_staleness_nonneg CHECK refuses it too) or + // on the computed interval, which means the assembly instant precedes + // the watermark the data claims to be current at. + RefusalNegativeStaleness RefusalReason = "negative_staleness" + + // RefusalIllegalAdvisoryTrust: the advisory row claims a trust level + // that is not legal for a string originating outside Anvil — i.e. + // anvil_generated. That is the mislabelling record.Trust documents area + // B committing. + RefusalIllegalAdvisoryTrust RefusalReason = "illegal_advisory_trust" + + // RefusalTrustValidationStep: the row's trust level and its named + // validation step disagree. record.Trust says `verified` means the bytes + // "passed an explicit validation step that is named in the record ... + // Never a default", so `verified` with no step named is refused; and a + // step named beside `untrusted` is refused too, because that is a + // validation claim attached to bytes nothing validated. + RefusalTrustValidationStep RefusalReason = "trust_validation_step" + + // RefusalExcerptTooLong: the advisory excerpt exceeds + // record.MaxAdvisoryExcerptBytes. Ingestion is supposed to have trimmed + // it; this checks rather than assumes, because the excerpt is the + // highest-risk external string on the record and the read path's cap + // (taskcard.go) trims only the AGENT-FACING copy — the untrimmed string + // would still persist in the record and the store. + RefusalExcerptTooLong RefusalReason = "excerpt_too_long" + + // RefusalNoLicenceDeclared: the advisory row states neither an SPDX id + // nor a manual note. plan/00-SPINE.md S8 and the cache's + // advisory_license_declared CHECK: a row that records neither is data + // Anvil cannot prove it may use, and it must not be re-published into a + // record. + RefusalNoLicenceDeclared RefusalReason = "no_licence_declared" + + // RefusalReasoningNotAnvilGenerated: a reasoning part fell outside the + // closed vocabulary, so the string about to be labelled anvil_generated + // might carry external bytes. See composeReasoning. + RefusalReasoningNotAnvilGenerated RefusalReason = "reasoning_not_anvil_generated" + + // RefusalFingerprint: internal/record refused to compute the digest. + // Reported, never papered over: a finding that cannot be identified + // cannot be tracked across scans. + RefusalFingerprint RefusalReason = "fingerprint_refused" + + // RefusalContractViolation: the assembled Result failed + // internal/record's own validation. It means this file built something + // the contract forbids, and the finding is refused rather than emitted. + RefusalContractViolation RefusalReason = "contract_violation" +) + +// refusalReasonOrder is every reason, in a fixed order. +var refusalReasonOrder = []RefusalReason{ + RefusalNoTargetID, + RefusalUnrecognisedCollector, + RefusalInconsistentMatch, + RefusalAdvisoryMismatch, + RefusalNoAdvisoryID, + RefusalNoPurl, + RefusalNoManifestPath, + RefusalUnsupportedHostEcosystem, + RefusalNoWatermark, + RefusalNoAssemblyTime, + RefusalNegativeStaleness, + RefusalIllegalAdvisoryTrust, + RefusalTrustValidationStep, + RefusalExcerptTooLong, + RefusalNoLicenceDeclared, + RefusalReasoningNotAnvilGenerated, + RefusalFingerprint, + RefusalContractViolation, +} + +// RefusalReasons returns every legal reason, in a fixed order. +func RefusalReasons() []RefusalReason { + out := make([]RefusalReason, len(refusalReasonOrder)) + copy(out, refusalReasonOrder) + return out +} + +// Valid reports whether r is one of the closed set. +func (r RefusalReason) Valid() bool { + for _, v := range refusalReasonOrder { + if v == r { + return true + } + } + return false +} + +// Refusal is a refusal to emit one record, carrying enough identity to be +// counted and reported rather than merely logged. +type Refusal struct { + Reason RefusalReason + Collector string + Package string + Source string + SourceID string + Detail string + // Err is internal/record's own error when the refusal came from the + // fingerprint or the contract, and nil otherwise. + Err error +} + +func (r *Refusal) Error() string { + var b strings.Builder + b.WriteString("lanea: refusing to emit: ") + b.WriteString(string(r.Reason)) + if r.Source != "" || r.SourceID != "" { + b.WriteString(" [" + r.Source + "/" + r.SourceID + "]") + } + if r.Package != "" { + b.WriteString(" package=" + strconv.Quote(r.Package)) + } + if r.Collector != "" { + b.WriteString(" collector=" + strconv.Quote(r.Collector)) + } + if r.Detail != "" { + b.WriteString(": " + r.Detail) + } + if r.Err != nil { + b.WriteString(": " + r.Err.Error()) + } + return b.String() +} + +// Unwrap exposes internal/record's own error so a caller can match on it. +func (r *Refusal) Unwrap() error { return r.Err } + +// --------------------------------------------------------------------------- +// Inputs +// --------------------------------------------------------------------------- + +// AdvisoryRow is the `advisory` row A.17's deciding range came from, as this +// step needs it. +// +// It is a READ SHAPE, not a second finding: it holds nothing the comparator +// already decided and nothing derived. Its fields mirror +// internal/ingest/cache/schema.go's `advisory` columns one for one, plus the +// per-feed freshness SLO, which lives in internal/ingest/config's FeedConfig +// because "a duration written as a Go constant is precisely the hard-coded +// cadence this package forbids". +// +// This package does not import internal/ingest/cache or internal/ingest/config: +// cache links a SQL driver and neither belongs in the dependency graph of a +// pure field-population step, for exactly the reason internal/match states. +// The caller reads the row and fills this in. +type AdvisoryRow struct { + // Source and SourceID are `advisory`'s primary key. They must be the + // pair the MatchResult was decided on. + Source string + SourceID string + // CVEID is the nullable alias, `advisory.cve_id`. + CVEID string + + // FeedID is the `feed_state` row that supplied this advisory; it becomes + // AdvisoryContext.SourceFeed. + FeedID string + // SnapshotDigest identifies the advisory corpus this row was read from. + SnapshotDigest string + + // LicenseSPDX is `advisory.license_spdx`, and LicenseManualNote is + // `advisory.license_manual_note` — spine S8's manual-override field + // carrying the quoted operative sentence. At least one must be + // non-blank, mirroring the cache's advisory_license_declared CHECK. + LicenseSPDX string + LicenseManualNote string + + // Trust is `advisory.anvil_trust`. Legal values are untrusted and + // verified ONLY: every byte in the advisory table originated outside + // Anvil, and the column's CHECK says so. + Trust record.Trust + // TrustValidationStep NAMES the explicit validation step the row's bytes + // passed, and is REQUIRED when Trust is verified — record.Trust defines + // that value as "passed an explicit validation step that is named in the + // record ... Never a default", so the label without the name is not the + // value the contract describes. It must be EMPTY when Trust is + // untrusted. There is no cache column for it today; see deviation 4. + TrustValidationStep string + + // AsOf is `advisory.as_of`: THE CACHE WATERMARK. It is when this data + // was current, never when the record was assembled. It is the SECOND + // term of the emitted staleness_seconds; the first is + // Emitter.AssembledAt. + AsOf time.Time + // StalenessSeconds is `advisory.staleness_seconds`: the PUBLISHER LAG + // ingestion measured at write time (internal/ingest/delta measures it + // against Last-Modified at the last successful sync). + // + // IT IS NOT THE RECORD'S staleness_seconds AND IS NOT EMITTED. The + // record's field is assembly time minus AsOf — a different quantity, + // computed in Emit — and copying this column into it publishes two + // definitions under one name, which during a feed outage reports + // weeks-old data as an hour old. This field is carried because + // AdvisoryRow mirrors the cache's columns one for one and because a + // negative value is a row this package refuses to re-publish, not + // because the record consumes it. + StalenessSeconds int + // FreshnessSLOSeconds is the feed's freshness_slo_seconds. Zero means + // the caller did not state one, and BeyondFreshnessSLO then answers + // false rather than guessing a default. + FreshnessSLOSeconds int + + // ParseDegraded is `advisory.parse_degraded`: A.16 sets it when a record + // arrived in a dataVersion the parser did not fully understand. See + // remediableByAgent for what this package does about it. + ParseDegraded bool + // DataVersion is `advisory.data_version`, carried for the operator who + // has to fix the parser. + DataVersion string + + // ExcerptText is ingestion's advisory excerpt. Empty when there is none. + // + // WHAT IS ESTABLISHED HERE AND WHAT IS NOT. The BOUND is checked: Emit + // refuses an excerpt longer than record.MaxAdvisoryExcerptBytes rather + // than asserting that ingestion trimmed it, because the read path's cap + // (record/taskcard.go) trims only the card the agent sees, so an + // oversized string would still reach the record and the store. + // + // SANITISATION IS NOT CHECKED HERE, and this file does not claim it is. + // A.3 sanitises at ingest — plan/00-SPINE.md S7, "sanitize at ingest, not + // at prompt time" — and the vocabulary of invisible and bidi characters + // that check is defined against lives in internal/ingest/invisible, which + // this package does not import and must not re-spell: a second definition + // of "invisible" that drifts from the first is the same defect as a + // second fingerprint. What would settle it is that vocabulary being + // exported somewhere internal/record may depend on. Until then this is an + // UNVERIFIED PRECONDITION, stated as one. + ExcerptText string + + // The ranking inputs research/24 names non-negotiable. + // CVSSVector is `advisory.cvss_vector` and CVSSScore `advisory.cvss_score`. + CVSSVector string + CVSSScore *float64 + // EPSSScore/EPSSPercentile/EPSSAsOf and the KEV flags. + EPSSScore *float64 + EPSSPercentile *float64 + EPSSAsOf *time.Time + KEVMember bool + KEVRansomwareUse bool +} + +// Emitter carries the scan context a MatchResult does not: the target the scan +// ran against, and the instant this record is being assembled at. +// +// There is deliberately no CLOCK field — no func() time.Time, no interface with +// a Now on it. See the package header, section 3: the assembly instant is a +// VALUE the caller states once for the whole record, not a source this package +// can read repeatedly. That is also why it is here and not on AdvisoryRow: one +// record has one assembly time, and a per-row field would let two findings in +// one record disagree about when that record was built. +type Emitter struct { + // TargetID is `target.target_id` rendered as text — field 1 of every + // anvil-fp/v1 tier. Required. + TargetID string + + // AssembledAt is the record-assembly instant, owned by the caller that + // owns the scan (O.2 already holds it). REQUIRED: the zero value is + // refused rather than treated as "unset", because + // AdvisoryContext.StalenessSeconds is DEFINED as this minus + // AdvisoryRow.AsOf, and a zero default would either measure the age + // against the Unix epoch or quietly reinstate the copy-through bug this + // field exists to remove. + AssembledAt time.Time +} + +// --------------------------------------------------------------------------- +// Output +// --------------------------------------------------------------------------- + +// Emission is one emitted finding: the canonical record.Result, plus the two +// Lane-A-owned facts the frozen record contract has no slot for. +// +// IT IS NOT A PARALLEL FINDING STRUCT. It carries no fingerprint, no package +// identity, no advisory identity and no copy of any field Result already +// holds. Everything about the finding is read from Result; these two exist +// because the contract has nowhere to put them and dropping them would lose, +// respectively, a compliance obligation and the only machine-readable +// statement that a feed missed its SLO. Both are reported to the orchestrator +// as record-contract gaps rather than fixed here — this step must not write to +// the record schema. +type Emission struct { + // Result is the canonical record. Everything about the finding is here. + Result record.Result `json:"result"` + + // LicenseManualNote is plan/00-SPINE.md S8's manual-override field: the + // quoted operative sentence from the publisher's own licence text, + // required whenever the SPDX id is NONE, NOASSERTION or a LicenseRef-. + // It is a QUOTATION FROM OUTSIDE ANVIL and carries its own trust inline. + // Nil when the row states none. + LicenseManualNote *record.TrustedString `json:"licenseManualNote,omitempty"` + + // FreshnessSLOSeconds is the feed's SLO, carried so a consumer can + // evaluate BeyondFreshnessSLO without holding the feed table. Zero when + // the caller stated none. + FreshnessSLOSeconds int `json:"freshnessSloSeconds"` + + // TrustValidationStep names the explicit validation step the advisory + // row's bytes passed, and is non-empty exactly when the excerpt and the + // licence note are classified `verified`. See deviation 4: the frozen + // record has no slot for it, and `verified` without it is not the value + // record.Trust defines. + TrustValidationStep string `json:"trustValidationStep,omitempty"` +} + +// RemediableByAgent is the CLAMPED answer, never a copy. +// +// It is a method and not a field for the reason internal/collector/host gives: +// a field is an assignable location, and Lane A exit criterion 21 requires +// that no code path, flag or config key be capable of setting this true for a +// host finding. +// +// AND THE CLAMP TRAVELS IN THE BYTES, NOT ONLY IN THIS METHOD. Every shape that +// leaves this package is built from clamped, so a Result that arrived from +// somewhere other than Emit — hand-built, deserialised, or mutated after +// emission — cannot present a host finding as actionable through ANY of them: +// the top-level `remediableByAgent` mirror, the nested canonical record inside +// the same object, and Results(), which is the projection that actually reaches +// the SARIF log and the store. An earlier revision clamped the mirror only, so +// the object disagreed with itself; the claim is now the whole object's, and +// TestAResultThatDidNotComeFromEmitIsClampedInTheCrossingBytes asserts it on the +// SERIALISED BYTES rather than on this method. +func (e Emission) RemediableByAgent() bool { + return e.Result.Properties.RemediableByAgent && !record.IsHostFinding(&e.Result) +} + +// clamped returns a copy of e whose Result carries the clamped answer. +// +// The copy is what makes the guarantee hold for a Result this package did not +// build: e is a value, and Result is a value field on it, so writing the +// clamped answer here cannot reach the caller's own struct. The maps and +// pointers inside Result are shared with that struct and are NOT written. +func (e Emission) clamped() Emission { + e.Result.Properties.RemediableByAgent = e.RemediableByAgent() + return e +} + +// BeyondFreshnessSLO reports whether the advisory data this finding rests on +// is older than its feed's freshness SLO. +// +// It answers false when no SLO was stated: an unstated SLO is not a met one, +// but it is also not a breach this package can assert, and asserting one from +// a default would be a claim about a feed nobody configured. +func (e Emission) BeyondFreshnessSLO() bool { + a := e.Result.Properties.Advisory + return a != nil && e.FreshnessSLOSeconds > 0 && a.StalenessSeconds > e.FreshnessSLOSeconds +} + +// MarshalJSON emits the Emission with `remediableByAgent` and +// `beyondFreshnessSlo` included at the top level. +// +// THIS IS THE ARTIFACT THAT CROSSES THE BOUNDARY, so it is the one that most +// needs both guarantees to travel in the BYTES rather than in a method a +// consumer has to know to call. A.12 found exactly this omission on the +// analogous artifact one lane over. BOTH LEVELS carry the clamped value — the +// mirror and the nested record — so no part of the serialised form can disagree +// with the method or with any other part. +func (e Emission) MarshalJSON() ([]byte, error) { + c := e.clamped() + type alias Emission + return json.Marshal(struct { + alias + RemediableByAgent bool `json:"remediableByAgent"` + BeyondFreshnessSLO bool `json:"beyondFreshnessSlo"` + }{alias(c), c.RemediableByAgent(), c.BeyondFreshnessSLO()}) +} + +// Results projects emissions onto the record.Results a Run carries, in the +// order given. The order is the caller's: match.Match already imposes a total +// order on its output, and re-sorting here would create a second one. +// +// EACH RESULT IS CLAMPED ON THE WAY OUT. This is the projection that reaches +// the SARIF log, record.Validate and the store, so it is the one on which "a +// Result that did not come from Emit cannot present a host finding as +// actionable" has to be true rather than merely documented. +func Results(es []Emission) []record.Result { + out := make([]record.Result, 0, len(es)) + for _, e := range es { + out = append(out, e.clamped().Result) + } + return out +} + +// --------------------------------------------------------------------------- +// The host locator vocabulary +// --------------------------------------------------------------------------- + +// hostPackageManagers maps a host ecosystem onto the `package_manager` +// segment of the anvil-fp/v1 host locator. +// +// IT IS AN EXACT-MATCH ALLOWLIST and an unlisted ecosystem is refused, because +// the manager segment is hashed: a guessed manager forks the identity of every +// finding for that ecosystem at once, silently, and fingerprint-keyed +// suppression stops applying with nothing logging an error. +// +// THE `deb` ENTRY IS `apt`, NOT `dpkg`, AND THE CHOICE IS NOT FREE. The frozen +// conformance vector testdata/fingerprint_corpus/host-01-openssl-debian.json +// hashes the locator `apt:openssl:amd64` for a Debian host package, and its +// golden digest was produced by an independent implementation of +// FINGERPRINT-SPEC.md. Spelling it `dpkg` here would produce a valid-looking +// digest that disagrees with the corpus — the exact silent fork S6's +// one-fingerprint rule exists to prevent. emit_test.go pins the emitted digest +// against that golden file rather than against this map. +var hostPackageManagers = map[string]string{ + match.EcosystemDeb: "apt", + match.EcosystemRPM: "rpm", + match.EcosystemAPK: "apk", +} + +// HostPackageManager resolves a host ecosystem to its locator segment. +func HostPackageManager(ecosystem string) (string, bool) { + m, ok := hostPackageManagers[ecosystem] + return m, ok +} + +// hostIdentifier composes the host tier's `host_identifier`: the package +// identity as the manager names it. +// +// The architecture suffix is INCLUDED when the collector reported one. +// `openssl:amd64` and `openssl:i386` are two installed packages the manager +// treats separately, so they are two findings — the corpus vector says so in +// its own notes, and internal/collector/host moved the `:arch` qualifier off +// the name precisely so that this decision is made here and not by accident. +// Dropping it would collide two findings onto one digest on every multi-arch +// host. +func hostIdentifier(m match.MatchResult) string { + if m.Arch == "" { + return m.Package + } + return m.Package + ":" + m.Arch +} + +// --------------------------------------------------------------------------- +// The reasoning vocabulary +// --------------------------------------------------------------------------- + +// The closed set of sentence fragments `anvil/reasoning` may be built from. +// +// A string classified anvil_generated must be one Anvil wrote — not one Anvil +// formatted around bytes that came from elsewhere. A format string with a %s +// in it cannot make that promise, so there is no format string here: the +// reasoning is JOINED from these constants and from base-10 integers computed +// in this process, and composeReasoning refuses anything else. +const ( + reasonDeterministic = "Deterministic version comparison: the installed version falls inside a range the advisory declares affected. " + + "No model, no network and no clock participated in this decision." + reasonVendorRange = "The deciding range came from a vendor advisory rather than an upstream one." + reasonVendorDefended = "A vendor advisory range displaced at least one upstream range for the same advisory and package." + reasonHostReadOnly = "This is a host package finding. The coding agent's write surface is the git repository, so it cannot remediate a host package and is never handed one as actionable." + reasonRepoFixedNamed = "A fixed version is named, so a dependency version bump is expressible." + reasonRepoNoFixed = "The advisory names no fixed version, so there is no bump to propose." + reasonParseDegraded = "The advisory record parsed with loss, so this finding is partial: it is reported, and it is not offered to the coding agent as actionable." + reasonWatermark = "Age is measured from the ingestion cache watermark to the instant this record was assembled, which the caller supplied; no clock was read while deciding this finding." + reasonBeyondSLO = "The advisory data is older than the feed's stated freshness SLO." + reasonStaleThenSLONum = "Staleness in seconds, then the SLO in seconds:" + reasonStaleNum = "Staleness in seconds:" +) + +// reasoningVocabulary is the closed set above, as a set. +var reasoningVocabulary = map[string]bool{ + reasonDeterministic: true, + reasonVendorRange: true, + reasonVendorDefended: true, + reasonHostReadOnly: true, + reasonRepoFixedNamed: true, + reasonRepoNoFixed: true, + reasonParseDegraded: true, + reasonWatermark: true, + reasonBeyondSLO: true, + reasonStaleThenSLONum: true, + reasonStaleNum: true, +} + +// composeReasoning joins parts into the `anvil/reasoning` string, refusing any +// part that is neither a member of reasoningVocabulary nor a base-10 integer. +// +// This is the guard that makes the anvil_generated label on this one string +// true by construction. It is reachable — TestComposeReasoningRefusesAnExternalString +// drives it with a package name and asserts the refusal — because a guard that +// has never failed has not been tested. +func composeReasoning(parts []string) (string, error) { + if len(parts) == 0 { + return "", &Refusal{ + Reason: RefusalReasoningNotAnvilGenerated, + Detail: "no reasoning parts; an empty explanation is not an explanation", + } + } + for i, p := range parts { + if reasoningVocabulary[p] || isBase10Integer(p) { + continue + } + return "", &Refusal{ + Reason: RefusalReasoningNotAnvilGenerated, + Detail: "reasoning part " + strconv.Itoa(i) + " " + strconv.Quote(p) + + " is neither a member of the closed reasoning vocabulary nor an integer Anvil computed; " + + "a string that interpolates external bytes cannot be labelled " + string(record.TrustAnvilGenerated), + } + } + return strings.Join(parts, " "), nil +} + +// isBase10Integer reports whether s is one or more ASCII digits and nothing +// else. No sign, no separators, no Unicode digits: strconv.Atoi would accept a +// leading '+' or '-' and a Unicode-aware check would accept digits from other +// scripts, and neither is something this package produces. +func isBase10Integer(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return false + } + } + return true +} + +// --------------------------------------------------------------------------- +// remediable_by_agent — the ONE place this flag is computed +// --------------------------------------------------------------------------- + +// remediableByAgent answers whether the coding agent may be handed this +// finding as actionable. +// +// IT IS AN ALLOWLIST. The answer is false unless every condition below holds, +// so a collector, detector kind, evidence class or ecosystem that does not +// exist today arrives as NOT remediable rather than as remediable-by-omission. +// A denylist of the form "false when host" fails open on precisely the case +// nobody anticipated, and this project has already paid for that shape once. +// +// The conditions, and why each is necessary: +// +// 1. collector == repo-sca. plan/00-SPINE.md S7: the agent writes to the git +// repository and the host agent is read-only, "not behind a flag". A host +// package has no file for the agent to edit. +// 2. detectorKind == sca AND evidenceClass == sca. These are re-derived from +// the collector by detectorFor, and Emit refuses a MatchResult whose own +// values disagree. Both are checked here as well because +// record.IsHostFinding reads BOTH, and internal/record's own validator +// rejects a remediable result carrying either host value — a guard that +// trusts one field to imply another is a guard with a seam in it. +// 3. THE ECOSYSTEM IS NOT AN OS PACKAGE MANAGER'S. S7 is about the THING, +// not about who found it: `openssl` from apt is unfixable by a coding +// agent whether a host collector or a repo-SCA collector noticed it, +// because the write surface is the git repository either way. Without +// this arm a repo-sca match carrying Ecosystem=deb emits +// remediable_by_agent=true, and the agent is handed "bump openssl in +// Dockerfile" as actionable work against an apt package. That path is not +// reachable today only because internal/collector/repo refuses Trivy's +// os-pkgs class — one lane up, in a package this one's guarantee must not +// depend on. The test is membership of hostPackageManagers, the map this +// file already owns, so the ecosystem dimension is an allowlist rather +// than a denylist-by-omission. +// 4. fixedVersion != "". With no fixed version there is no bump to make, and +// dispatching an agent after a patch that does not exist wastes the one +// tier that costs money. +// 5. !parseDegraded. A.16 sets parse_degraded when the advisory arrived in a +// dataVersion the parser did not fully understand — and the fixed version +// in condition 4 was parsed out of that same record. Handing a +// repo-credentialed agent a bump to a version half-parsed from a record +// Anvil admits it did not understand is the "partial finding presenting +// as whole" defect A.16 exists to prevent. The finding is still EMITTED; +// it is emitted as report-only. THIS IS A DEVIATION from A.19's sketched +// expression, which reads only the collector, and it is reported: it can +// only ever move the answer from true to false. +// +// The function takes no options, reads no configuration and consults no clock, +// so there is no location for an override to live in. +func remediableByAgent( + collector string, + kind record.DetectorKind, + class record.EvidenceClass, + ecosystem string, + fixedVersion string, + parseDegraded bool, +) bool { + if collector != match.CollectorRepoSCA { + return false + } + if kind != record.DetectorKindSCA || class != record.EvidenceClassSCA { + return false + } + if isHostEcosystem(ecosystem) { + return false + } + if fixedVersion == "" { + return false + } + return !parseDegraded +} + +// isHostEcosystem reports whether an ecosystem names an OS package manager's +// world — i.e. whether hostPackageManagers can resolve it. +// +// One map, read from both directions: the host tier uses it to build the +// locator segment, and the allowlist above uses it to refuse an OS package that +// arrived under a repository collector's label. Two lists would drift. +func isHostEcosystem(ecosystem string) bool { + _, ok := hostPackageManagers[ecosystem] + return ok +} + +// detectorFor re-derives the frozen enums from the collector. +// +// They are DERIVED, not copied off the MatchResult, and Emit then refuses a +// MatchResult whose own values disagree. Reaching a gate is not obeying it: +// trusting a struct field that says `sca` next to a collector that says `host` +// is how an authorization decision gets made by whichever producer wrote the +// struct last. +func detectorFor(collector string) (record.DetectorKind, record.EvidenceClass, bool) { + switch collector { + case match.CollectorHost: + return record.DetectorKindHost, record.EvidenceClassHost, true + case match.CollectorRepoSCA: + return record.DetectorKindSCA, record.EvidenceClassSCA, true + default: + return "", "", false + } +} + +// --------------------------------------------------------------------------- +// Emit +// --------------------------------------------------------------------------- + +// Emit produces the canonical record for one A.17 match. +// +// a must be the `advisory` row the match's deciding range came from; a +// mismatch on (source, source_id) or on the CVE alias is refused rather than +// reconciled, because a record that attributes a finding to the wrong advisory +// is worse than no record. +// +// Every failure is a *Refusal. Nothing is defaulted, and nothing is emitted +// partially: a finding this package cannot state completely is one it declines +// to state. +func (e Emitter) Emit(m match.MatchResult, a AdvisoryRow) (Emission, error) { + refuse := func(reason RefusalReason, detail string, err error) (Emission, error) { + return Emission{}, &Refusal{ + Reason: reason, + Collector: m.Collector, + Package: m.Package, + Source: m.Source, + SourceID: m.SourceID, + Detail: detail, + Err: err, + } + } + + if strings.TrimSpace(e.TargetID) == "" { + return refuse(RefusalNoTargetID, + "the emitter carries no target id; it is field 1 of every anvil-fp/v1 tier", nil) + } + if e.AssembledAt.IsZero() { + return refuse(RefusalNoAssemblyTime, + "the emitter carries no record-assembly instant; staleness_seconds is defined as "+ + "assembly time minus as_of, this package holds no clock to supply the first term, "+ + "and defaulting it would either measure the age against the Unix epoch or report "+ + "stale feed data as fresh", nil) + } + + kind, class, ok := detectorFor(m.Collector) + if !ok { + return refuse(RefusalUnrecognisedCollector, + "collector must be "+strconv.Quote(match.CollectorHost)+" or "+ + strconv.Quote(match.CollectorRepoSCA)+"; remediable_by_agent is derived from it "+ + "and an authorization decision must not be defaulted", nil) + } + if m.Detector != kind || m.EvidenceClass != class { + return refuse(RefusalInconsistentMatch, + "collector implies detector="+string(kind)+" evidenceClass="+string(class)+ + " but the match carries detector="+string(m.Detector)+ + " evidenceClass="+string(m.EvidenceClass), nil) + } + + // The row must be the row this match was decided on. + if a.Source != m.Source || a.SourceID != m.SourceID { + return refuse(RefusalAdvisoryMismatch, + "advisory row is ["+a.Source+"/"+a.SourceID+"] but the match was decided on ["+ + m.Source+"/"+m.SourceID+"]", nil) + } + if a.CVEID != "" && m.CVEID != "" && a.CVEID != m.CVEID { + return refuse(RefusalAdvisoryMismatch, + "advisory row's cve alias is "+strconv.Quote(a.CVEID)+ + " but the match carries "+strconv.Quote(m.CVEID), nil) + } + + advisoryID := advisoryIDOf(m) + if advisoryID == "" { + return refuse(RefusalNoAdvisoryID, + "the match carries neither a cve id nor a source id", nil) + } + if m.Purl == "" { + return refuse(RefusalNoPurl, + "neither the package nor the deciding advisory range carried a purl, and this step "+ + "will not synthesise one: the namespace comes from os-release, which a MatchResult "+ + "does not carry, and a guessed namespace forks identity", nil) + } + + // --- The advisory row's own invariants. --- + if a.AsOf.IsZero() { + return refuse(RefusalNoWatermark, + "the advisory row carries no as_of; a finding whose data age is unstated reports "+ + "stale data as fresh, which converts an absence into a false assurance", nil) + } + if a.StalenessSeconds < 0 { + return refuse(RefusalNegativeStaleness, + "the row's own publisher-lag column is "+strconv.Itoa(a.StalenessSeconds)+ + "; the cache's advisory_staleness_nonneg CHECK refuses it too", nil) + } + // THE CONTRACT'S OWN QUANTITY: record-assembly time minus as_of. Not the + // row's publisher-lag column, which is a different measurement under the + // same name — see the package header, section 3. + staleness, ok := stalenessSeconds(e.AssembledAt, a.AsOf) + if !ok { + return refuse(RefusalNegativeStaleness, + "the record is being assembled at "+e.AssembledAt.UTC().Format(time.RFC3339)+ + ", which precedes the advisory watermark "+a.AsOf.UTC().Format(time.RFC3339)+ + "; a negative age is not a freshness claim this step can make", nil) + } + if n := len(a.ExcerptText); n > record.MaxAdvisoryExcerptBytes { + return refuse(RefusalExcerptTooLong, + "the advisory excerpt is "+strconv.Itoa(n)+" bytes and the record's cap is "+ + strconv.Itoa(record.MaxAdvisoryExcerptBytes)+"; the read path trims only the "+ + "agent-facing card, so an untrimmed excerpt would persist in the record and the store", nil) + } + if !a.Trust.Valid() { + return refuse(RefusalIllegalAdvisoryTrust, + "advisory anvil_trust "+strconv.Quote(string(a.Trust))+" is not a legal record.Trust value", nil) + } + if !a.Trust.LegalForExternalString() { + return refuse(RefusalIllegalAdvisoryTrust, + "advisory anvil_trust is "+strconv.Quote(string(a.Trust))+ + "; every byte in the advisory table originated outside Anvil, so only "+ + string(record.TrustUntrusted)+" and "+string(record.TrustVerified)+" are legal", nil) + } + step := strings.TrimSpace(a.TrustValidationStep) + if a.Trust == record.TrustVerified && step == "" { + return refuse(RefusalTrustValidationStep, + "the advisory row asserts "+string(record.TrustVerified)+" and names no validation step; "+ + "record.Trust defines that value as bytes that passed an explicit validation step "+ + "NAMED IN THE RECORD, never as a default, and a verified label with nothing behind "+ + "it is the mislabelling the trust vocabulary exists to prevent", nil) + } + if a.Trust != record.TrustVerified && step != "" { + return refuse(RefusalTrustValidationStep, + "the advisory row names the validation step "+strconv.Quote(step)+" but is classified "+ + strconv.Quote(string(a.Trust))+"; a validation claim attached to bytes nothing "+ + "validated launders the step's name onto untrusted text", nil) + } + if strings.TrimSpace(a.LicenseSPDX) == "" && strings.TrimSpace(a.LicenseManualNote) == "" { + return refuse(RefusalNoLicenceDeclared, + "the advisory row states neither license_spdx nor license_manual_note; spine S8 makes "+ + "that a row Anvil cannot prove it may use, and it must not be re-published into a record", nil) + } + + // --- The fingerprint. anvil-fp/v1, computed by internal/record. --- + var ( + digest string + err error + ) + switch kind { + case record.DetectorKindHost: + manager, known := HostPackageManager(m.Ecosystem) + if !known { + return refuse(RefusalUnsupportedHostEcosystem, + "host ecosystem "+strconv.Quote(m.Ecosystem)+" has no package-manager mapping; "+ + "the manager segment is hashed into the locator and guessing it forks identity", nil) + } + digest, err = record.Host(record.HostInput{ + TargetID: e.TargetID, + AdvisoryID: advisoryID, + Purl: m.Purl, + PackageManager: manager, + HostIdentifier: hostIdentifier(m), + }) + case record.DetectorKindSCA: + if m.ManifestRelPath == "" { + return refuse(RefusalNoManifestPath, + "a repo-sca match carries no manifest relpath; it is the SCA tier's locator, and "+ + "the same package pulled in by two manifests is two findings", nil) + } + digest, err = record.Sca(record.ScaInput{ + TargetID: e.TargetID, + AdvisoryID: advisoryID, + Purl: m.Purl, + ManifestRelPath: m.ManifestRelPath, + }) + } + if err != nil { + return refuse(RefusalFingerprint, + "internal/record refused to compute the anvil-fp/v1 digest", err) + } + + // --- The seven Lane-A-owned fields. --- + remediable := remediableByAgent(m.Collector, kind, class, m.Ecosystem, m.FixedVersion, a.ParseDegraded) + + beyondSLO := a.FreshnessSLOSeconds > 0 && staleness > a.FreshnessSLOSeconds + + reasoning, err := composeReasoning(reasoningParts(m, a, staleness, beyondSLO)) + if err != nil { + return Emission{}, err + } + + res := record.Result{ + RuleID: advisoryID, + Message: record.Message{Text: messageText(m, advisoryID)}, + PartialFingerprints: map[string]string{ + record.PartialFingerprintAnvilFindingID: digest, + }, + Properties: record.ResultProperties{ + FindingID: digest, + Half: record.HalfSast, + // Detector certainty, not data freshness. The comparison is + // deterministic, so it is 1. Degradation is expressed as a + // VERDICT, per plan/00-SPINE.md S6: "INSUFFICIENT_CONTEXT as a + // valid detector verdict, not just a confidence float." + Confidence: 1, + Verdict: verdictFor(a.ParseDegraded), + RemediableByAgent: remediable, + Reasoning: reasoning, + Detector: record.DetectorRef{ + Kind: kind, + // Model and Revision are EMPTY, and that is the honest + // statement: plan/00-SPINE.md S1 makes Lane A zero-inference + // and internal/match documents that there is no model + // anywhere in its call graph. Naming one here would assert + // an inference step that does not exist. The build stamp + // belongs on run.tool.driver, which this step does not own. + AdvisoryItemID: advisoryID, + }, + EvidenceClass: class, + Trust: record.TrustAssertion{ + // UNCONDITIONALLY untrusted. Package names, versions, purls, + // manifest paths, advisory ids and the excerpt all came from + // outside Anvil, and so does every string composed around + // them. Fields names the one string Anvil itself wrote, and + // composeReasoning is what makes that label true. + Default: record.TrustUntrusted, + Fields: map[string]record.Trust{ + reasoningPointer: record.TrustAnvilGenerated, + }, + }, + Advisory: &record.AdvisoryContext{ + IDs: []string{advisoryID}, + CveIDs: cveIDs(m, a), + SourceFeed: a.FeedID, + SnapshotDigest: a.SnapshotDigest, + LicenseSpdx: a.LicenseSPDX, + // AsOf IS THE CACHE WATERMARK, never a clock reading. + // StalenessSeconds is the contract's own interval — the + // caller's assembly instant minus that watermark — and NOT + // the row's publisher-lag column, which measures something + // else. See the package header, section 3. + AsOf: a.AsOf.UTC(), + StalenessSeconds: staleness, + ParseDegraded: a.ParseDegraded, + Excerpt: excerpt(a), + }, + Risk: riskOf(a), + }, + } + if loc := manifestLocation(m); loc != nil { + res.Locations = []record.Location{*loc} + } + + // The contract's own validator, applied before this leaves the function. + // It re-checks the host rule from the other side (it refuses a remediable + // result carrying DetectorKindHost or EvidenceClassHost) and it re-checks + // the trust classification. A producer that fails here has built something + // the record forbids, and must not hand it on. + if err := validateEmitted(&res); err != nil { + return refuse(RefusalContractViolation, + "the assembled result failed internal/record's own validation", err) + } + + return Emission{ + Result: res, + LicenseManualNote: manualNote(a), + FreshnessSLOSeconds: a.FreshnessSLOSeconds, + TrustValidationStep: step, + }, nil +} + +// stalenessSeconds is AdvisoryContext.StalenessSeconds as the frozen contract +// defines it: record-assembly time minus the advisory watermark, in whole +// seconds, truncated toward zero. +// +// It reports false rather than a negative number when the watermark is in the +// future relative to assembly. A negative age is not a freshness claim this +// step can make, and clamping it to zero would state "current as of now" about +// data whose own timestamps say otherwise. +func stalenessSeconds(assembledAt, asOf time.Time) (int, bool) { + d := assembledAt.Sub(asOf) + if d < 0 { + return 0, false + } + return int(d / time.Second), true +} + +// EmitAll emits every match in order, resolving each one's advisory row +// through lookup. +// +// The order is the caller's — match.Match already imposes a total order on its +// output — and the first refusal stops the run, which is deterministic for the +// same reason. A match whose advisory row cannot be resolved is a refusal, not +// a skip: a finding silently dropped between the comparator and the record is +// indistinguishable from a clean scan. +func (e Emitter) EmitAll( + ms []match.MatchResult, + lookup func(source, sourceID string) (AdvisoryRow, bool), +) ([]Emission, error) { + if lookup == nil { + return nil, &Refusal{ + Reason: RefusalAdvisoryMismatch, + Detail: "EmitAll needs a lookup for the advisory rows the matches were decided on", + } + } + out := make([]Emission, 0, len(ms)) + for _, m := range ms { + row, ok := lookup(m.Source, m.SourceID) + if !ok { + return nil, &Refusal{ + Reason: RefusalAdvisoryMismatch, + Collector: m.Collector, + Package: m.Package, + Source: m.Source, + SourceID: m.SourceID, + Detail: "no advisory row resolved for the range that decided this match; " + + "a finding dropped here is indistinguishable from a clean scan", + } + } + em, err := e.Emit(m, row) + if err != nil { + return nil, err + } + out = append(out, em) + } + return out, nil +} + +// --------------------------------------------------------------------------- +// Field derivation +// --------------------------------------------------------------------------- + +// reasoningPointer is the RFC 6901 pointer, relative to the result, of the +// `anvil/reasoning` string. The '/' inside the property key is escaped as +// '~1' per RFC 6901 §3. +const reasoningPointer = "/properties/anvil~1reasoning" + +// advisoryIDOf is the advisory identity hashed into the fingerprint and +// written to `ruleId`. +// +// The CVE id when there is one, the source id otherwise. That is A.17's own +// precedence group (AffectedRange.advisoryKey), and it is the right way round: +// GHSA advisories frequently carry no CVE at all (research/06 Risk #2), and +// grouping those under one empty key would let unrelated advisories collide. +// It is used VERBATIM and never case-folded — GHSA identifiers mix case +// meaningfully and folding them would fork identity against the advisory +// table (FINGERPRINT-SPEC.md §2.2, field 3). +func advisoryIDOf(m match.MatchResult) string { + if m.CVEID != "" { + return m.CVEID + } + return m.SourceID +} + +// cveIDs returns the CVE aliases, or nil when the advisory carries none. Nil +// rather than a one-element slice holding "": an empty string in a list of +// identifiers is a value a consumer will try to look up. +func cveIDs(m match.MatchResult, a AdvisoryRow) []string { + switch { + case m.CVEID != "": + return []string{m.CVEID} + case a.CVEID != "": + return []string{a.CVEID} + default: + return nil + } +} + +// verdictFor maps A.16's parse_degraded flag onto the record's verdict. +// +// A deterministic version comparison against a fully-understood advisory is a +// true positive by construction: the installed version IS inside the declared +// range. When the advisory record parsed with loss, the range itself is +// partial, and plan/00-SPINE.md S6's insufficient_context is the exact +// statement — "this may well be a real defect and the detector could not see +// enough to tell". The consumption pipeline demotes it to report-only and +// never silently drops it, which is the handling a partial finding needs. +// +// STALENESS DOES NOT ENTER HERE. See the package header, section 3. +func verdictFor(parseDegraded bool) record.Verdict { + if parseDegraded { + return record.VerdictInsufficientContext + } + return record.VerdictTruePositive +} + +// reasoningParts builds the ordered parts of `anvil/reasoning`. Every element +// is a constant from the closed vocabulary or an integer computed here; +// composeReasoning enforces that. +// +// staleness is the COMPUTED interval Emit wrote onto the record, passed in +// rather than re-derived, so that the number a human reads in the prose and the +// number a machine reads in the field cannot drift apart. +func reasoningParts(m match.MatchResult, a AdvisoryRow, staleness int, beyondSLO bool) []string { + parts := []string{reasonDeterministic} + if m.VendorAdvisory { + parts = append(parts, reasonVendorRange) + } + if m.DistroBackportDefended { + parts = append(parts, reasonVendorDefended) + } + if m.Collector == match.CollectorHost { + parts = append(parts, reasonHostReadOnly) + } else if m.FixedVersion != "" { + parts = append(parts, reasonRepoFixedNamed) + } else { + parts = append(parts, reasonRepoNoFixed) + } + if a.ParseDegraded { + parts = append(parts, reasonParseDegraded) + } + parts = append(parts, reasonWatermark) + if beyondSLO { + parts = append(parts, reasonBeyondSLO, reasonStaleThenSLONum, + strconv.Itoa(staleness), strconv.Itoa(a.FreshnessSLOSeconds)) + } else { + parts = append(parts, reasonStaleNum, strconv.Itoa(staleness)) + } + return parts +} + +// messageText is the human-readable statement of the finding. +// +// It INTERPOLATES EXTERNAL STRINGS — the package name, the installed version, +// the advisory id, the range expression — and is therefore untrusted, which +// the result's TrustAssertion.Default already says. That is exactly why the +// reasoning string is built separately and from constants: one field states +// the facts as reported, the other states what Anvil did, and only the second +// can honestly be labelled anvil_generated. +func messageText(m match.MatchResult, advisoryID string) string { + var b strings.Builder + b.WriteString(advisoryID) + b.WriteString(": ") + b.WriteString(m.Package) + if m.Arch != "" { + b.WriteString(" (") + b.WriteString(m.Arch) + b.WriteString(")") + } + b.WriteString(" ") + b.WriteString(m.InstalledVersion) + b.WriteString(" is inside the affected range ") + b.WriteString(m.MatchedRange) + if m.FixedVersion != "" { + b.WriteString("; fixed in ") + b.WriteString(m.FixedVersion) + } else { + b.WriteString("; the advisory names no fixed version") + } + b.WriteString(".") + return b.String() +} + +// excerpt wraps the advisory excerpt with the trust the `advisory` row itself +// recorded. Nil when there is no excerpt: an empty TrustedString would put an +// empty quotation on the record. +// +// The BOUND is checked by Emit before this is reached, and `verified` without a +// named validation step is refused there too. Sanitisation is NOT established +// here; AdvisoryRow.ExcerptText says exactly what is and is not known about +// these bytes, and this function makes no claim beyond that. +func excerpt(a AdvisoryRow) *record.TrustedString { + if a.ExcerptText == "" { + return nil + } + return &record.TrustedString{Text: a.ExcerptText, Trust: a.Trust} +} + +// manualNote wraps spine S8's manual-override sentence with its trust. It is a +// QUOTATION from a publisher's LICENSE file, so it is external text and takes +// the row's own trust level — never anvil_generated, however Anvil-shaped the +// struct around it looks. +func manualNote(a AdvisoryRow) *record.TrustedString { + if strings.TrimSpace(a.LicenseManualNote) == "" { + return nil + } + return &record.TrustedString{Text: a.LicenseManualNote, Trust: a.Trust} +} + +// cvss40Prefix is the CVSS v4.0 vector prefix. record.Risk.CvssV4Base is +// explicitly the v4 base score. +const cvss40Prefix = "CVSS:4.0/" + +// riskOf carries research/24's ranking inputs. record.Risk names Lane A +// ingestion as its producer, which is this step. +// +// CvssV4Base IS SET ONLY FOR A v4 VECTOR. The cache stores one cvss_score +// column beside one cvss_vector, and the score for a v3.1 vector is not a v4 +// base score: writing it into a field named CvssV4Base would put a number in +// the record that is wrong in a way no consumer can detect. A v3.1 row +// therefore contributes EPSS and KEV and leaves the CVSS slot null, which is +// the honest reading. +func riskOf(a AdvisoryRow) *record.Risk { + r := record.Risk{ + EpssScore: a.EPSSScore, + EpssPercentile: a.EPSSPercentile, + EpssModelDate: a.EPSSAsOf, + KevMember: a.KEVMember, + KevRansomwareUse: a.KEVRansomwareUse, + } + if a.CVSSScore != nil && strings.HasPrefix(a.CVSSVector, cvss40Prefix) { + r.CvssV4Base = a.CVSSScore + } + if r.CvssV4Base == nil && r.EpssScore == nil && r.EpssPercentile == nil && + r.EpssModelDate == nil && !r.KevMember && !r.KevRansomwareUse { + return nil + } + return &r +} + +// manifestLocation is the SARIF-native location for a repository dependency: +// the manifest that declared it. +// +// There is no Region, so no line hash is required and none is invented — the +// dependency is declared by the file, not by a line, and a fabricated line +// number would be a claim about the file this step has not read. A host +// package has no repo-relative location at all and gets none, which is also +// what makes it visibly not something the agent can edit. +func manifestLocation(m match.MatchResult) *record.Location { + if m.Collector != match.CollectorRepoSCA || m.ManifestRelPath == "" { + return nil + } + uri := record.CanonicalRepoRelPath(m.ManifestRelPath) + if uri == "" { + return nil + } + return &record.Location{ + PhysicalLocation: &record.PhysicalLocation{ + ArtifactLocation: record.ArtifactLocation{URI: uri}, + }, + } +} + +// validateEmitted runs internal/record's own per-result checks on a freshly +// assembled result, by putting it through the contract's whole-record +// validator inside a minimal, well-formed envelope. +// +// The envelope is scaffolding for the check and is thrown away: this step does +// not own the audit envelope, and the scan controller (O.2) assembles the real +// one. What matters is that Result.validate runs — it is where the record +// refuses a remediable host finding from the other side, and where +// ValidateResultTrust refuses an anvil_generated classification on an external +// string. +func validateEmitted(res *record.Result) error { + // THE RECORD'S OWN TRUST VALIDATOR IS VACUOUS FOR A LANE A RESULT, AND + // SAYING SO IS THE POINT. record.Result.ExternalStringPointers enumerates + // SARIF-native slots that hold external text — region snippets, code-flow + // snippets, the DAST response body — and an SCA or host finding carries + // none of them. So ValidateResultTrust below iterates an empty list and + // would accept ANY Default, including the anvil_generated that record.Trust + // documents area B being caught with. Verified empirically: setting Default + // to anvil_generated in this file passes record's whole-record Validate. + // + // The untrusted default is therefore not enforced by the contract for this + // shape; it is enforced HERE, on the emitted value, so that the guarantee + // is a property of the code rather than of a test that could be deleted. + if res.Properties.Trust.Default != record.TrustUntrusted { + return fmt.Errorf( + "anvil/trust.default is %q; every Lane A result is built from package names, "+ + "versions, purls, manifest paths and advisory text that originated outside "+ + "Anvil, so the default is %q unconditionally (00-SPINE.md S6)", + res.Properties.Trust.Default, record.TrustUntrusted) + } + for ptr, tr := range res.Properties.Trust.Fields { + if tr == record.TrustAnvilGenerated && ptr != reasoningPointer { + return fmt.Errorf( + "%s is classified %q; the only string this step writes itself is %s", + ptr, tr, reasoningPointer) + } + } + + sealedAt := time.Unix(0, 0).UTC() + createdAt := time.Unix(0, 0).UTC() + log := record.SARIFLog{ + Schema: record.SARIFSchemaURI, + Version: record.SARIFVersion, + Properties: record.AuditProperties{ + SchemaVersion: record.SchemaVersion, + AuditID: "validate-emitted", + State: record.StateBothSealed, + Version: 1, + CreatedAt: createdAt, + // Placeholders in throwaway scaffolding: the enums have no + // "not applicable" member and this envelope is never emitted. + Target: record.Target{ + Provenance: record.TargetProvenanceNoTargetDeclared, + Provisioning: record.TargetProvisioningEphemeralManifest, + }, + DastStatus: record.DastStatusNotRun, + Deadline: record.Deadline{ + DeadlineAt: createdAt.Add(record.DefaultClaimTimeoutSeconds * time.Second), + ClaimTimeoutSeconds: record.DefaultClaimTimeoutSeconds, + }, + }, + Runs: []record.Run{{ + AutomationDetails: record.RunAutomationDetails{CorrelationGUID: "validate-emitted"}, + Results: []record.Result{*res}, + Properties: record.RunProperties{ + Half: record.HalfSast, + Status: record.HalfStatusSealed, + SealedAt: &sealedAt, + }, + }}, + } + if err := log.Validate(); err != nil { + return err + } + // Belt and braces: ValidateResultTrust is reached through Validate, but + // it is the check that would have caught area B's mislabelling and it is + // cheap enough to state twice rather than to depend on one call chain. + return record.ValidateResultTrust(res) +} + +// assert at compile time that Emission satisfies json.Marshaler, so that a +// future edit removing MarshalJSON — and with it the serialised +// remediableByAgent guarantee — fails to build rather than silently shipping. +var _ json.Marshaler = Emission{} diff --git a/internal/record/lanea/emit_test.go b/internal/record/lanea/emit_test.go new file mode 100644 index 0000000..ec47b81 --- /dev/null +++ b/internal/record/lanea/emit_test.go @@ -0,0 +1,1781 @@ +// Tests for A.19's record emission. +// +// =========================================================================== +// WHAT THESE TESTS ARE BUILT TO CATCH, AND WHERE THEIR INPUTS COME FROM +// =========================================================================== +// +// Two properties are load-bearing and A.20 exists to attack them: +// +// 1. remediable_by_agent is FALSE for every host finding, with no code path, +// flag or config key able to override it. It is an authorization decision +// gating what a repo-credentialed coding agent may act on. +// 2. anvil/trust is correct on every string originating outside Anvil. +// +// So property 1 is tested EXHAUSTIVELY over the whole input space of the +// function that computes it (TestRemediableByAgentIsAnAllowlistOverItsWholeInputSpace), +// then again END TO END on the SERIALISED BYTES of every shape that leaves +// this package, and then again on the task card — the artifact that actually +// reaches the agent. Testing the struct field alone is what A.12 found +// insufficient one lane over. +// +// THE FINGERPRINT CORPUS IS NOT THIS PACKAGE'S. TestEmittedFingerprintMatches +// TheFrozenConformanceGoldens reads testdata/fingerprint_corpus/*.json for its +// inputs and *.golden for its expected digests. Those goldens were produced by +// scripts/compute_golden_fingerprints.py — an implementation of +// FINGERPRINT-SPEC.md written WITHOUT reading internal/record/fingerprint.go — +// so the test proves this package emits the canonical digest rather than +// proving it agrees with itself. A test whose corpus comes from the +// implementation is not a test. +// +// EVERY GUARD IS DRIVEN RED. TestEveryRefusalReasonIsReachable drives all +// eighteen refusal reasons and fails if any one of them cannot be provoked; a +// guard that has never failed has not been tested. +// +// =========================================================================== +// NO TEST HERE MAY USE AN INPUT FIELD AS ITS OWN EXPECTED VALUE +// =========================================================================== +// +// A.20 found two tests in this file that could not fail for the reason they +// claimed. The staleness test set `StalenessSeconds` on the input row and +// asserted the same number came out, which proves copy-through and nothing +// else — and copy-through was the defect. Its "fresh" control was worse: it +// depended on the wall-clock distance to a hard-coded fixture date, so it would +// have started failing on a calendar day rather than on a code change. +// +// So `advisory.staleness_seconds` is now a DECOY in the fixtures +// (fixturePublisherLag), deliberately unlike anything the record should carry, +// and the emitted interval is asserted against arithmetic the test states +// itself: a watermark and an assembly instant chosen here, subtracted here. +// Nothing in the assertion comes from the code under test. +package lanea + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/match" + "github.com/Susquehanna-Syntax/Anvil/internal/record" +) + +// --------------------------------------------------------------------------- +// Fixtures +// +// The strings below are deliberately DISTINCTIVE so that +// TestReasoningCarriesNoByteFromOutsideAnvil can look for them: a package +// named "openssl" would be indistinguishable from prose about openssl. +// --------------------------------------------------------------------------- + +const ( + fixtureTargetID = "t-lanea-0001" + fixtureHostPkg = "zzhostpkgqx" + fixtureRepoPkg = "zzrepopkgqx" + fixtureHostVer = "9.9.9zzverqx-1" + fixtureRepoVer = "4.4.4zzverqx" + fixtureExcerpt = "ZZEXCERPTQX advisory prose that originated outside Anvil." + fixtureManualNote = "ZZLICENCEQX: \"You may redistribute this database under the ODbL.\"" + fixtureManifest = "services/zzmanifestqx/pom.xml" + + // fixtureValidationStep names the explicit validation step a row + // asserting `verified` has to point at. record.Trust says that value + // means bytes that "passed an explicit validation step that is named in + // the record ... Never a default", so a fixture that sets `verified` + // without one is a fixture the emitter refuses. + fixtureValidationStep = "zzstepqx: detached OpenPGP signature over the snapshot" +) + +// fixtureWatermark is the `advisory.as_of` cache watermark every fixture row +// carries. It is in the past and is never "now": the whole point of as_of is +// that it is not the emission instant. +var fixtureWatermark = time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + +// fixtureAssemblyLagSeconds is the interval the default fixtures span, and +// fixtureAssembledAt is the record-assembly instant that produces it. Both are +// fixed constants, so every default emission carries stalenessSeconds = 3600 +// on every calendar day this suite is ever run. +const fixtureAssemblyLagSeconds = 3600 + +var fixtureAssembledAt = fixtureWatermark.Add(fixtureAssemblyLagSeconds * time.Second) + +// fixturePublisherLag is `advisory.staleness_seconds`: the publisher lag +// ingestion stamped at write time. +// +// IT IS A DECOY AND THAT IS ITS WHOLE JOB. The record's staleness_seconds is a +// different quantity — assembly time minus as_of — and this value is chosen to +// resemble it in no way at all. Any assertion in this file that passes while +// this number is being copied onto the record is an assertion that has stopped +// testing anything, and several of them fail loudly if it is. +const fixturePublisherLag = 1234567 + +func hostMatch() match.MatchResult { + return match.MatchResult{ + Source: "ubuntu", + SourceID: "USN-9999-1", + CVEID: "CVE-2026-99999", + Collector: match.CollectorHost, + Ecosystem: match.EcosystemDeb, + Scheme: match.SchemeDebian, + Package: fixtureHostPkg, + Purl: "pkg:deb/ubuntu/" + fixtureHostPkg, + Arch: "amd64", + InstalledVersion: fixtureHostVer, + MatchedRange: "[0, 9.9.9zzverqx-2)", + FixedVersion: "9.9.9zzverqx-2", + VendorAdvisory: true, + DistroBackportDefended: true, + Detector: record.DetectorKindHost, + EvidenceClass: record.EvidenceClassHost, + Trust: record.TrustAnvilGenerated, + RemediableByAgent: false, + } +} + +func repoMatch() match.MatchResult { + return match.MatchResult{ + Source: "ghsa", + SourceID: "GHSA-zzzz-qqqq-xxxx", + Collector: match.CollectorRepoSCA, + Ecosystem: "maven", + Package: fixtureRepoPkg, + Purl: "pkg:maven/org.zzrepoqx/" + fixtureRepoPkg + "@" + fixtureRepoVer, + ManifestRelPath: fixtureManifest, + InstalledVersion: fixtureRepoVer, + MatchedRange: "[4.0.0, 4.5.0)", + FixedVersion: "4.5.0", + Detector: record.DetectorKindSCA, + EvidenceClass: record.EvidenceClassSCA, + Trust: record.TrustAnvilGenerated, + RemediableByAgent: true, + } +} + +func hostRow() AdvisoryRow { + return AdvisoryRow{ + Source: "ubuntu", + SourceID: "USN-9999-1", + CVEID: "CVE-2026-99999", + FeedID: "ubuntu-usn", + SnapshotDigest: "sha256:zzsnapshotqx", + LicenseSPDX: "LicenseRef-ubuntu-usn", + LicenseManualNote: fixtureManualNote, + Trust: record.TrustUntrusted, + AsOf: fixtureWatermark, + StalenessSeconds: fixturePublisherLag, + FreshnessSLOSeconds: 86400, + ExcerptText: fixtureExcerpt, + } +} + +func repoRow() AdvisoryRow { + return AdvisoryRow{ + Source: "ghsa", + SourceID: "GHSA-zzzz-qqqq-xxxx", + FeedID: "ghsa", + SnapshotDigest: "sha256:zzsnapshotqx", + LicenseSPDX: "CC-BY-4.0", + Trust: record.TrustUntrusted, + AsOf: fixtureWatermark, + StalenessSeconds: fixturePublisherLag, + FreshnessSLOSeconds: 86400, + ExcerptText: fixtureExcerpt, + } +} + +// emitter is the fixture Emitter. It carries a FIXED assembly instant: the +// package holds no clock, the caller owns the one it uses, and a test that let +// that instant be "now" would be a test whose expected values change daily. +func emitter() Emitter { + return Emitter{TargetID: fixtureTargetID, AssembledAt: fixtureAssembledAt} +} + +func mustEmit(t *testing.T, m match.MatchResult, a AdvisoryRow) Emission { + t.Helper() + em, err := emitter().Emit(m, a) + if err != nil { + t.Fatalf("Emit: %v", err) + } + return em +} + +// --------------------------------------------------------------------------- +// PROPERTY 1 — remediable_by_agent +// --------------------------------------------------------------------------- + +// TestRemediableByAgentIsAnAllowlistOverItsWholeInputSpace enumerates the +// ENTIRE input space of the one function that computes the flag and asserts +// the guarantee on every point of it. There are no untested combinations left +// for a config key or a future collector to hide in. +// +// THE ECOSYSTEM IS ONE OF THE ENUMERATED DIMENSIONS. A.20 found it missing: +// the other four arms were covered exhaustively while an OS package arriving +// under the repo-SCA label went through as agent-fixable. The enumeration below +// includes the three host ecosystems, three repository ones, the empty string +// and a near-miss capitalisation, so the arm is allowlisted over its whole +// space rather than spot-checked. +func TestRemediableByAgentIsAnAllowlistOverItsWholeInputSpace(t *testing.T) { + collectors := []string{ + match.CollectorHost, match.CollectorRepoSCA, + "", "HOST", "host ", "repo_sca", "sbom", "container", "anything-new", + } + kinds := append(record.DetectorKindValues(), record.DetectorKind(""), record.DetectorKind("HOST")) + classes := append(record.EvidenceClassValues(), record.EvidenceClass(""), record.EvidenceClass("Host")) + ecosystems := []string{ + match.EcosystemDeb, match.EcosystemRPM, match.EcosystemAPK, + "maven", "npm", "pypi", "", "DEB", + } + fixed := []string{"", "1.2.3"} + degraded := []bool{false, true} + + trueCount, hostEcoPoints, points := 0, 0, 0 + for _, c := range collectors { + for _, k := range kinds { + for _, cl := range classes { + for _, eco := range ecosystems { + for _, f := range fixed { + for _, d := range degraded { + got := remediableByAgent(c, k, cl, eco, f, d) + points++ + + // THE HOST RULE, from all FOUR directions: the + // collector that found it, the two frozen enums, + // and the ecosystem of the thing itself. + hostEco := eco == match.EcosystemDeb || + eco == match.EcosystemRPM || eco == match.EcosystemAPK + if c == match.CollectorHost || k == record.DetectorKindHost || + cl == record.EvidenceClassHost || hostEco { + if hostEco { + hostEcoPoints++ + } + if got { + t.Fatalf("remediableByAgent(%q,%q,%q,%q,%q,%t) = true; "+ + "an OS package is not agent-fixable and a host finding is "+ + "never remediable (00-SPINE.md S7)", + c, k, cl, eco, f, d) + } + continue + } + + // THE ALLOWLIST: true on exactly one shape. + want := c == match.CollectorRepoSCA && + k == record.DetectorKindSCA && + cl == record.EvidenceClassSCA && + f != "" && !d + if got != want { + t.Fatalf("remediableByAgent(%q,%q,%q,%q,%q,%t) = %t, want %t", + c, k, cl, eco, f, d, got, want) + } + if got { + trueCount++ + } + } + } + } + } + } + } + if trueCount == 0 { + t.Fatal("no input produced true; a guard that can never say yes has not been exercised") + } + if hostEcoPoints == 0 { + t.Fatal("no point of the space carried a host ecosystem, so the arm A.20 found " + + "missing is still not covered") + } + if want := len(collectors) * len(kinds) * len(classes) * len(ecosystems) * len(fixed) * len(degraded); points != want { + t.Fatalf("enumerated %d points, want %d; the space is not being walked whole", points, want) + } +} + +// TestAnOSPackageUnderTheRepoLabelIsNotRemediableInTheSERIALISEDBytes is A.20's +// finding 2, end to end. +// +// A repo-SCA MatchResult whose ecosystem is deb, rpm or apk is an OS package +// that a repository collector noticed. The coding agent's write surface is the +// git repository either way, so it cannot bump an apt package however the +// finding was labelled — and being handed "bump openssl in Dockerfile" as +// actionable work is the same authorization defect S7 exists to prevent. +// +// The path is unreachable today only because internal/collector/repo refuses +// Trivy's os-pkgs class. That is one lane up. This asserts it here. +func TestAnOSPackageUnderTheRepoLabelIsNotRemediableInTheSERIALISEDBytes(t *testing.T) { + for _, eco := range []string{match.EcosystemDeb, match.EcosystemRPM, match.EcosystemAPK} { + t.Run(eco, func(t *testing.T) { + m := repoMatch() + m.Ecosystem = eco + m.Purl = "pkg:" + eco + "/vendor/" + fixtureRepoPkg + "@" + fixtureRepoVer + m.RemediableByAgent = true // the most hostile input available + + em := mustEmit(t, m, repoRow()) + + if em.Result.Properties.RemediableByAgent { + t.Error("an OS package under the repo-sca label was emitted as agent-remediable") + } + if em.RemediableByAgent() { + t.Error("Emission.RemediableByAgent() = true for an OS package") + } + assertJSONBool(t, em, "remediableByAgent", false) + assertJSONBool(t, em, "result/properties/anvil~1remediableByAgent", false) + assertJSONBool(t, em.Result, "properties/anvil~1remediableByAgent", false) + assertJSONBool(t, Results([]Emission{em})[0], "properties/anvil~1remediableByAgent", false) + }) + } + + // THE CONTROL: the same match in a repository ecosystem IS remediable, so + // the arm above is shown to refuse the OS packages rather than everything. + control := mustEmit(t, repoMatch(), repoRow()) + if !control.RemediableByAgent() { + t.Fatal("a maven dependency with a named fixed version is not remediable; the ecosystem " + + "arm is refusing everything and proves nothing") + } + assertJSONBool(t, control, "remediableByAgent", true) +} + +// TestHostFindingsAreNeverRemediableInTheSERIALISEDBytes is the end-to-end +// half. It varies every host-side input that could plausibly be argued into +// flipping the flag — a named fixed version, a vendor advisory, a defended +// backport, a degraded parse, a verified feed, an arch — and asserts the +// answer in the BYTES of every shape that leaves this package. +func TestHostFindingsAreNeverRemediableInTheSERIALISEDBytes(t *testing.T) { + fixedVersions := []string{"", "9.9.9zzverqx-2"} + vendor := []bool{false, true} + degraded := []bool{false, true} + trusts := []record.Trust{record.TrustUntrusted, record.TrustVerified} + arches := []string{"", "amd64", "i386"} + + seen := 0 + for _, fv := range fixedVersions { + for _, v := range vendor { + for _, d := range degraded { + for _, tr := range trusts { + for _, arch := range arches { + m := hostMatch() + m.FixedVersion = fv + m.VendorAdvisory = v + m.DistroBackportDefended = v + m.Arch = arch + // The most hostile input available: a MatchResult + // that ASSERTS it is remediable. + m.RemediableByAgent = true + + a := hostRow() + a.ParseDegraded = d + a.Trust = tr + if tr == record.TrustVerified { + a.TrustValidationStep = fixtureValidationStep + } + + em := mustEmit(t, m, a) + seen++ + + if em.Result.Properties.RemediableByAgent { + t.Fatalf("host finding emitted RemediableByAgent=true (fixed=%q vendor=%t degraded=%t)", + fv, v, d) + } + if em.RemediableByAgent() { + t.Fatal("Emission.RemediableByAgent() = true for a host finding") + } + + assertJSONBool(t, em.Result, "properties/anvil~1remediableByAgent", false) + assertJSONBool(t, em, "remediableByAgent", false) + // BOTH LEVELS of the crossing object, not just the + // mirror: A.20 found them able to disagree. + assertJSONBool(t, em, "result/properties/anvil~1remediableByAgent", false) + assertJSONBool(t, Results([]Emission{em})[0], + "properties/anvil~1remediableByAgent", false) + } + } + } + } + } + if seen == 0 { + t.Fatal("no host emission was exercised") + } +} + +// TestTheArtifactThatReachesTheAgentSaysHostIsNotActionable pushes emitted +// results through internal/record's OWN read path — the task card is what the +// coding agent actually receives — and asserts the host finding arrives not +// actionable, with a stated reason, while the repo dependency arrives +// actionable. A record-level flag that the card ignored would be a flag that +// never reached the boundary it exists to guard. +func TestTheArtifactThatReachesTheAgentSaysHostIsNotActionable(t *testing.T) { + hostEm := mustEmit(t, hostMatch(), hostRow()) + repoEm := mustEmit(t, repoMatch(), repoRow()) + + log := sealedLog(t, Results([]Emission{hostEm, repoEm})) + cards, err := record.NewReader(nil).CardsFromLog(&log) + if err != nil { + t.Fatalf("CardsFromLog: %v", err) + } + if len(cards) != 2 { + t.Fatalf("got %d cards, want 2", len(cards)) + } + + byID := map[string]record.TaskCard{} + for _, c := range cards { + byID[c.FindingID] = c + } + hostCard, ok := byID[hostEm.Result.Properties.FindingID] + if !ok { + t.Fatal("no card for the host finding") + } + repoCard, ok := byID[repoEm.Result.Properties.FindingID] + if !ok { + t.Fatal("no card for the repo finding") + } + + if hostCard.RemediableByAgent { + t.Error("the host TASK CARD says remediableByAgent=true") + } + if hostCard.Actionable { + t.Error("the host TASK CARD is actionable") + } + if len(hostCard.ActionBlockers) == 0 { + t.Error("the host card is not actionable and says nothing about why") + } + assertJSONBool(t, hostCard, "remediableByAgent", false) + assertJSONBool(t, hostCard, "actionable", false) + + if !repoCard.RemediableByAgent || !repoCard.Actionable { + t.Errorf("the repo dependency card is not actionable (remediable=%t actionable=%t blockers=%v); "+ + "a guard that refuses everything has not been shown to permit anything", + repoCard.RemediableByAgent, repoCard.Actionable, repoCard.ActionBlockers) + } + assertJSONBool(t, repoCard, "remediableByAgent", true) +} + +// TestParseDegradedDemotesARepoFindingRatherThanDroppingIt pins A.16's flag +// end to end: the finding is still emitted, it carries parseDegraded in the +// bytes, its verdict is insufficient_context (report-only, never dropped), and +// it is not offered to the agent. +func TestParseDegradedDemotesARepoFindingRatherThanDroppingIt(t *testing.T) { + m := repoMatch() + a := repoRow() + a.ParseDegraded = true + + em := mustEmit(t, m, a) + + if em.Result.Properties.Verdict != record.VerdictInsufficientContext { + t.Errorf("verdict = %q, want %q", em.Result.Properties.Verdict, record.VerdictInsufficientContext) + } + if em.Result.Properties.RemediableByAgent { + t.Error("a partially-parsed advisory was offered to the agent as remediable") + } + assertJSONBool(t, em.Result, "properties/anvil~1advisory/parseDegraded", true) + assertJSONBool(t, em.Result, "properties/anvil~1remediableByAgent", false) + + // And the undegraded control, so the demotion is shown to be caused by + // the flag rather than by everything being demoted. + clean := mustEmit(t, m, repoRow()) + if clean.Result.Properties.Verdict != record.VerdictTruePositive { + t.Errorf("undegraded verdict = %q, want %q", clean.Result.Properties.Verdict, record.VerdictTruePositive) + } + if !clean.Result.Properties.RemediableByAgent { + t.Error("an undegraded repo dependency with a fixed version is not remediable") + } +} + +// --------------------------------------------------------------------------- +// PROPERTY 2 — anvil/trust +// --------------------------------------------------------------------------- + +// TestTrustDefaultIsUntrustedOnEveryEmission asserts the classification in the +// bytes, for both collectors: every string in a Lane A result came from +// outside Anvil except the reasoning. +func TestTrustDefaultIsUntrustedOnEveryEmission(t *testing.T) { + for _, em := range []Emission{ + mustEmit(t, hostMatch(), hostRow()), + mustEmit(t, repoMatch(), repoRow()), + } { + if got := em.Result.Properties.Trust.Default; got != record.TrustUntrusted { + t.Fatalf("trust default = %q, want %q", got, record.TrustUntrusted) + } + assertJSONString(t, em.Result, "properties/anvil~1trust/default", string(record.TrustUntrusted)) + + fields := em.Result.Properties.Trust.Fields + if got := fields[reasoningPointer]; got != record.TrustAnvilGenerated { + t.Fatalf("trust.fields[%q] = %q, want %q", reasoningPointer, got, record.TrustAnvilGenerated) + } + // Nothing else may claim anvil_generated. + for ptr, tr := range fields { + if tr == record.TrustAnvilGenerated && ptr != reasoningPointer { + t.Fatalf("%s is classified %q; only the reasoning is Anvil's own", ptr, tr) + } + } + if em.Result.Properties.Advisory.Excerpt.Trust != record.TrustUntrusted { + t.Fatalf("the advisory excerpt is %q, not the row's own trust", + em.Result.Properties.Advisory.Excerpt.Trust) + } + // record's own check, which is the one that caught area B. + if err := record.ValidateResultTrust(&em.Result); err != nil { + t.Fatalf("ValidateResultTrust: %v", err) + } + } +} + +// TestReasoningCarriesNoByteFromOutsideAnvil is the empirical half of the +// anvil_generated claim. Every distinctive external string in the fixtures is +// searched for in the one field labelled Anvil's own. +func TestReasoningCarriesNoByteFromOutsideAnvil(t *testing.T) { + external := []string{ + fixtureHostPkg, fixtureRepoPkg, fixtureHostVer, fixtureRepoVer, + fixtureExcerpt, fixtureManualNote, fixtureManifest, + "CVE-2026-99999", "GHSA-zzzz-qqqq-xxxx", "USN-9999-1", + "ubuntu", "ghsa", "amd64", "LicenseRef-ubuntu-usn", "CC-BY-4.0", + "pkg:deb/ubuntu/", "pkg:maven/", + } + for _, em := range []Emission{ + mustEmit(t, hostMatch(), hostRow()), + mustEmit(t, repoMatch(), repoRow()), + } { + reasoning := em.Result.Properties.Reasoning + if reasoning == "" { + t.Fatal("empty reasoning labelled anvil_generated") + } + for _, s := range external { + if strings.Contains(reasoning, s) { + t.Fatalf("the anvil_generated reasoning contains the external string %q: %q", s, reasoning) + } + } + // And the message, which DOES interpolate external bytes, is + // covered by the untrusted default rather than being sanitised into + // a claim it is Anvil's. + if !strings.Contains(em.Result.Message.Text, em.Result.Properties.Advisory.IDs[0]) { + t.Error("the message does not name the advisory it rests on") + } + } +} + +// TestComposeReasoningRefusesAnExternalString drives the guard RED. A guard +// that has never failed has not been tested. +func TestComposeReasoningRefusesAnExternalString(t *testing.T) { + if _, err := composeReasoning([]string{reasonDeterministic, fixtureHostPkg}); err == nil { + t.Fatal("composeReasoning accepted a package name into an anvil_generated string") + } else { + var ref *Refusal + if !asRefusal(err, &ref) || ref.Reason != RefusalReasoningNotAnvilGenerated { + t.Fatalf("wrong refusal: %v", err) + } + } + if _, err := composeReasoning(nil); err == nil { + t.Fatal("composeReasoning accepted an empty explanation") + } + // The positive control: the vocabulary plus an integer is accepted. + got, err := composeReasoning([]string{reasonDeterministic, reasonStaleNum, "3600"}) + if err != nil { + t.Fatalf("composeReasoning refused its own vocabulary: %v", err) + } + if !strings.Contains(got, "3600") { + t.Fatalf("integer lost: %q", got) + } + // Near-misses that must still be refused. + for _, bad := range []string{"36 00", "+3600", "3600.0", "٣٦٠٠", reasonDeterministic + "!"} { + if _, err := composeReasoning([]string{bad}); err == nil { + t.Errorf("composeReasoning accepted %q", bad) + } + } +} + +// TestAnAdvisoryRowClaimingAnvilGeneratedIsRefused: the record contract pins +// anvil_generated as illegal for an external string, and every byte in the +// advisory table is external. +func TestAnAdvisoryRowClaimingAnvilGeneratedIsRefused(t *testing.T) { + a := hostRow() + a.Trust = record.TrustAnvilGenerated + _, err := emitter().Emit(hostMatch(), a) + if err == nil { + t.Fatal("an advisory row claiming anvil_generated was accepted") + } + var ref *Refusal + if !asRefusal(err, &ref) || ref.Reason != RefusalIllegalAdvisoryTrust { + t.Fatalf("wrong refusal: %v", err) + } +} + +// --------------------------------------------------------------------------- +// as_of / staleness_seconds +// --------------------------------------------------------------------------- + +// TestStaleAdvisorySurfacesItsStalenessRatherThanReportingClean is A.19's +// second named validation item, rebuilt after A.20 showed the original could +// not fail. +// +// THE ORIGINAL WAS CIRCULAR. It set `StalenessSeconds` on the input row and +// asserted the same number came out. That proves copy-through — and +// copy-through WAS the defect, because `advisory.staleness_seconds` is the +// publisher lag at write time while the record's field is assembly time minus +// as_of. During a feed outage neither cache column advances, so the copy +// reported three-week-old data as an hour old, inside its SLO, in the prose a +// human reads. The test could not see it: the outage case is exactly the case +// where the input row's number stays small. +// +// SO THE EXPECTED VALUE IS ARITHMETIC THIS TEST DOES ITSELF. The watermark and +// the assembly instant are both stated here, twenty-one days apart, and the row +// carries a SMALL frozen publisher lag — the outage shape. Nothing in the +// assertion is read back from the emitter. +func TestStaleAdvisorySurfacesItsStalenessRatherThanReportingClean(t *testing.T) { + const ( + slo = 86400 + // The interval the test asserts, computed here and nowhere else. + wantStaleness = 21 * 24 * 3600 + // What the outage froze in the cache: an hour, and wrong by a factor + // of 504. Emitting THIS is the blocker A.20 found. + frozenPublisherLag = 3600 + ) + watermark := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + assembledAt := watermark.Add(wantStaleness * time.Second) + + a := hostRow() + a.AsOf = watermark + a.FreshnessSLOSeconds = slo + a.StalenessSeconds = frozenPublisherLag + + e := Emitter{TargetID: fixtureTargetID, AssembledAt: assembledAt} + em, err := e.Emit(hostMatch(), a) + if err != nil { + t.Fatalf("Emit: %v", err) + } + + // 1. The finding EXISTS. Nothing was silently dropped. + if em.Result.Properties.FindingID == "" { + t.Fatal("a stale advisory produced no finding") + } + + // 2. as_of is the CACHE WATERMARK — not the assembly instant, which is + // the value a clock-reading implementation would have stamped here. + got := em.Result.Properties.Advisory.AsOf + if !got.Equal(watermark) { + t.Fatalf("as_of = %s, want the cache watermark %s", got, watermark) + } + if got.Equal(assembledAt) { + t.Fatalf("as_of = %s, which is the record-assembly instant; as_of states when the DATA "+ + "was current, and stamping assembly time on it reports stale data as fresh", got) + } + assertJSONString(t, em.Result, "properties/anvil~1advisory/asOf", watermark.Format(time.RFC3339)) + + // 3. staleness_seconds is THE CONTRACT'S INTERVAL: assembly minus as_of. + // Not the row's column, which says an hour. + if n := em.Result.Properties.Advisory.StalenessSeconds; n != wantStaleness { + t.Fatalf("stalenessSeconds = %d, want %d (assembly %s minus watermark %s). "+ + "%d is the row's frozen publisher-lag column, which is a different quantity: "+ + "emitting it reports twenty-one-day-old data as an hour old", + n, wantStaleness, assembledAt.Format(time.RFC3339), watermark.Format(time.RFC3339), + frozenPublisherLag) + } + assertJSONNumber(t, em.Result, "properties/anvil~1advisory/stalenessSeconds", float64(wantStaleness)) + + // 4. The SLO breach itself is machine-readable on the crossing artifact + // (the frozen record has no SLO slot — see the package header). Under + // the row's own number this would read "inside the SLO". + if !em.BeyondFreshnessSLO() { + t.Fatal("BeyondFreshnessSLO() = false for data three weeks past a one-day SLO") + } + assertJSONBool(t, em, "beyondFreshnessSlo", true) + assertJSONNumber(t, em, "freshnessSloSeconds", float64(slo)) + + // 5. And it is said in the reasoning, which is what a human reads — with + // the SAME number, so the prose and the field cannot drift apart. + reasoning := em.Result.Properties.Reasoning + if !strings.Contains(reasoning, reasonBeyondSLO) { + t.Errorf("the reasoning does not mention the SLO breach: %q", reasoning) + } + if !strings.Contains(reasoning, strconv.Itoa(wantStaleness)) { + t.Errorf("the reasoning does not state the emitted interval %d: %q", wantStaleness, reasoning) + } + if strings.Contains(reasoning, strconv.Itoa(frozenPublisherLag)) { + t.Errorf("the reasoning states the row's publisher-lag column %d, which is not the age "+ + "of this data: %q", frozenPublisherLag, reasoning) + } + + // 6. THE CONTROL, and it is CLOCK-FREE. A.20's second circular finding was + // that the old control depended on the wall-clock distance to a + // hard-coded fixture date, so it would have begun failing on a calendar + // day rather than on a code change. Here the same watermark one hour + // before assembly is inside the same SLO, on every day, forever. + freshAt := watermark.Add(time.Hour) + fresh, err := Emitter{TargetID: fixtureTargetID, AssembledAt: freshAt}.Emit(hostMatch(), a) + if err != nil { + t.Fatalf("Emit (fresh control): %v", err) + } + if n := fresh.Result.Properties.Advisory.StalenessSeconds; n != 3600 { + t.Fatalf("the control emitted stalenessSeconds = %d, want 3600", n) + } + if fresh.BeyondFreshnessSLO() { + t.Fatal("BeyondFreshnessSLO() = true for data one hour into a one-day SLO") + } + assertJSONBool(t, fresh, "beyondFreshnessSlo", false) + + // 7. Staleness does NOT suppress the finding or demote it. An outage is + // what serve_stale exists for; refusing every finding during one is + // the over-refusal that gets a control switched off. + if em.Result.Properties.Verdict != record.VerdictTruePositive { + t.Errorf("stale data demoted the verdict to %q", em.Result.Properties.Verdict) + } +} + +// TestStalenessIsMeasuredFromTheWatermarkNotCopiedFromTheRow walks a table of +// (watermark, assembly) pairs whose expected intervals are stated as arithmetic +// and whose rows all carry the SAME decoy publisher lag. +// +// One row of the table is enough to catch copy-through; the table is here so +// that the emitted number is shown to TRACK the interval rather than to match +// it once. A constant would satisfy a single-point test. +func TestStalenessIsMeasuredFromTheWatermarkNotCopiedFromTheRow(t *testing.T) { + watermark := time.Date(2026, 3, 14, 15, 9, 26, 0, time.UTC) + for _, c := range []struct { + name string + lag time.Duration + want int + }{ + {"zero: assembled at the watermark", 0, 0}, + {"one second", time.Second, 1}, + {"one hour", time.Hour, 3600}, + {"one day", 24 * time.Hour, 86400}, + {"three weeks", 21 * 24 * time.Hour, 1814400}, + {"sub-second truncates toward zero", 1500 * time.Millisecond, 1}, + } { + t.Run(c.name, func(t *testing.T) { + a := hostRow() + a.AsOf = watermark + a.StalenessSeconds = fixturePublisherLag // the decoy + em, err := Emitter{ + TargetID: fixtureTargetID, + AssembledAt: watermark.Add(c.lag), + }.Emit(hostMatch(), a) + if err != nil { + t.Fatalf("Emit: %v", err) + } + if n := em.Result.Properties.Advisory.StalenessSeconds; n != c.want { + t.Fatalf("stalenessSeconds = %d, want %d", n, c.want) + } + assertJSONNumber(t, em.Result, "properties/anvil~1advisory/stalenessSeconds", float64(c.want)) + }) + } +} + +// TestAnAssemblyInstantIsRequiredRatherThanDefaulted: the zero value is +// refused, because a zero default would measure every finding's age against the +// Unix epoch — and, worse, would be the silent no-op that reinstates the +// copy-through the whole field exists to remove. +func TestAnAssemblyInstantIsRequiredRatherThanDefaulted(t *testing.T) { + _, err := Emitter{TargetID: fixtureTargetID}.Emit(hostMatch(), hostRow()) + if err == nil { + t.Fatal("an emitter with no assembly instant emitted a record; staleness_seconds is " + + "defined as assembly time minus as_of and there was no assembly time") + } + var ref *Refusal + if !asRefusal(err, &ref) || ref.Reason != RefusalNoAssemblyTime { + t.Fatalf("wrong refusal: %v", err) + } + + // And an assembly instant BEFORE the watermark is refused rather than + // clamped to zero: "current as of now" is not a claim to make about data + // whose own timestamps say otherwise. + _, err = Emitter{ + TargetID: fixtureTargetID, + AssembledAt: fixtureWatermark.Add(-time.Second), + }.Emit(hostMatch(), hostRow()) + if err == nil { + t.Fatal("a negative age was emitted") + } + if !asRefusal(err, &ref) || ref.Reason != RefusalNegativeStaleness { + t.Fatalf("wrong refusal for a negative age: %v", err) + } +} + +// TestAnUnstatedSLOIsNotAMetSLO: zero means "the caller stated none", and the +// package answers false rather than inventing a default it could then claim a +// breach against. +func TestAnUnstatedSLOIsNotAMetSLO(t *testing.T) { + const wantStaleness = 99999999 + a := hostRow() + a.FreshnessSLOSeconds = 0 + em, err := Emitter{ + TargetID: fixtureTargetID, + AssembledAt: fixtureWatermark.Add(wantStaleness * time.Second), + }.Emit(hostMatch(), a) + if err != nil { + t.Fatalf("Emit: %v", err) + } + if em.BeyondFreshnessSLO() { + t.Fatal("a breach was asserted against an SLO nobody configured") + } + if n := em.Result.Properties.Advisory.StalenessSeconds; n != wantStaleness { + t.Fatalf("stalenessSeconds = %d, want %d; the staleness was lost when the SLO was unstated", + n, wantStaleness) + } +} + +// TestThisPackageReadsNoClock is structural, and it is the property that makes +// the assembly instant an INPUT rather than a reading. as_of comes from the +// cache watermark and staleness_seconds is measured against an instant the +// caller states; the only way to be sure neither is a clock reading is that +// there is no clock to read. +func TestThisPackageReadsNoClock(t *testing.T) { + src, err := os.ReadFile("emit.go") + if err != nil { + t.Fatalf("reading emit.go: %v", err) + } + code := stripComments(t, "emit.go", src) + for _, banned := range []string{ + "time.Now", "time.Since", "time.Until", "time.Tick", "time.After", + "func() time.Time", "rand.", "os.Getenv", + } { + if strings.Contains(code, banned) { + t.Errorf("emit.go references %s; as_of must come from the cache watermark, the "+ + "assembly instant must ARRIVE from the caller, and nothing here may depend on a "+ + "clock, a random source or the environment", banned) + } + } +} + +// TestThePackageImportsNothingItShouldNot pins the dependency graph, the way +// internal/match pins its own. A SQL driver or an HTTP client appearing in a +// pure field-population step is a change worth failing a build over. +func TestThePackageImportsNothingItShouldNot(t *testing.T) { + allowed := map[string]bool{ + "encoding/json": true, "fmt": true, "strconv": true, "strings": true, "time": true, + "github.com/Susquehanna-Syntax/Anvil/internal/match": true, + "github.com/Susquehanna-Syntax/Anvil/internal/record": true, + } + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "emit.go", nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("parsing emit.go: %v", err) + } + for _, imp := range f.Imports { + path := strings.Trim(imp.Path.Value, `"`) + if !allowed[path] { + t.Errorf("emit.go imports %q, which is not on the allowlist", path) + } + } +} + +// --------------------------------------------------------------------------- +// The fingerprint: one algorithm, and it is not this package's +// --------------------------------------------------------------------------- + +// corpusVector is the subset of a testdata/fingerprint_corpus fixture this +// test reads. +type corpusVector struct { + ID string `json:"id"` + Tier string `json:"tier"` + Input struct { + TargetID string `json:"target_id"` + AdvisoryID string `json:"advisory_id"` + Purl string `json:"purl"` + ManifestRelPath string `json:"manifest_rel_path"` + PackageManager string `json:"package_manager"` + HostIdentifier string `json:"host_identifier"` + } `json:"input"` + Mutations []struct { + Name string `json:"name"` + Input struct { + TargetID string `json:"target_id"` + AdvisoryID string `json:"advisory_id"` + Purl string `json:"purl"` + ManifestRelPath string `json:"manifest_rel_path"` + PackageManager string `json:"package_manager"` + HostIdentifier string `json:"host_identifier"` + } `json:"input"` + } `json:"mutations"` +} + +// inexpressibleMutations are corpus mutations this package CANNOT produce, +// with the reason. They are listed rather than skipped silently: a mutation +// that is neither exercised nor listed fails the test, so a corpus that grows +// cannot be quietly under-covered. +// +// Both entries are inexpressible because they vary a locator segment this +// package derives from a closed vocabulary rather than accepting from a +// caller — which is a stronger guarantee than the mutation asks for, not a +// weaker one. +var inexpressibleMutations = map[string]string{ + "package_manager_uppercased": "the manager segment comes from hostPackageManagers, " + + "which has no uppercase member", + "package_manager_and_identifier_padded": "both segments are composed from " + + "MatchResult fields, so no padding can enter", +} + +// TestEmittedFingerprintMatchesTheFrozenConformanceGoldens is the +// one-fingerprint check, and its expected values come from a file this +// package did not write and cannot influence. +func TestEmittedFingerprintMatchesTheFrozenConformanceGoldens(t *testing.T) { + dir := filepath.Join("..", "..", "..", "testdata", "fingerprint_corpus") + + for _, name := range []string{"host-01-openssl-debian", "sca-01-log4shell-maven"} { + t.Run(name, func(t *testing.T) { + var v corpusVector + raw, err := os.ReadFile(filepath.Join(dir, name+".json")) + if err != nil { + t.Fatalf("reading fixture: %v", err) + } + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatalf("parsing fixture: %v", err) + } + goldens := readGoldenDigests(t, filepath.Join(dir, name+".golden")) + + check := func(label, targetID, advisoryID, purl, manifest, manager, hostID string) { + t.Helper() + want, ok := goldens[label] + if !ok { + t.Fatalf("no golden digest labelled %q", label) + } + m, a := matchFromCorpus(t, v.Tier, advisoryID, purl, manifest, manager, hostID) + em, err := Emitter{TargetID: targetID, AssembledAt: fixtureAssembledAt}.Emit(m, a) + if err != nil { + t.Fatalf("%s: Emit: %v", label, err) + } + got := em.Result.PartialFingerprints[record.PartialFingerprintAnvilFindingID] + if got != want { + t.Fatalf("%s: emitted digest %s, golden %s "+ + "(a second fingerprint under one name breaks regression matching forever)", + label, got, want) + } + if em.Result.Properties.FindingID != want { + t.Fatalf("%s: findingId %s disagrees with the digest %s", + label, em.Result.Properties.FindingID, want) + } + } + + in := v.Input + check("base", in.TargetID, in.AdvisoryID, in.Purl, in.ManifestRelPath, + in.PackageManager, in.HostIdentifier) + + exercised := 0 + for _, mu := range v.Mutations { + if reason, listed := inexpressibleMutations[mu.Name]; listed { + t.Logf("mutation %q not exercised: %s", mu.Name, reason) + continue + } + check("mutation:"+mu.Name, mu.Input.TargetID, mu.Input.AdvisoryID, + mu.Input.Purl, mu.Input.ManifestRelPath, + mu.Input.PackageManager, mu.Input.HostIdentifier) + exercised++ + } + if exercised == 0 { + t.Fatal("no mutation was exercised; the corpus is being read but not used") + } + }) + } +} + +// matchFromCorpus builds the MatchResult and AdvisoryRow that a corpus vector +// describes. The host vector's `host_identifier` is split back into package +// and arch, which is the mapping this package performs in the other +// direction; the manager is NOT taken from the vector, because deriving it is +// the behaviour under test. +func matchFromCorpus(t *testing.T, tier, advisoryID, purl, manifest, manager, hostID string) (match.MatchResult, AdvisoryRow) { + t.Helper() + a := AdvisoryRow{ + Source: "corpus", + SourceID: advisoryID, + LicenseSPDX: "CC0-1.0", + Trust: record.TrustUntrusted, + AsOf: fixtureWatermark, + StalenessSeconds: 1, + } + m := match.MatchResult{ + Source: "corpus", + SourceID: advisoryID, + Purl: purl, + InstalledVersion: "0", + MatchedRange: "[0, 1)", + } + switch tier { + case "host": + if want := "apt"; manager != want { + t.Fatalf("the corpus vector's package_manager is %q; hostPackageManagers maps %q to %q, "+ + "and the two must agree or every Debian host finding forks identity", + manager, match.EcosystemDeb, want) + } + name, arch, _ := strings.Cut(hostID, ":") + m.Collector = match.CollectorHost + m.Ecosystem = match.EcosystemDeb + m.Package = name + m.Arch = arch + m.Detector = record.DetectorKindHost + m.EvidenceClass = record.EvidenceClassHost + case "sca": + m.Collector = match.CollectorRepoSCA + m.Ecosystem = "maven" + m.Package = "corpus-package" + m.ManifestRelPath = manifest + m.Detector = record.DetectorKindSCA + m.EvidenceClass = record.EvidenceClassSCA + default: + t.Fatalf("unexpected tier %q", tier) + } + return m, a +} + +// readGoldenDigests parses the TSV golden file: `digest label hex`. +func readGoldenDigests(t *testing.T, path string) map[string]string { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading golden: %v", err) + } + out := map[string]string{} + for _, line := range strings.Split(string(raw), "\n") { + line = strings.TrimRight(line, "\r") + if line == "" || strings.HasPrefix(line, "#") { + continue + } + parts := strings.Split(line, "\t") + if len(parts) != 3 || parts[0] != "digest" { + continue + } + out[parts[1]] = parts[2] + } + if len(out) == 0 { + t.Fatalf("%s carried no digests", path) + } + return out +} + +// TestThisPackageComputesNoDigestOfItsOwn is the structural half of the +// one-fingerprint rule: no hash import, no digest arithmetic, only delegation. +func TestThisPackageComputesNoDigestOfItsOwn(t *testing.T) { + src, err := os.ReadFile("emit.go") + if err != nil { + t.Fatalf("reading emit.go: %v", err) + } + code := stripComments(t, "emit.go", src) + for _, banned := range []string{"sha256", "sha1", "md5", "fnv", "crc32", "crypto/"} { + if strings.Contains(code, banned) { + t.Errorf("emit.go references %q; anvil-fp/v1 is defined once, in internal/record", banned) + } + } + for _, required := range []string{"record.Host(", "record.Sca("} { + if !strings.Contains(code, required) { + t.Errorf("emit.go does not call %s; the digest must be delegated", required) + } + } +} + +// --------------------------------------------------------------------------- +// The seven fields, over A.17's own output +// --------------------------------------------------------------------------- + +// TestEveryRecordFromTheComparatorCarriesTheSevenLaneAFields runs the REAL +// comparator over a real inventory, emits every result it produces, and +// asserts all seven Lane-A-owned fields on the SERIALISED bytes of each one. +// The findings are A.17's, not this test's construction. +func TestEveryRecordFromTheComparatorCarriesTheSevenLaneAFields(t *testing.T) { + ranges := []match.AffectedRange{ + { + Source: "ubuntu", SourceID: "USN-9999-1", CVEID: "CVE-2026-99999", + Ecosystem: match.EcosystemDeb, Package: fixtureHostPkg, + Purl: "pkg:deb/ubuntu/" + fixtureHostPkg, + Fixed: "9.9.9zzverqx-2", DistroBackport: true, + }, + { + Source: "alpine", SourceID: "CVE-2026-88888", CVEID: "CVE-2026-88888", + Ecosystem: match.EcosystemAPK, Package: "zzapkpkgqx", + Purl: "pkg:apk/alpine/zzapkpkgqx", Fixed: "2.0.0-r1", + }, + } + inventory := []match.PackageRecord{ + { + Collector: match.CollectorHost, Ecosystem: match.EcosystemDeb, + Name: fixtureHostPkg, Version: fixtureHostVer, Arch: "amd64", + }, + { + Collector: match.CollectorHost, Ecosystem: match.EcosystemAPK, + Name: "zzapkpkgqx", Version: "1.9.0-r0", + }, + } + + matcher, err := match.NewMatcher(match.NewStaticSource(ranges)) + if err != nil { + t.Fatalf("NewMatcher: %v", err) + } + findings, coverage, err := matcher.Match(context.Background(), inventory) + if err != nil { + t.Fatalf("Match: %v", err) + } + if err := coverage.AssertNotSilentlyClean(findings); err != nil { + t.Fatalf("the fixture run was silently clean: %v", err) + } + if len(findings) != 2 { + t.Fatalf("the comparator produced %d findings, want 2", len(findings)) + } + + rows := map[string]AdvisoryRow{ + "ubuntu/USN-9999-1": { + Source: "ubuntu", SourceID: "USN-9999-1", CVEID: "CVE-2026-99999", + FeedID: "ubuntu-usn", LicenseSPDX: "LicenseRef-ubuntu-usn", + LicenseManualNote: fixtureManualNote, Trust: record.TrustUntrusted, + AsOf: fixtureWatermark, StalenessSeconds: fixturePublisherLag, FreshnessSLOSeconds: 3600, + ParseDegraded: true, ExcerptText: fixtureExcerpt, + }, + "alpine/CVE-2026-88888": { + Source: "alpine", SourceID: "CVE-2026-88888", CVEID: "CVE-2026-88888", + FeedID: "alpine-secdb", LicenseSPDX: "MIT", Trust: record.TrustVerified, + TrustValidationStep: fixtureValidationStep, + AsOf: fixtureWatermark, StalenessSeconds: fixturePublisherLag, + FreshnessSLOSeconds: 3600, + }, + } + ems, err := emitter().EmitAll(findings, func(source, sourceID string) (AdvisoryRow, bool) { + r, ok := rows[source+"/"+sourceID] + return r, ok + }) + if err != nil { + t.Fatalf("EmitAll: %v", err) + } + if len(ems) != len(findings) { + t.Fatalf("emitted %d records for %d findings", len(ems), len(findings)) + } + + byFeed := map[string]AdvisoryRow{} + for _, r := range rows { + byFeed[r.FeedID] = r + } + + for i, em := range ems { + raw := marshal(t, em.Result) + src, ok := byFeed[em.Result.Properties.Advisory.SourceFeed] + if !ok { + t.Fatalf("result %d names feed %q, which no fixture row supplied", + i, em.Result.Properties.Advisory.SourceFeed) + } + + // 1. remediable_by_agent — present, and false: both findings are + // host findings. + assertJSONBool(t, em.Result, "properties/anvil~1remediableByAgent", false) + // 2. as_of, 3. staleness_seconds, 4. parse_degraded. + // The staleness is asserted against the interval this test set up — + // emitter()'s assembly instant minus the fixture watermark — and + // NOT against the row's own column, which carries the decoy. + assertJSONString(t, em.Result, "properties/anvil~1advisory/asOf", + fixtureWatermark.Format(time.RFC3339)) + assertJSONNumber(t, em.Result, "properties/anvil~1advisory/stalenessSeconds", + float64(fixtureAssemblyLagSeconds)) + if src.StalenessSeconds == fixtureAssemblyLagSeconds { + t.Errorf("result %d: the fixture row's publisher-lag column equals the interval under "+ + "test, so this assertion could pass by copy-through", i) + } + if _, err := jsonAt(raw, "properties/anvil~1advisory/parseDegraded"); err != nil { + t.Errorf("result %d: %v", i, err) + } + // 5. anvil/trust. + assertJSONString(t, em.Result, "properties/anvil~1trust/default", string(record.TrustUntrusted)) + // 6. license_spdx. + if _, err := jsonAt(raw, "properties/anvil~1advisory/licenseSpdx"); err != nil { + t.Errorf("result %d: %v", i, err) + } + // 7. license_manual_note, on the crossing artifact. + if src.LicenseManualNote != "" { + if em.LicenseManualNote == nil || em.LicenseManualNote.Text != src.LicenseManualNote { + t.Errorf("result %d lost the licence manual note", i) + } + if em.LicenseManualNote.Trust == record.TrustAnvilGenerated { + t.Errorf("result %d labels a quoted licence sentence as Anvil's own", i) + } + assertJSONString(t, em, "licenseManualNote/text", src.LicenseManualNote) + } else if em.LicenseManualNote != nil { + t.Errorf("result %d invented a licence manual note", i) + } + + // And the record's own validator agrees this is a legal result. + if err := record.ValidateResultTrust(&em.Result); err != nil { + t.Errorf("result %d: %v", i, err) + } + } + + // The whole assembled log validates: this is the record area's gate, not + // this package's. + log := sealedLog(t, Results(ems)) + if err := log.Validate(); err != nil { + t.Fatalf("the assembled record does not validate: %v", err) + } +} + +// --------------------------------------------------------------------------- +// Refusals +// --------------------------------------------------------------------------- + +// TestEveryRefusalReasonIsReachable drives all fifteen guards and fails if any +// one cannot be provoked. An unreachable refusal is a guard that is not there. +func TestEveryRefusalReasonIsReachable(t *testing.T) { + cases := map[RefusalReason]func() (Emitter, match.MatchResult, AdvisoryRow){ + RefusalNoTargetID: func() (Emitter, match.MatchResult, AdvisoryRow) { + return Emitter{}, hostMatch(), hostRow() + }, + RefusalUnrecognisedCollector: func() (Emitter, match.MatchResult, AdvisoryRow) { + m := hostMatch() + m.Collector = "container-image" + return emitter(), m, hostRow() + }, + RefusalInconsistentMatch: func() (Emitter, match.MatchResult, AdvisoryRow) { + m := hostMatch() + m.Detector = record.DetectorKindSCA + m.EvidenceClass = record.EvidenceClassSCA + return emitter(), m, hostRow() + }, + RefusalAdvisoryMismatch: func() (Emitter, match.MatchResult, AdvisoryRow) { + a := hostRow() + a.SourceID = "USN-0000-1" + return emitter(), hostMatch(), a + }, + RefusalNoAdvisoryID: func() (Emitter, match.MatchResult, AdvisoryRow) { + m, a := hostMatch(), hostRow() + m.CVEID, m.SourceID = "", "" + a.CVEID, a.SourceID = "", "" + return emitter(), m, a + }, + RefusalNoPurl: func() (Emitter, match.MatchResult, AdvisoryRow) { + m := hostMatch() + m.Purl = "" + return emitter(), m, hostRow() + }, + RefusalNoManifestPath: func() (Emitter, match.MatchResult, AdvisoryRow) { + m := repoMatch() + m.ManifestRelPath = "" + return emitter(), m, repoRow() + }, + RefusalUnsupportedHostEcosystem: func() (Emitter, match.MatchResult, AdvisoryRow) { + m := hostMatch() + m.Ecosystem = "gentoo" + return emitter(), m, hostRow() + }, + RefusalNoWatermark: func() (Emitter, match.MatchResult, AdvisoryRow) { + a := hostRow() + a.AsOf = time.Time{} + return emitter(), hostMatch(), a + }, + RefusalNoAssemblyTime: func() (Emitter, match.MatchResult, AdvisoryRow) { + return Emitter{TargetID: fixtureTargetID}, hostMatch(), hostRow() + }, + RefusalNegativeStaleness: func() (Emitter, match.MatchResult, AdvisoryRow) { + a := hostRow() + a.StalenessSeconds = -1 + return emitter(), hostMatch(), a + }, + RefusalTrustValidationStep: func() (Emitter, match.MatchResult, AdvisoryRow) { + a := hostRow() + a.Trust = record.TrustVerified // and no step named + return emitter(), hostMatch(), a + }, + RefusalExcerptTooLong: func() (Emitter, match.MatchResult, AdvisoryRow) { + a := hostRow() + a.ExcerptText = strings.Repeat("z", record.MaxAdvisoryExcerptBytes+1) + return emitter(), hostMatch(), a + }, + RefusalIllegalAdvisoryTrust: func() (Emitter, match.MatchResult, AdvisoryRow) { + a := hostRow() + a.Trust = record.TrustAnvilGenerated + return emitter(), hostMatch(), a + }, + RefusalNoLicenceDeclared: func() (Emitter, match.MatchResult, AdvisoryRow) { + a := hostRow() + a.LicenseSPDX, a.LicenseManualNote = "", " " + return emitter(), hostMatch(), a + }, + RefusalFingerprint: func() (Emitter, match.MatchResult, AdvisoryRow) { + m := hostMatch() + m.Purl = "not-a-purl" + return emitter(), m, hostRow() + }, + } + + for reason, build := range cases { + e, m, a := build() + _, err := e.Emit(m, a) + if err == nil { + t.Errorf("%s: Emit accepted the input", reason) + continue + } + var ref *Refusal + if !asRefusal(err, &ref) { + t.Errorf("%s: error is not a *Refusal: %v", reason, err) + continue + } + if ref.Reason != reason { + t.Errorf("wanted refusal %s, got %s (%v)", reason, ref.Reason, err) + } + if !ref.Reason.Valid() { + t.Errorf("%s is not in the closed set", ref.Reason) + } + } + + // The two reasons above are reached through composeReasoning and the + // contract validator rather than through an Emit input, so they are + // driven directly; every reason must be accounted for. + if _, err := composeReasoning([]string{"external"}); err == nil { + t.Error("RefusalReasoningNotAnvilGenerated is unreachable") + } + if err := validateEmitted(&record.Result{}); err == nil { + t.Error("validateEmitted accepted an empty result, so RefusalContractViolation is unreachable") + } + // And the trust arm of the same gate, which record's own validator is + // vacuous for on this shape — see validateEmitted's comment. + mislabelled := mustEmit(t, hostMatch(), hostRow()).Result + mislabelled.Properties.Trust.Default = record.TrustAnvilGenerated + if err := validateEmitted(&mislabelled); err == nil { + t.Error("validateEmitted accepted anvil_generated as the trust default") + } + stolen := mustEmit(t, hostMatch(), hostRow()).Result + stolen.Properties.Trust.Fields["/message/text"] = record.TrustAnvilGenerated + if err := validateEmitted(&stolen); err == nil { + t.Error("validateEmitted accepted anvil_generated on a string this step did not write") + } + + covered := map[RefusalReason]bool{ + RefusalReasoningNotAnvilGenerated: true, + RefusalContractViolation: true, + } + for r := range cases { + covered[r] = true + } + for _, r := range RefusalReasons() { + if !covered[r] { + t.Errorf("refusal reason %s is declared but never exercised", r) + } + } + if len(covered) != len(RefusalReasons()) { + t.Errorf("exercised %d reasons, declared %d", len(covered), len(RefusalReasons())) + } +} + +// TestEmitAllRefusesRatherThanSkipping: a match whose advisory row cannot be +// resolved must stop the run, because a finding dropped between the comparator +// and the record is indistinguishable from a clean scan. +func TestEmitAllRefusesRatherThanSkipping(t *testing.T) { + ms := []match.MatchResult{hostMatch(), repoMatch()} + got, err := emitter().EmitAll(ms, func(source, sourceID string) (AdvisoryRow, bool) { + if source == "ubuntu" { + return hostRow(), true + } + return AdvisoryRow{}, false + }) + if err == nil { + t.Fatalf("EmitAll silently dropped a finding and returned %d records", len(got)) + } + if got != nil { + t.Error("EmitAll returned records alongside its refusal") + } + if _, err := emitter().EmitAll(ms, nil); err == nil { + t.Error("EmitAll accepted a nil lookup") + } +} + +// --------------------------------------------------------------------------- +// Determinism +// --------------------------------------------------------------------------- + +// TestEmissionIsDeterministic: the same inputs produce byte-identical output. +// Lane A is zero-inference, and a record that varies run to run cannot be +// diffed, cached or trusted. +func TestEmissionIsDeterministic(t *testing.T) { + for _, pair := range []struct { + m match.MatchResult + a AdvisoryRow + }{{hostMatch(), hostRow()}, {repoMatch(), repoRow()}} { + first := marshal(t, mustEmit(t, pair.m, pair.a)) + for i := 0; i < 20; i++ { + if got := marshal(t, mustEmit(t, pair.m, pair.a)); !bytes.Equal(first, got) { + t.Fatalf("emission %d differs:\n%s\n%s", i, first, got) + } + } + } +} + +// --------------------------------------------------------------------------- +// Risk +// --------------------------------------------------------------------------- + +// TestCvssV4BaseIsSetOnlyForAV4Vector: writing a v3.1 score into a field named +// CvssV4Base puts a number in the record that is wrong in a way no consumer +// can detect. +func TestCvssV4BaseIsSetOnlyForAV4Vector(t *testing.T) { + score := 9.8 + v4 := repoRow() + v4.CVSSScore, v4.CVSSVector = &score, "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N" + if got := mustEmit(t, repoMatch(), v4).Result.Properties.Risk; got == nil || got.CvssV4Base == nil { + t.Fatal("a v4 vector did not populate CvssV4Base") + } else if *got.CvssV4Base != score { + t.Fatalf("CvssV4Base = %v, want %v", *got.CvssV4Base, score) + } + + v31 := repoRow() + v31.CVSSScore, v31.CVSSVector = &score, "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + em := mustEmit(t, repoMatch(), v31) + if em.Result.Properties.Risk != nil && em.Result.Properties.Risk.CvssV4Base != nil { + t.Fatal("a v3.1 score was written into CvssV4Base") + } + + // KEV alone still produces a Risk block; no risk inputs produce none. + kev := repoRow() + kev.KEVMember = true + if mustEmit(t, repoMatch(), kev).Result.Properties.Risk == nil { + t.Fatal("KEV membership was dropped") + } + if mustEmit(t, repoMatch(), repoRow()).Result.Properties.Risk != nil { + t.Fatal("a Risk block was invented for a row carrying no risk inputs") + } +} + +// --------------------------------------------------------------------------- +// The crossing bytes, for a Result this package did not build +// --------------------------------------------------------------------------- + +// TestAResultThatDidNotComeFromEmitIsClampedInTheCrossingBytes is A.20's +// finding 3, made true rather than deleted. +// +// Emission.RemediableByAgent's doc claims a Result arriving from anywhere other +// than Emit — hand-built, deserialised, or mutated after emission — cannot +// present a host finding as actionable. A.20 showed the clamp reached the +// top-level mirror only: the nested canonical record inside the SAME object, +// and Results(), both still said true. An object that disagrees with itself is +// not a guarantee, and this project's standard is that a claim which cannot be +// demonstrated is deleted rather than qualified. So the claim is now +// demonstrated, ON THE SERIALISED BYTES, for all three shapes and all three +// provenances. +func TestAResultThatDidNotComeFromEmitIsClampedInTheCrossingBytes(t *testing.T) { + // A Result that never went through Emit at all. + handBuilt := record.Result{ + RuleID: "CVE-2026-00000", + Message: record.Message{Text: "hand-built, never validated, asserting it is actionable"}, + Properties: record.ResultProperties{ + FindingID: "0000000000000000000000000000000000000000000000000000000000000000", + Half: record.HalfSast, + Confidence: 1, + Verdict: record.VerdictTruePositive, + RemediableByAgent: true, + Detector: record.DetectorRef{Kind: record.DetectorKindHost}, + EvidenceClass: record.EvidenceClassHost, + Trust: record.TrustAssertion{Default: record.TrustUntrusted}, + }, + } + + // The same host finding, emitted properly and then MUTATED afterwards. + mutated := mustEmit(t, hostMatch(), hostRow()) + mutated.Result.Properties.RemediableByAgent = true + + // And one that made the round trip through JSON before being mutated, + // which is the "deserialised" arm of the same sentence. + var deserialised record.Result + if err := json.Unmarshal(marshal(t, mustEmit(t, hostMatch(), hostRow()).Result), &deserialised); err != nil { + t.Fatalf("round trip: %v", err) + } + deserialised.Properties.RemediableByAgent = true + + for _, c := range []struct { + name string + em Emission + }{ + {"hand-built", Emission{Result: handBuilt}}, + {"mutated after emission", mutated}, + {"deserialised then mutated", Emission{Result: deserialised}}, + } { + t.Run(c.name, func(t *testing.T) { + if c.em.RemediableByAgent() { + t.Error("Emission.RemediableByAgent() = true") + } + // 1. the top-level mirror, + assertJSONBool(t, c.em, "remediableByAgent", false) + // 2. the nested canonical record in the same object — the half + // that used to disagree, + assertJSONBool(t, c.em, "result/properties/anvil~1remediableByAgent", false) + // 3. and Results(), which is what reaches the SARIF log and the + // store. + projected := Results([]Emission{c.em}) + if len(projected) != 1 { + t.Fatalf("Results() returned %d results", len(projected)) + } + if projected[0].Properties.RemediableByAgent { + t.Error("Results()[0] carries RemediableByAgent=true for a host finding") + } + assertJSONBool(t, projected[0], "properties/anvil~1remediableByAgent", false) + }) + } + + // THE CALLER'S OWN STRUCT IS NOT WRITTEN. The clamp is a copy, so a + // consumer cannot observe this package reaching into a value it was handed + // — and a test that passed because the input was quietly rewritten would + // be proving something else. + if !mutated.Result.Properties.RemediableByAgent { + t.Error("clamping rewrote the caller's own Result rather than a copy") + } + + // THE CONTROL: clamping refuses host findings, not everything. A repo + // dependency still crosses as actionable at both levels. + repo := mustEmit(t, repoMatch(), repoRow()) + assertJSONBool(t, repo, "remediableByAgent", true) + assertJSONBool(t, repo, "result/properties/anvil~1remediableByAgent", true) + assertJSONBool(t, Results([]Emission{repo})[0], "properties/anvil~1remediableByAgent", true) +} + +// --------------------------------------------------------------------------- +// The advisory excerpt, and the trust it claims +// --------------------------------------------------------------------------- + +// TestTheAdvisoryExcerptBoundIsCheckedRatherThanAssumed: A.19 asserted the +// excerpt was pre-trimmed by ingestion and copied it through unexamined, so a +// 24 KB excerpt reached the record and the store. The read path's cap trims +// only the agent-facing card, so "bounded downstream" was not bounded here. +// +// NOTE WHAT THIS DOES NOT CLAIM. Sanitisation is still not verified in this +// package and AdvisoryRow.ExcerptText now says so instead of asserting it: the +// invisible-character vocabulary lives in internal/ingest/invisible, which this +// package does not import and must not re-spell. +func TestTheAdvisoryExcerptBoundIsCheckedRatherThanAssumed(t *testing.T) { + a := hostRow() + a.ExcerptText = strings.Repeat("z", record.MaxAdvisoryExcerptBytes+1) + _, err := emitter().Emit(hostMatch(), a) + if err == nil { + t.Fatalf("an excerpt of %d bytes was emitted; the record's cap is %d", + len(a.ExcerptText), record.MaxAdvisoryExcerptBytes) + } + var ref *Refusal + if !asRefusal(err, &ref) || ref.Reason != RefusalExcerptTooLong { + t.Fatalf("wrong refusal: %v", err) + } + + // The boundary itself is accepted: a guard that refuses the legal maximum + // would be refusing conforming ingestion output. + atCap := hostRow() + atCap.ExcerptText = strings.Repeat("z", record.MaxAdvisoryExcerptBytes) + em := mustEmit(t, hostMatch(), atCap) + if em.Result.Properties.Advisory.Excerpt == nil { + t.Fatal("an excerpt exactly at the cap was dropped") + } + if got := len(em.Result.Properties.Advisory.Excerpt.Text); got != record.MaxAdvisoryExcerptBytes { + t.Fatalf("the emitted excerpt is %d bytes, want %d", got, record.MaxAdvisoryExcerptBytes) + } +} + +// TestVerifiedTrustMustNameItsValidationStep: record.Trust defines `verified` +// as bytes that "passed an explicit validation step that is named in the record +// ... Never a default". A.19 passed the level through with no step named +// anywhere, which publishes the label without the thing that gives it meaning. +// +// Not live today — internal/ingest/delta always binds `untrusted` — so this is +// the guard standing in front of the signature-checked snapshot path before it +// lands, rather than a fix for a live defect. +func TestVerifiedTrustMustNameItsValidationStep(t *testing.T) { + unnamed := hostRow() + unnamed.Trust = record.TrustVerified + _, err := emitter().Emit(hostMatch(), unnamed) + if err == nil { + t.Fatal("a row asserting verified with no validation step named was accepted") + } + var ref *Refusal + if !asRefusal(err, &ref) || ref.Reason != RefusalTrustValidationStep { + t.Fatalf("wrong refusal: %v", err) + } + + // The other direction: a step named beside untrusted bytes is a validation + // claim nothing validated, and is refused too. + laundered := hostRow() + laundered.TrustValidationStep = fixtureValidationStep + if _, err := emitter().Emit(hostMatch(), laundered); err == nil { + t.Error("a validation step was named on an untrusted row") + } else if !asRefusal(err, &ref) || ref.Reason != RefusalTrustValidationStep { + t.Errorf("wrong refusal: %v", err) + } + + // Whitespace is not a name. + blank := hostRow() + blank.Trust = record.TrustVerified + blank.TrustValidationStep = " " + if _, err := emitter().Emit(hostMatch(), blank); err == nil { + t.Error("a blank validation step satisfied the naming requirement") + } + + // AND THE NAME TRAVELS. Refusing an unnamed step would be pointless if the + // name were then dropped: the contract's requirement is that the step is + // NAMED, not merely that one existed. + named := hostRow() + named.Trust = record.TrustVerified + named.TrustValidationStep = fixtureValidationStep + em := mustEmit(t, hostMatch(), named) + if em.TrustValidationStep != fixtureValidationStep { + t.Fatalf("the validation step was lost: %q", em.TrustValidationStep) + } + assertJSONString(t, em, "trustValidationStep", fixtureValidationStep) + if em.Result.Properties.Advisory.Excerpt.Trust != record.TrustVerified { + t.Fatalf("the excerpt is %q, not the row's own trust", + em.Result.Properties.Advisory.Excerpt.Trust) + } + + // An untrusted row names nothing, and the key is absent from the bytes + // rather than present and empty. + plain := mustEmit(t, hostMatch(), hostRow()) + if plain.TrustValidationStep != "" { + t.Errorf("an untrusted row produced the validation step %q", plain.TrustValidationStep) + } + if _, err := jsonAt(marshal(t, plain), "trustValidationStep"); err == nil { + t.Error("an untrusted emission carries a trustValidationStep key") + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func asRefusal(err error, out **Refusal) bool { return errors.As(err, out) } + +func marshal(t *testing.T, v any) []byte { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return raw +} + +// jsonAt walks a '/'-separated path through the marshalled document, decoding +// '~1' as '/' per RFC 6901 so that `anvil/trust` can be addressed. +func jsonAt(raw []byte, path string) (any, error) { + var doc any + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, err + } + cur := doc + for _, seg := range strings.Split(path, "/") { + seg = strings.ReplaceAll(seg, "~1", "/") + obj, ok := cur.(map[string]any) + if !ok { + return nil, errAt(path, "not an object at "+seg) + } + v, present := obj[seg] + if !present { + return nil, errAt(path, "key "+seg+" is absent from the serialised bytes") + } + cur = v + } + return cur, nil +} + +type pathError struct{ path, msg string } + +func (e *pathError) Error() string { return e.path + ": " + e.msg } + +func errAt(path, msg string) error { return &pathError{path, msg} } + +func assertJSONBool(t *testing.T, v any, path string, want bool) { + t.Helper() + got, err := jsonAt(marshal(t, v), path) + if err != nil { + t.Fatalf("%v", err) + } + b, ok := got.(bool) + if !ok { + t.Fatalf("%s is %T, want bool", path, got) + } + if b != want { + t.Fatalf("%s = %t in the serialised bytes, want %t", path, b, want) + } +} + +func assertJSONString(t *testing.T, v any, path, want string) { + t.Helper() + got, err := jsonAt(marshal(t, v), path) + if err != nil { + t.Fatalf("%v", err) + } + s, ok := got.(string) + if !ok { + t.Fatalf("%s is %T, want string", path, got) + } + if s != want { + t.Fatalf("%s = %q in the serialised bytes, want %q", path, s, want) + } +} + +func assertJSONNumber(t *testing.T, v any, path string, want float64) { + t.Helper() + got, err := jsonAt(marshal(t, v), path) + if err != nil { + t.Fatalf("%v", err) + } + n, ok := got.(float64) + if !ok { + t.Fatalf("%s is %T, want number", path, got) + } + if n != want { + t.Fatalf("%s = %v in the serialised bytes, want %v", path, n, want) + } +} + +// stripComments returns the file's code with every comment removed, so that a +// banned token quoted inside a doc comment does not fail a structural test. +func stripComments(t *testing.T, name string, src []byte) string { + t.Helper() + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, name, src, parser.ParseComments) + if err != nil { + t.Fatalf("parsing %s: %v", name, err) + } + out := make([]byte, len(src)) + copy(out, src) + base := fset.File(f.Pos()).Base() + for _, group := range f.Comments { + for i := int(group.Pos()) - base; i < int(group.End())-base && i < len(out); i++ { + if out[i] != '\n' { + out[i] = ' ' + } + } + } + return string(out) +} + +// sealedLog wraps results in a minimal sealed SAST-half record, so the read +// path and the contract validator can be exercised over what this package +// emits. The envelope belongs to the scan controller (O.2); this is a stand-in +// for a test and nothing more. +func sealedLog(t *testing.T, results []record.Result) record.SARIFLog { + t.Helper() + created := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + sealed := created.Add(time.Hour) + const auditID = "audit-lanea-test" + sort.SliceStable(results, func(i, j int) bool { + return results[i].Properties.FindingID < results[j].Properties.FindingID + }) + return record.SARIFLog{ + Schema: record.SARIFSchemaURI, + Version: record.SARIFVersion, + Properties: record.AuditProperties{ + SchemaVersion: record.SchemaVersion, + AuditID: auditID, + State: record.StateBothSealed, + Version: 1, + CreatedAt: created, + Target: record.Target{ + RepoURL: "https://example.invalid/zzrepoqx.git", + Ref: "refs/heads/main", + Commit: "0000000000000000000000000000000000000000", + Provenance: record.TargetProvenanceNoTargetDeclared, + Provisioning: record.TargetProvisioningEphemeralManifest, + }, + DastStatus: record.DastStatusNotRun, + Deadline: record.Deadline{ + DeadlineAt: created.Add(record.DefaultClaimTimeoutSeconds * time.Second), + ClaimTimeoutSeconds: record.DefaultClaimTimeoutSeconds, + }, + }, + Runs: []record.Run{{ + Tool: record.Tool{Driver: record.ToolComponent{Name: "anvil-lane-a"}}, + AutomationDetails: record.RunAutomationDetails{ID: auditID, CorrelationGUID: auditID}, + Results: results, + Properties: record.RunProperties{ + Half: record.HalfSast, + Status: record.HalfStatusSealed, + SealedAt: &sealed, + }, + }}, + } +} diff --git a/test/conformance/lanea/fixtures/README.md b/test/conformance/lanea/fixtures/README.md new file mode 100644 index 0000000..e19e4b3 --- /dev/null +++ b/test/conformance/lanea/fixtures/README.md @@ -0,0 +1,37 @@ +# Lane A conformance corpus (A.21) + +Every file in this directory is **hand-written to the publisher's documented +shape**. None of it was produced by running any part of Anvil. + +That rule is not decoration. A test whose corpus comes from the implementation +asserts that the implementation still does what it did the day the corpus was +captured — which is a change-detector, not a test. Recorded output also +presupposes a successful run, and this project already has one live example of +what that hides: the Trivy E2E job found that the SCA collector cannot run at +all without a pre-seeded database, because every prior test used a recorded +report. + +## Provenance, file by file + +| File | Shape transcribed from | What it is here for | +|---|---|---| +| `feeds.yaml` | `internal/ingest/config`'s documented schema and `feeds.example.yaml` | The harness's own feed table. Exit criterion 1 forbids feed facts in Go, and a harness that built its rows in Go would be exempting itself from the rule it checks. `${BASE}` is substituted with the in-process test server's origin, because the port is assigned by the kernel. | +| `cvelistv5/CVE-2026-1001.json` | CVE Record Format 5.1 | The chain's main advisory, and the **injection corpus**: its description carries U+200B, U+202E, U+200D and an HTML comment holding an instruction addressed to a model. `anvilinjectionprobe` is the token the harness searches for to prove the comment did not reach the index. | +| `cvelistv5/CVE-2026-1002.json` | CVE Record Format, `dataVersion: 5.9` | A record schema the parser has never seen. Exit criterion 23: persisted with `parse_degraded = 1`, never dropped. | +| `cvelistv5/CVE-2026-1003.json` | CVE Record Format, `state: REJECTED` | Exit criterion 22: tombstoned, kept, and removed from the search index. This is the record that exposed the A.8/A.14 index divergence. | +| `osv/GHSA-deb-2026-0001.json` | OSV schema 1.6, Debian ecosystem | Carries the publisher's own ecosystem spelling, `Debian:11`. It is here to demonstrate a gap — SEAM 1 in `lanea_test.go` — not to pass a check. | +| `kev/known_exploited_vulnerabilities.json` | CISA KEV catalogue | Two entries, and the feed whose registry metadata says `NOASSERTION` over a CC0 body (exit criterion 10). | +| `alpine/v3.19-main.json` | Alpine `secdb` branch file | The one publisher in this corpus whose decoder writes the cache's own short ecosystem vocabulary (`apk`), so it is the one that reaches the comparator. Produces the harness's single end-to-end finding. | +| `redhat/RHSA-2026-0001.json` | Red Hat CSAF/VEX 2.0 | A backported fix as an RPM NEVRA. Its `fixed` product id is what exposed the arch-suffix defect in `SplitNEVRA`. | +| `trivy/report.json` | Trivy JSON report, `SchemaVersion: 2`, `Class: lang-pkgs` | Exercises `repo.ParseReport` with no binary present. `lang-pkgs` and not `os-pkgs` because the collector deliberately skips os-pkgs as the host collector's territory — which is exactly why the repository half and the comparator have disjoint domains (SEAM 3). | +| `inventory/corpus-probe-packages.json` | `internal/collector/host`'s `Package` JSON tags | Host-shaped packages chosen to intersect this corpus. **It is NOT a recording of `internal/collector/host` and is never evidence about it** — the chain runs `host.Collect` for real wherever a package manager exists and cites that for the host-inventory link. These rows stay because no real inventory can ever intersect a synthetic corpus. **Note what is missing: there is no purl field, because `host.Package` has none.** That absence is SEAM 2. | +| `licence/cc0-1.0-excerpt.txt` | The CC0 1.0 Universal deed | The publisher licence body the gate is given. Its digest is pinned in a manifest the harness generates at run time, because a pin written by hand is a number nobody checked. | +| `licence/tier0-notes.md` | `mirror/tier0/LICENSE-NOTES.md`'s shape | Anvil's own record. It can only make the gate's conclusion stricter; it never admits a feed on its own. | + +## Identifiers + +Every identifier is synthetic and in a range no publisher uses: +`CVE-2026-1001`…`CVE-2026-1006`, `GHSA-deb1-2026-0001`, `RHSA-2026:0001`. +Every host named anywhere is under `.invalid`. Nothing in this directory +resolves, and the harness makes no network call: its only listener is an +in-process `httptest` TLS server. diff --git a/test/conformance/lanea/fixtures/alpine/v3.19-main.json b/test/conformance/lanea/fixtures/alpine/v3.19-main.json new file mode 100644 index 0000000..9150fda --- /dev/null +++ b/test/conformance/lanea/fixtures/alpine/v3.19-main.json @@ -0,0 +1,32 @@ +{ + "apkurl": "{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-{{pkg.ver}}.apk", + "archs": [ + "x86_64", + "aarch64" + ], + "distroversion": "v3.19", + "packages": [ + { + "pkg": { + "name": "openssl", + "secfixes": { + "3.1.4-r5": [ + "CVE-2026-1001" + ] + } + } + }, + { + "pkg": { + "name": "busybox", + "secfixes": { + "1.36.1-r16": [ + "CVE-2026-1005" + ] + } + } + } + ], + "reponame": "main", + "urlprefix": "https://example.invalid/alpine" +} diff --git a/test/conformance/lanea/fixtures/cvelistv5/CVE-2026-1001.json b/test/conformance/lanea/fixtures/cvelistv5/CVE-2026-1001.json new file mode 100644 index 0000000..01f9794 --- /dev/null +++ b/test/conformance/lanea/fixtures/cvelistv5/CVE-2026-1001.json @@ -0,0 +1,49 @@ +{ + "containers": { + "cna": { + "affected": [ + { + "packageName": "openssl", + "product": "openssl", + "vendor": "openssl", + "versions": [ + { + "lessThan": "3.1.4", + "status": "affected", + "version": "3.1.0", + "versionType": "semver" + } + ] + } + ], + "descriptions": [ + { + "lang": "en", + "value": "openssl before 3.1.4-r5 mishandles a synthetic condition.​ Reported by ‮trebor‍." + } + ], + "metrics": [ + { + "cvssV3_1": { + "baseScore": 9.8, + "baseSeverity": "CRITICAL", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + } + } + ], + "references": [ + { + "url": "https://example.invalid/advisory/CVE-2026-1001" + } + ] + } + }, + "cveMetadata": { + "cveId": "CVE-2026-1001", + "datePublished": "2026-01-14T00:00:00Z", + "dateUpdated": "2026-02-02T00:00:00Z", + "state": "PUBLISHED" + }, + "dataType": "CVE_RECORD", + "dataVersion": "5.1" +} diff --git a/test/conformance/lanea/fixtures/cvelistv5/CVE-2026-1002.json b/test/conformance/lanea/fixtures/cvelistv5/CVE-2026-1002.json new file mode 100644 index 0000000..14ceb02 --- /dev/null +++ b/test/conformance/lanea/fixtures/cvelistv5/CVE-2026-1002.json @@ -0,0 +1,39 @@ +{ + "containers": { + "cna": { + "affected": [ + { + "packageName": "widget", + "product": "widget", + "vendor": "example", + "versions": [ + { + "lessThan": "1.2.3", + "status": "affected", + "version": "1.0.0" + } + ] + } + ], + "descriptions": [ + { + "lang": "en", + "value": "A synthetic advisory published in a record schema this parser has never seen." + } + ], + "references": [ + { + "url": "https://example.invalid/advisory/CVE-2026-1002" + } + ] + } + }, + "cveMetadata": { + "cveId": "CVE-2026-1002", + "datePublished": "2026-01-20T00:00:00Z", + "dateUpdated": "2026-02-03T00:00:00Z", + "state": "PUBLISHED" + }, + "dataType": "CVE_RECORD", + "dataVersion": "5.9" +} diff --git a/test/conformance/lanea/fixtures/cvelistv5/CVE-2026-1003.json b/test/conformance/lanea/fixtures/cvelistv5/CVE-2026-1003.json new file mode 100644 index 0000000..987b245 --- /dev/null +++ b/test/conformance/lanea/fixtures/cvelistv5/CVE-2026-1003.json @@ -0,0 +1,21 @@ +{ + "containers": { + "cna": { + "descriptions": [ + { + "lang": "en", + "value": "This synthetic identifier was withdrawn by its CNA as a duplicate." + } + ] + } + }, + "cveMetadata": { + "cveId": "CVE-2026-1003", + "datePublished": "2026-01-21T00:00:00Z", + "dateRejected": "2026-02-04T00:00:00Z", + "dateUpdated": "2026-02-04T00:00:00Z", + "state": "REJECTED" + }, + "dataType": "CVE_RECORD", + "dataVersion": "5.1" +} diff --git a/test/conformance/lanea/fixtures/feeds.yaml b/test/conformance/lanea/fixtures/feeds.yaml new file mode 100644 index 0000000..beaa41b --- /dev/null +++ b/test/conformance/lanea/fixtures/feeds.yaml @@ -0,0 +1,94 @@ +# Anvil Lane A conformance corpus — the feed table this harness runs against. +# +# IT IS A YAML FILE AND NOT A GO SLICE, deliberately. Lane A exit criterion 1 +# says no feed URL and no cadence may exist as a Go literal, and a harness that +# built its own feed rows in Go would be exempting itself from the rule it is +# there to check. Every row below is parsed by internal/ingest/config, so the +# loader is inside the chain under test rather than beside it. +# +# ${BASE} is substituted with the httptest server's origin at run time. It is +# the ONE thing the harness supplies, because the port is assigned by the +# kernel and cannot be written down in advance. Nothing else about a feed — +# cadence, tier, licence, auth, bootstrap mechanism — is decided in Go. +# +# Every row is synthetic. No host here resolves and no request leaves the +# process: the server is an in-process httptest TLS listener. + +version: 1 + +feeds: + # The CVE 5.x baseline: three records, one of them in a dataVersion the + # parser has never seen and one REJECTED, bootstrapped from a bulk archive. + - id: cvelistv5-fixture + url: ${BASE}/cvelistv5/all.zip + enabled: true + auth_mode: none + sync_mechanism: conditional_get_etag + interval_seconds: 900 + reconcile_interval_seconds: 86400 + baseline_interval_seconds: 604800 + freshness_slo_seconds: 21600 + on_failure: serve_stale + license_tier: 0 + license_spdx: CC0-1.0 + bootstrap_mechanism: bulk_archive + + # Alpine secdb: the one publisher in this corpus whose decoder writes the + # cache's own short ecosystem vocabulary ("apk"), which is why it is the one + # that reaches the comparator. See the harness's ECOSYSTEM VOCABULARY note. + - id: alpine-secdb-fixture + url: ${BASE}/alpine/all.zip + enabled: true + auth_mode: none + sync_mechanism: conditional_get_last_modified + interval_seconds: 86400 + freshness_slo_seconds: 172800 + on_failure: serve_stale + license_tier: 0 + license_spdx: CC0-1.0 + bootstrap_mechanism: bulk_archive + + # Red Hat CSAF/VEX: a backported fix, written into `affected` with + # distro_backport = 1 and the ecosystem "rpm". + - id: redhat-csaf-fixture + url: ${BASE}/redhat/all.zip + enabled: true + auth_mode: none + sync_mechanism: conditional_get_etag + interval_seconds: 86400 + freshness_slo_seconds: 172800 + on_failure: serve_stale + license_tier: 0 + license_spdx: CC0-1.0 + bootstrap_mechanism: bulk_archive + + # CISA KEV. Its forge metadata reports NOASSERTION over a CC0 body, which is + # exit criterion 10: the LICENCE FILE BODY decides and the metadata does not, + # and the disagreement makes spine S8's manual note mandatory. + - id: cisa-kev-fixture + url: ${BASE}/kev/known_exploited_vulnerabilities.json + enabled: true + auth_mode: none + sync_mechanism: conditional_get_etag + interval_seconds: 900 + freshness_slo_seconds: 7200 + on_failure: serve_stale + license_tier: 0 + license_spdx: CC0-1.0 + license_manual_note: "The GitHub API reports NOASSERTION for this repository; the checked-in README states the catalog is public domain under CC0 1.0. Trust the LICENSE/README text, not the API metadata (research/01 S16/S17)." + bootstrap_mechanism: incremental_api + + # A Debian OSV export, carried in the publisher's own ecosystem spelling + # ("Debian:11"). It is in this corpus to demonstrate a gap, not to pass a + # check: see the harness's ECOSYSTEM VOCABULARY note. + - id: osv-debian-fixture + url: ${BASE}/osv/GHSA-deb-2026-0001.json + enabled: true + auth_mode: none + sync_mechanism: conditional_get_last_modified + interval_seconds: 86400 + freshness_slo_seconds: 172800 + on_failure: serve_stale + license_tier: 0 + license_spdx: CC0-1.0 + bootstrap_mechanism: incremental_api diff --git a/test/conformance/lanea/fixtures/inventory/corpus-probe-packages.json b/test/conformance/lanea/fixtures/inventory/corpus-probe-packages.json new file mode 100644 index 0000000..a217d46 --- /dev/null +++ b/test/conformance/lanea/fixtures/inventory/corpus-probe-packages.json @@ -0,0 +1,36 @@ +{ + "_comment": [ + "CORPUS PROBE PACKAGES. Host-shaped packages chosen to intersect the", + "synthetic advisory corpus in this directory: openssl 3.1.3-r0 (apk) and", + "busybox 1.36.1-r16 (apk) against the Alpine secdb fixture, and", + "python3-requests 2.25.1-3.el9 (rpm) sitting exactly ON the Red Hat CSAF", + "fixture's fixed version so the backport false-positive class is asserted.", + "The deb openssl row is SEAM 1's probe: it cannot be consulted at all.", + "", + "THIS FILE IS NOT A RECORDING OF internal/collector/host AND IS NEVER", + "EVIDENCE ABOUT IT. It used to be called host-packages.json and it used to", + "stand in for host.Collect's output; the harness's own tripwire fired on CI", + "the moment a real package manager appeared, because a hand-written file is", + "the worst available evidence for a collector that can be run. The chain now", + "runs host.Collect wherever a package manager exists and cites THAT for the", + "host-inventory link. See hostInventoryForChain.", + "", + "What a real inventory cannot do is intersect a synthetic corpus: no machine", + "has openssl 3.1.3-r0 with CVE-2026-1001 against it, and none ever will. So", + "these four rows stay, under a name that says what they are, and the", + "comparator and emitter links are proven over them on every host.", + "", + "The keys are host.Package's JSON tags (ecosystem/package/version/arch), so", + "a change to that struct breaks this file rather than silently ignoring it.", + "", + "NOTE WHAT IS MISSING: there is no purl field, because host.Package has no", + "purl field. That is not an omission in this fixture; it is the seam the", + "harness reports." + ], + "packages": [ + {"ecosystem": "apk", "package": "openssl", "version": "3.1.3-r0"}, + {"ecosystem": "apk", "package": "busybox", "version": "1.36.1-r16"}, + {"ecosystem": "rpm", "package": "python3-requests", "version": "2.25.1-3.el9", "arch": "noarch"}, + {"ecosystem": "deb", "package": "openssl", "version": "1.1.1n-0+deb11u3", "arch": "amd64"} + ] +} diff --git a/test/conformance/lanea/fixtures/kev/known_exploited_vulnerabilities.json b/test/conformance/lanea/fixtures/kev/known_exploited_vulnerabilities.json new file mode 100644 index 0000000..4d18f82 --- /dev/null +++ b/test/conformance/lanea/fixtures/kev/known_exploited_vulnerabilities.json @@ -0,0 +1,38 @@ +{ + "catalogVersion": "2026.02.05", + "count": 2, + "dateReleased": "2026-02-05T12:00:00.0000Z", + "title": "Synthetic Known Exploited Vulnerabilities Catalog", + "vulnerabilities": [ + { + "cveID": "CVE-2026-1001", + "cwes": [ + "CWE-787" + ], + "dateAdded": "2026-02-05", + "dueDate": "2026-02-26", + "knownRansomwareCampaignUse": "Unknown", + "notes": "https://example.invalid/kev/CVE-2026-1001", + "product": "OpenSSL", + "requiredAction": "Apply mitigations per vendor instructions.", + "shortDescription": "A synthetic entry used only by Anvil's conformance corpus.", + "vendorProject": "OpenSSL", + "vulnerabilityName": "OpenSSL Synthetic Remote Code Execution Vulnerability" + }, + { + "cveID": "CVE-2026-1004", + "cwes": [ + "CWE-200" + ], + "dateAdded": "2026-02-05", + "dueDate": "2026-02-26", + "knownRansomwareCampaignUse": "Known", + "notes": "https://example.invalid/kev/CVE-2026-1004", + "product": "python3-requests", + "requiredAction": "Apply updates per vendor instructions.", + "shortDescription": "A second synthetic entry.", + "vendorProject": "Example", + "vulnerabilityName": "Example Synthetic Header Leak Vulnerability" + } + ] +} diff --git a/test/conformance/lanea/fixtures/licence/cc0-1.0-excerpt.txt b/test/conformance/lanea/fixtures/licence/cc0-1.0-excerpt.txt new file mode 100644 index 0000000..64de601 --- /dev/null +++ b/test/conformance/lanea/fixtures/licence/cc0-1.0-excerpt.txt @@ -0,0 +1,6 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + +The person who associated a work with this deed has dedicated the work to the +public domain by waiving all rights to the work worldwide under copyright law. diff --git a/test/conformance/lanea/fixtures/licence/tier0-notes.md b/test/conformance/lanea/fixtures/licence/tier0-notes.md new file mode 100644 index 0000000..d6032ca --- /dev/null +++ b/test/conformance/lanea/fixtures/licence/tier0-notes.md @@ -0,0 +1,5 @@ +# Anvil's own record for the conformance corpus + +Every block below is Anvil's note about a synthetic feed. A note can only make +the gate's conclusion STRICTER; it can never admit a feed on its own. The +publisher's verbatim text is the evidence, and it lives beside this file. diff --git a/test/conformance/lanea/fixtures/osv/GHSA-deb-2026-0001.json b/test/conformance/lanea/fixtures/osv/GHSA-deb-2026-0001.json new file mode 100644 index 0000000..13b44f2 --- /dev/null +++ b/test/conformance/lanea/fixtures/osv/GHSA-deb-2026-0001.json @@ -0,0 +1,45 @@ +{ + "affected": [ + { + "package": { + "ecosystem": "Debian:11", + "name": "openssl", + "purl": "pkg:deb/debian/openssl" + }, + "ranges": [ + { + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.1.1n-0+deb11u4" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "aliases": [ + "CVE-2026-1001" + ], + "details": "The Debian security team tracks CVE-2026-1001 for the openssl source package.", + "id": "GHSA-deb1-2026-0001", + "modified": "2026-02-02T00:00:00Z", + "published": "2026-01-15T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://example.invalid/dsa/CVE-2026-1001" + } + ], + "schema_version": "1.6.0", + "severity": [ + { + "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "type": "CVSS_V3" + } + ], + "summary": "openssl is vulnerable in Debian 11" +} diff --git a/test/conformance/lanea/fixtures/redhat/RHSA-2026-0001.json b/test/conformance/lanea/fixtures/redhat/RHSA-2026-0001.json new file mode 100644 index 0000000..7f90ecd --- /dev/null +++ b/test/conformance/lanea/fixtures/redhat/RHSA-2026-0001.json @@ -0,0 +1,44 @@ +{ + "document": { + "category": "csaf_vex", + "csaf_version": "2.0", + "notes": [ + { + "category": "description", + "text": "A synthetic backported fix: the upstream version is not moved, only the release." + } + ], + "references": [ + { + "url": "https://example.invalid/errata/RHSA-2026:0001" + } + ], + "title": "Red Hat Security Advisory: python3-requests security update", + "tracking": { + "current_release_date": "2026-02-02T00:00:00Z", + "id": "RHSA-2026:0001", + "initial_release_date": "2026-01-30T00:00:00Z", + "status": "final" + } + }, + "vulnerabilities": [ + { + "cve": "CVE-2026-1004", + "product_status": { + "fixed": [ + "Red Hat Enterprise Linux 9:python3-requests-0:2.25.1-3.el9.noarch" + ] + }, + "scores": [ + { + "cvss_v3": { + "baseScore": 5.9, + "baseSeverity": "MEDIUM", + "vectorString": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:N/A:N" + } + } + ], + "title": "python3-requests: synthetic header leak" + } + ] +} diff --git a/test/conformance/lanea/fixtures/trivy/report.json b/test/conformance/lanea/fixtures/trivy/report.json new file mode 100644 index 0000000..cabb830 --- /dev/null +++ b/test/conformance/lanea/fixtures/trivy/report.json @@ -0,0 +1,37 @@ +{ + "SchemaVersion": 2, + "CreatedAt": "2026-02-06T00:00:00Z", + "ArtifactName": "containers/api", + "ArtifactType": "filesystem", + "Results": [ + { + "Target": "services/api/package-lock.json", + "Class": "lang-pkgs", + "Type": "npm", + "Vulnerabilities": [ + { + "VulnerabilityID": "CVE-2026-1006", + "PkgName": "left-pad", + "PkgIdentifier": { + "PURL": "pkg:npm/left-pad@1.3.0" + }, + "InstalledVersion": "1.3.0", + "FixedVersion": "1.3.1", + "Status": "fixed", + "Severity": "HIGH", + "Title": "left-pad: synthetic padding overflow", + "Description": "A synthetic advisory used only by Anvil's Lane A conformance corpus.", + "PrimaryURL": "https://example.invalid/advisory/CVE-2026-1006", + "References": ["https://example.invalid/advisory/CVE-2026-1006"], + "PublishedDate": "2026-01-30T00:00:00Z", + "LastModifiedDate": "2026-02-02T00:00:00Z", + "DataSource": { + "ID": "ghsa", + "Name": "GitHub Security Advisory npm", + "URL": "https://example.invalid/ghsa" + } + } + ] + } + ] +} diff --git a/test/conformance/lanea/lanea_test.go b/test/conformance/lanea/lanea_test.go new file mode 100644 index 0000000..5a7315f --- /dev/null +++ b/test/conformance/lanea/lanea_test.go @@ -0,0 +1,2207 @@ +// Package lanea_test is Lane A's end-to-end conformance harness (A.21). +// +// =========================================================================== +// WHAT THIS HARNESS IS FOR, AND WHY IT IS NOT SEVEN UNIT TESTS IN A TRENCHCOAT +// =========================================================================== +// +// Every package in Lane A has its own tests and they pass. This harness exists +// because a chain of internally-consistent links is not a working chain: the +// value of end-to-end is catching what unit tests structurally cannot, which is +// a DISAGREEMENT BETWEEN two components that each behave exactly as their own +// fixtures say they should. +// +// This project has a live example. The Trivy E2E job found that the SCA +// collector CANNOT RUN AT ALL without a pre-seeded database, because every +// prior test used recorded output — and recorded output presupposes a +// successful run. This harness found three more of the same class; they are +// named in THE OPEN SEAMS below and each one is asserted here rather than +// described. +// +// =========================================================================== +// THE REPORT IS THE DELIVERABLE, AND IT NAMES WHAT IT COULD NOT PROVE +// =========================================================================== +// +// A harness that quietly tests six of seven links is worse than one that +// fails, because the report is what gets believed. So this file does not +// return a pass/fail over "the chain": it returns a LEDGER with one entry per +// link, each either PROVEN (naming the assertion that proved it) or UNPROVEN +// (naming what is missing and what would settle it). TestLaneAChain fails if +// any link is left undecided, if a PROVEN entry carries no evidence, or if an +// UNPROVEN entry carries no settlement condition. +// +// AN UNPROVEN LINK DOES NOT FAIL THE RUN, BUT A STALE REASON DOES. Every +// "cannot be proven here" claim is re-checked against the machine on every +// run: if trivy appears on PATH, if a package manager appears, if a licence +// body is acquired into mirror/, the harness FAILS and tells you to promote +// the link to a real proof. A reason on file is a claim about today, and this +// project has already paid for three claims that were corrected and stayed +// false. +// +// =========================================================================== +// THE OPEN SEAMS THIS HARNESS FOUND +// =========================================================================== +// +// All three are the same defect wearing three hats: FOUR COMPONENTS EACH +// DEFINE THEIR OWN PACKAGE-IDENTITY VOCABULARY AND NOBODY OWNS THE MAPPING. +// None is visible from inside any one package, because each package's fixtures +// speak its own dialect. +// +// SEAM 1 — ECOSYSTEM VOCABULARY, INGESTION SIDE. +// internal/ingest/decode writes the publisher's own ecosystem spelling into +// `affected.ecosystem`: "Debian:11", "Alpine:v3.19", "PyPI", "Red Hat". +// internal/match's ecosystemAllowlist is exact-match over {deb, rpm, apk} and +// its comment says "the ingestion layer owns normalisation into this +// vocabulary". The ingestion layer does not normalise. It is HALF normalised, +// which is worse than neither: the Alpine secdb and Red Hat CSAF decoders +// write "apk" and "rpm" and do reach the comparator, so a spot check on +// either says the chain works, while every OSV- and CVE-5.x-sourced range — +// which is most of the corpus — is silently unreachable. +// PROVEN BY: TestLaneAChain/comparator, which ingests a real-shaped Debian +// OSV export and finds it cannot be consulted for a deb host package. +// +// SEAM 2 — NO PURL ON HOST PACKAGES. +// internal/collector/host's Package is {ecosystem, package, version, arch} +// and carries no purl. internal/match passes PackageRecord.Purl through +// untouched. internal/record/lanea refuses any emission with no purl +// (RefusalNoPurl) and says so deliberately: "this package will not +// synthesise one — the namespace (debian vs ubuntu, redhat vs fedora) comes +// from os-release". The host collector HAS os-release and does not build one. +// So no host finding can be emitted today. A.19's own tests do not see it +// because their host fixtures supply a purl. +// PROVEN BY: TestLaneAChain/emission, which runs the real host inventory +// shape through and captures the refusal. +// +// SEAM 3 — ECOSYSTEM VOCABULARY, REPO SIDE. +// internal/collector/repo's Finding.Ecosystem is Trivy's `Type` ("redhat", +// "npm", "gomod"). Same allowlist, same mismatch, same silence. +// PROVEN BY: TestLaneAChain/comparator. +// +// Where the harness has to cross one of these seams to exercise the links +// BEHIND it, it does so through bridgeEcosystem / bridgePurl — named, +// commented, TEST-ONLY functions. Their existence is the finding. They are not +// a fix and must not be copied into internal/. +// +// =========================================================================== +// THE CORPUS +// =========================================================================== +// +// Every fixture under fixtures/ is hand-written to the publisher's documented +// shape and holds only synthetic identifiers (CVE-2026-1001..1005, RHSA-2026: +// 0001, GHSA-deb1-2026-0001). NONE of it was produced by running any part of +// Anvil: a test whose corpus comes from the implementation is not a test. The +// feed table itself is fixtures/feeds.yaml, parsed by internal/ingest/config, +// so exit criterion 1's "no feed fact in Go" holds for the harness too. +// +// No network. The only listener is an in-process httptest TLS server and the +// only hosts named anywhere are .invalid. +// +// =========================================================================== +// THE ONE INPUT THAT IS NOT A FIXTURE: THE HOST INVENTORY +// =========================================================================== +// +// internal/collector/host is the only Lane A component this harness can run +// FOR REAL, on the machine the test is running on, and it does so whenever it +// can. The rule is in hostInventoryForChain and it is one sentence: +// +// host.Collect's REAL OUTPUT whenever a supported package manager is present +// on this machine, and the corpus probe alone when none is. +// +// Both branches run the full chain. What differs is what the ledger may claim: +// on a Linux CI host the host-inventory link is PROVEN and names the collected +// package count, and on a developer's Windows machine it stays UNPROVEN and +// says so. THE LEDGER LINE NAMES WHICH — a run that proved the collector and a +// run that proved only a file must not render the same, because telling those +// two apart is the entire point of that link. +// +// This replaced a tripwire that FAILED THE RUN when a package manager appeared +// ("the fixture inventory is no longer the best available evidence"). It fired +// on CI, correctly, and the fix is to honour it rather than to quieten it. The +// tripwires that remain point the other way: a present package manager that +// enumerates nothing, or real packages collected and then not submitted, both +// fail — those are the shapes in which the better evidence gets acquired and +// then dropped on the floor. +package lanea_test + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "database/sql" + "embed" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "testing" + "testing/fstest" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/collector/host" + "github.com/Susquehanna-Syntax/Anvil/internal/collector/repo" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/bootstrap" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/config" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/delta" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/license" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/sanitize" + "github.com/Susquehanna-Syntax/Anvil/internal/match" + "github.com/Susquehanna-Syntax/Anvil/internal/record" + "github.com/Susquehanna-Syntax/Anvil/internal/record/lanea" +) + +//go:embed fixtures +var fixtures embed.FS + +// fixtureClock is the one instant this harness knows. Every as_of, staleness +// and detected_at derives from it, which is what makes two consecutive runs +// byte-identical rather than merely similar. +var fixtureClock = time.Date(2026, 2, 6, 12, 0, 0, 0, time.UTC) + +// repoRoot is this file's directory relative to the module root, used by the +// static scans to walk internal/. It is derived, never written down. +const repoRoot = "../../.." + +// --------------------------------------------------------------------------- +// The chain-link ledger +// --------------------------------------------------------------------------- + +// linkID names one link of the chain the packet asks this harness to prove. +type linkID string + +const ( + linkFeedTable linkID = "feed table (internal/ingest/config)" + linkLicenceGate linkID = "licence gate (internal/ingest/license)" + linkSanitize linkID = "sanitize (internal/ingest/sanitize)" + linkCache linkID = "cache write (bootstrap + delta -> internal/ingest/cache)" + linkComparator linkID = "comparator (internal/match)" + linkEmission linkID = "emission (internal/record/lanea)" + linkHostCollect linkID = "host inventory (internal/collector/host)" + linkRepoCollect linkID = "repo SCA scan (internal/collector/repo)" + linkDeterminism linkID = "two-run determinism over the whole chain" + linkSelfHeal linkID = "weekly self-heal (internal/ingest/reconcile)" + linkAccelerator linkID = "DB accelerator (internal/mirror/accelerator)" + linkProductionUp linkID = "a production caller wiring the chain together" +) + +// chainLinks is the roll call. A link absent from the ledger at the end of the +// run is a FAILURE, not an omission: the whole point of this file is that the +// report cannot quietly shorten the chain. +var chainLinks = []linkID{ + linkFeedTable, linkLicenceGate, linkSanitize, linkCache, + linkComparator, linkEmission, linkHostCollect, linkRepoCollect, + linkDeterminism, linkSelfHeal, linkAccelerator, linkProductionUp, +} + +// verdict is one link's entry. +type verdict struct { + // proven is true only when this run actually exercised the link. + proven bool + // evidence names the assertion that proved it. Required when proven. + evidence string + // missing says what was not available. Required when not proven. + missing string + // settles says what would turn this into a proof. Required when not + // proven: "we could not check it" without "here is what would check it" + // is an excuse, not a report. + settles string +} + +type ledger struct{ v map[linkID]verdict } + +func newLedger() *ledger { return &ledger{v: map[linkID]verdict{}} } + +func (l *ledger) proven(id linkID, evidence string) { + l.v[id] = verdict{proven: true, evidence: evidence} +} + +func (l *ledger) unproven(id linkID, missing, settles string) { + l.v[id] = verdict{missing: missing, settles: settles} +} + +// checkLedger is the ledger's own validator, written as a PURE FUNCTION over +// the map so that its negative control can call the shipping code instead of a +// re-implementation of it. See TestTheLedgerRefusesAnUnsupportedClaim. +func checkLedger(l *ledger) []string { + var problems []string + for _, id := range chainLinks { + v, ok := l.v[id] + switch { + case !ok: + problems = append(problems, fmt.Sprintf( + "link %q was never decided. A harness that returns without deciding a link has "+ + "reported a pass over a shortened chain, which is the failure this file exists "+ + "to prevent.", id)) + case v.proven && strings.TrimSpace(v.evidence) == "": + problems = append(problems, fmt.Sprintf( + "link %q is claimed PROVEN with no evidence. A claim that cannot be demonstrated is "+ + "deleted, not qualified.", id)) + case !v.proven && strings.TrimSpace(v.missing) == "": + problems = append(problems, fmt.Sprintf( + "link %q is UNPROVEN with no statement of what is missing", id)) + case !v.proven && strings.TrimSpace(v.settles) == "": + problems = append(problems, fmt.Sprintf( + "link %q is UNPROVEN with no settlement condition. \"We could not check it\" without "+ + "\"here is what would check it\" is an excuse, not a report.", id)) + } + } + return problems +} + +func (l *ledger) report(t *testing.T) { + t.Helper() + for _, p := range checkLedger(l) { + t.Error(p) + } + provenN := 0 + var b strings.Builder + b.WriteString("\n=== LANE A CHAIN LEDGER ===\n") + for _, id := range chainLinks { + v := l.v[id] + if v.proven { + provenN++ + fmt.Fprintf(&b, " PROVEN %s\n %s\n", id, v.evidence) + continue + } + fmt.Fprintf(&b, " UNPROVEN %s\n missing: %s\n settles: %s\n", + id, v.missing, v.settles) + } + fmt.Fprintf(&b, "\n %d of %d links proven end to end in this run.\n", provenN, len(chainLinks)) + b.WriteString(" An UNPROVEN link is not a passing link. Read the list above before quoting\n" + + " this test as evidence that Lane A works.\n") + t.Log(b.String()) +} + +// --------------------------------------------------------------------------- +// The chain +// --------------------------------------------------------------------------- + +// chainOutput is everything one run of the chain produced, in a form two runs +// can be compared byte for byte. +type chainOutput struct { + advisory []string + affected []string + alias []string + fts []string + feedState []string + + // hostSourceLabel and hostRows are the host-side INPUT, carried into the + // two-run comparison rather than only the output. + // + // WHY THE INPUT IS COMPARED TOO. On a machine with a package manager the + // host packages come from host.Collect, which reads a live package + // database, and none of those packages matches this corpus's synthetic + // advisories — so a difference between two collections would produce no + // difference in any of the fields below and the determinism claim would + // be true about a chain that never saw the data. Comparing the rows makes + // the collector's own output part of what "byte-identical across two + // runs" asserts. + hostSourceLabel string + hostRows []string + + matches []match.MatchResult + coverage match.CoverageReport + emissionJSON []byte + + sanitizer map[string]int + decisions map[string]license.Decision + feeds config.FeedSet +} + +// TestLaneAChain is the harness. It runs the whole chain twice and decides +// every link. +func TestLaneAChain(t *testing.T) { + led := newLedger() + + first := runChain(t, led) + second := runChain(t, newLedger()) // a second, independent run; its ledger is discarded + + t.Run("determinism", func(t *testing.T) { + diffs := diffChains(first, second) + if len(diffs) > 0 { + for _, d := range diffs { + t.Errorf("two consecutive runs over the same fixed corpus differ: %s", d) + } + led.unproven(linkDeterminism, + "the two runs differed; see the failures above", + "fix the non-determinism, then re-run: the diff is the evidence") + return + } + led.proven(linkDeterminism, fmt.Sprintf( + "two consecutive runs produced byte-identical output across %d advisory rows, "+ + "%d affected rows, %d cve_alias rows, %d advisory_fts rows, %d feed_state rows, "+ + "%d match results and %d bytes of emitted records (zero deltas). The INPUT is "+ + "compared too: %d host package row(s) from %s were identical across the two "+ + "independent collections. Read that source label before weighing it — on a host "+ + "with a package manager it says host.Collect read a live package database twice "+ + "and got the same bytes; where it says \"corpus probe ONLY\" it says a file was "+ + "read twice, and proves correspondingly less", + len(first.advisory), len(first.affected), len(first.alias), len(first.fts), + len(first.feedState), len(first.matches), len(first.emissionJSON), + len(first.hostRows), first.hostSourceLabel)) + }) + + // The links this run could not exercise, each re-checked against the + // machine so a stale reason fails rather than persists. The host link is + // NOT here: it is decided inside runChain, where its evidence is produced. + decideRepoCollector(t, led) + decideNotWired(t, led) + + led.report(t) +} + +// runChain runs feed table -> licence gate -> sanitize -> cache -> comparator +// -> record emission once, deciding each link as it goes. +func runChain(t *testing.T, led *ledger) chainOutput { + t.Helper() + ctx := context.Background() + out := chainOutput{decisions: map[string]license.Decision{}, sanitizer: map[string]int{}} + + // --- LINK 1: the feed table --------------------------------------- + srv := fixtureServer(t) + feeds := loadFeeds(t, srv.URL) + out.feeds = feeds + if len(feeds.Feeds) != 5 { + t.Fatalf("the fixture feed table parsed to %d rows, want 5", len(feeds.Feeds)) + } + for _, f := range feeds.Feeds { + if f.IntervalSeconds <= 0 || f.FreshnessSLOSeconds < f.IntervalSeconds { + t.Errorf("feed %q: cadence %ds / SLO %ds came out of the loader unusable", + f.ID, f.IntervalSeconds, f.FreshnessSLOSeconds) + } + } + led.proven(linkFeedTable, fmt.Sprintf( + "internal/ingest/config.Parse loaded all %d rows of fixtures/feeds.yaml — every URL, cadence, "+ + "tier and licence in this run came from that file and none from Go", len(feeds.Feeds))) + + // --- LINK 2: the licence gate -------------------------------------- + mirror := admittingMirror(t, feeds) + for _, f := range feeds.Feeds { + d, err := license.Resolve(license.FromFeed(f, metadataSPDX(f.ID), mirror)) + if err != nil { + t.Fatalf("feed %q: the licence gate refused a pinned, acquired body: %v", f.ID, err) + } + if d.Refused() { + t.Fatalf("feed %q: refused decision with no error", f.ID) + } + out.decisions[f.ID] = d + } + kev := out.decisions["cisa-kev-fixture"] + if !kev.MetadataOverridden { + t.Error("exit criterion 10: the KEV row declares CC0-1.0 while its registry metadata says " + + "NOASSERTION, and the gate did not record that the body overrode the metadata") + } + if !kev.NoteRequired || strings.TrimSpace(kev.ManualNote) == "" { + t.Error("exit criterion 10: a metadata disagreement did not make spine S8's manual note mandatory") + } + led.proven(linkLicenceGate, fmt.Sprintf( + "license.Resolve admitted all %d rows against a PINNED, digest-matched licence body "+ + "(SYNTHETIC — see TestTheLicenceGateAdmitsNothingInThisWorkingTree, which proves the "+ + "real mirror/ tree admits nothing), and recorded the KEV metadata override with a "+ + "mandatory manual note", len(out.decisions))) + + // --- LINKS 3+4: sanitize, then the two writers into the cache ------- + db := openCache(t) + + var bootstrapped int + for _, f := range feeds.Feeds { + if f.BootstrapMechanism != config.BootstrapBulkArchive { + continue + } + b := &bootstrap.Bootstrapper{ + DB: db, + Mirror: mirror, + WorkDir: t.TempDir(), + HTTP: srv.Client(), + Clock: func() time.Time { return fixtureClock }, + Lookup: func(string) (string, bool) { return "", false }, + } + res, err := b.Bootstrap(ctx, f) + if err != nil { + t.Fatalf("bootstrap %q: %v (refused: %s)", f.ID, err, res.RefusedBecause) + } + if res.RecordsUpserted == 0 { + t.Fatalf("bootstrap %q imported nothing", f.ID) + } + bootstrapped += res.RecordsUpserted + mergeCounts(out.sanitizer, res.Sanitizer.Counts()) + } + + var deltaUpserts int + for _, f := range feeds.Feeds { + if f.BootstrapMechanism != config.BootstrapIncrementalAPI { + continue + } + body := readFixture(t, fixturePathFor(f.ID)) + recs, st, err := delta.Decode(f.ID, body) + if err != nil { + t.Fatalf("delta decode %q: %v", f.ID, err) + } + mergeCounts(out.sanitizer, st.Counts()) + bs, err := delta.Apply(ctx, db, f, out.decisions[f.ID], recs, fixtureClock, 0) + if err != nil { + t.Fatalf("delta apply %q: %v", f.ID, err) + } + deltaUpserts += bs.Upserts + } + + // The sanitizer must have found something, or the injection corpus never + // reached it and this link would be passing vacuously. + if total := sumCounts(out.sanitizer); total == 0 { + t.Fatal("the sanitizer removed nothing across the whole corpus, but the CVE-2026-1001 " + + "fixture carries a zero-width space, a bidi override, a zero-width joiner and an HTML " + + "comment. Either the corpus stopped carrying them or the sanitizer stopped running; " + + "a zero count here is not a clean corpus, it is a broken harness.") + } + assertStoredStringsAreSanitized(t, db) + assertRawJSONIsVerbatim(t, db) + led.proven(linkSanitize, fmt.Sprintf( + "the injection corpus reached the cache through both writers; sanitize removed %d "+ + "characters across %d categories, every queryable string in `advisory` passes "+ + "sanitize.AssertSanitized, and raw_json still holds the publisher's bytes verbatim "+ + "(proving the sanitiser ran on FIELDS and not on the stored document)", + sumCounts(out.sanitizer), len(out.sanitizer))) + + assertCacheInvariants(t, db) + led.proven(linkCache, fmt.Sprintf( + "A.8 upserted %d records and A.14 upserted %d into one migrated FTS5 cache; every advisory "+ + "row carries a licence declaration, the REJECTED record is tombstoned rather than "+ + "deleted and left the FTS index, and the unknown dataVersion is persisted with "+ + "parse_degraded=1", bootstrapped, deltaUpserts)) + + out.advisory = dump(t, db, advisoryDumpSQL) + out.affected = dump(t, db, affectedDumpSQL) + out.alias = dump(t, db, aliasDumpSQL) + out.fts = dump(t, db, ftsDumpSQL) + out.feedState = dump(t, db, feedStateDumpSQL) + + // --- LINK 5: the comparator ---------------------------------------- + // The host side is real collector output wherever a package manager + // exists and the corpus probe where none does; see hostInventoryForChain + // for the rule and decideHostLink for what each case entitles the ledger + // to claim. + hostSrc := hostInventoryForChain(t) + out.hostSourceLabel = hostSrc.label() + out.hostRows = hostSrc.rows() + + scan := repoScan(t) + // Mapped SEPARATELY so decideHostLink can assert that every REAL package + // reached the comparator. A single combined count could not tell a + // dropped collector package from a probe package standing in its place. + collectedRecords := hostPackageRecords(hostSrc.collected) + probeRecords := hostPackageRecords(hostSrc.probe) + hostRecords := append(collectedRecords, probeRecords...) + records := append(hostRecords, repoPackageRecords(scan)...) + + m, err := match.NewMatcher(&cacheSource{db: db}) + if err != nil { + t.Fatalf("match.NewMatcher: %v", err) + } + results, cov, err := m.Match(ctx, records) + if err != nil { + t.Fatalf("match: %v", err) + } + out.matches, out.coverage = results, cov + + assertComparatorSeams(t, db, cov, results) + decideHostLink(t, led, hostSrc, len(collectedRecords), results) + led.proven(linkComparator, fmt.Sprintf( + "the comparator ran over %d collector-shaped packages against the cache's own `affected` "+ + "rows and produced %d findings with a populated CoverageReport (Complete=%v, "+ + "RangesConsidered=%d, PackagesWithNoAdvisoryData=%d). SEAM 1 and SEAM 3 are asserted "+ + "here: the Debian OSV export landed with ecosystem %q and could not be consulted for a "+ + "deb host package, and the repository collector's %q findings were refused as an "+ + "unimplemented scheme (%d refused, ecosystems %v). Complete is FALSE because of that "+ + "refusal, which is the report doing its job. HOST INPUT: %s — read that before "+ + "quoting the submitted count, because the findings below come from the corpus probe "+ + "and NOT from any real package this machine has installed", + cov.PackagesSubmitted, len(results), cov.Complete, cov.RangesConsidered, + cov.PackagesWithNoAdvisoryData, "Debian:11", "npm", + cov.PackagesRefusedScheme, cov.EcosystemsRefused, hostSrc.label())) + + // --- LINK 6: record emission --------------------------------------- + emissions := emitAll(t, db, out.feeds, results, led) + blob, err := json.MarshalIndent(emissions, "", " ") + if err != nil { + t.Fatalf("marshalling emissions: %v", err) + } + out.emissionJSON = blob + + return out +} + +// --------------------------------------------------------------------------- +// Emission, and the seam in front of it +// --------------------------------------------------------------------------- + +func emitAll(t *testing.T, db *sql.DB, feeds config.FeedSet, results []match.MatchResult, led *ledger) []lanea.Emission { + t.Helper() + if len(results) == 0 { + t.Fatal("the comparator produced no findings at all, so emission cannot be exercised; " + + "the corpus is supposed to produce one host finding from the Alpine secdb ranges") + } + lookup := advisoryLookup(t, db, feeds) + e := lanea.Emitter{ + TargetID: "anvil-conformance-target", + // REQUIRED, and deliberately not defaultable. A.20 found that + // staleness_seconds was being copied from the cache column (the + // PUBLISHER lag) rather than computed as the contract quantity, + // record-assembly time minus AsOf -- so a twenty-one-day-old cache + // reported as one hour old and inside its SLO. Emit now refuses a + // zero AssembledAt instead of defaulting it, because a zero default + // silently reinstates exactly that bug. + // + // fixtureClock is this harness's single instant, so the cache as_of + // and this assembly time derive from the same constant and the run + // stays byte-identical across invocations. + AssembledAt: fixtureClock, + } + + // SEAM 2, asserted rather than described: the host rows as the host + // collector actually reports them carry no purl, and the emitter refuses. + var hostAsCollected []match.MatchResult + for _, r := range results { + if r.Collector != cache.CollectorHost { + continue + } + bare := r + bare.Purl = "" // what host.Package actually supplies: nothing + hostAsCollected = append(hostAsCollected, bare) + } + if len(hostAsCollected) == 0 { + t.Fatal("no host finding to test the purl seam with") + } + if _, err := e.EmitAll(hostAsCollected, lookup); err == nil { + t.Error("SEAM 2 HAS CLOSED: a host match with no purl was emitted. internal/collector/host " + + "now supplies one, or internal/record/lanea stopped requiring one. Delete bridgePurl, " + + "emit the real inventory, and promote the host link to PROVEN.") + } else { + var ref *lanea.Refusal + if !errors.As(err, &ref) || ref.Reason != lanea.RefusalNoPurl { + t.Errorf("the host emission failed for an unexpected reason: %v", err) + } + } + + emissions, err := e.EmitAll(results, lookup) + if err != nil { + t.Fatalf("emitting the bridged results: %v", err) + } + if len(emissions) != len(results) { + t.Fatalf("%d matches produced %d emissions", len(results), len(emissions)) + } + + // Exit criterion 21, end to end. + hosts := 0 + for i, em := range emissions { + if em.Result.Properties.Detector.Kind != record.DetectorKindHost { + t.Errorf("emission %d is not host-sourced, but every finding this corpus can produce "+ + "through a collector is: see the repo-SCA note in the ledger", i) + continue + } + hosts++ + if em.RemediableByAgent() { + t.Errorf("emission %d is host-sourced and carries remediable_by_agent=true", i) + } + } + if hosts == 0 { + t.Error("no host-sourced record was emitted, so exit criterion 21 passed vacuously") + } + + led.proven(linkEmission, fmt.Sprintf( + "%d canonical record(s) emitted through internal/record/lanea from the comparator's output "+ + "and the cache's own advisory rows, all host-sourced and all remediable_by_agent=false. "+ + "SEAM 2 is asserted here — the same matches WITHOUT the harness's bridgePurl are "+ + "refused with RefusalNoPurl, which is what the real inventory produces. NOTE WHAT THIS "+ + "DOES NOT SAY: exit criterion 21 holds here over a set that contains ONLY host records, "+ + "because no collector output can currently produce a non-host one. "+ + "TestTheRemediablePathIsReachableAtAll is the positive control that keeps the "+ + "assertion from being true only because nothing is.", len(emissions))) + return emissions +} + +// TestTheRemediablePathIsReachableAtAll is the positive control for exit +// criterion 21. +// +// "No host record is remediable" is a weak claim if NO record is ever +// remediable, and today none produced by a collector is: internal/collector/repo +// reports only lang-pkgs findings (it skips os-pkgs as A.9's territory), and +// internal/match implements deb, rpm and apk only. The two halves of the SCA +// path have disjoint domains, so no repository finding survives the comparator. +// +// The input below is therefore a PROBE and is labelled one: a repo-SCA package +// record in an ecosystem the comparator implements. No collector emits this +// shape today. The comparator still computes the verdict, so what is proven is +// real — remediable_by_agent CAN be true, and the difference between the two +// collectors is what decides it. +func TestTheRemediablePathIsReachableAtAll(t *testing.T) { + ctx := context.Background() + src := match.NewStaticSource([]match.AffectedRange{{ + Source: "redhat-csaf-fixture", SourceID: "RHSA-2026:0001", CVEID: "CVE-2026-1004", + Ecosystem: match.EcosystemRPM, Package: "python3-requests", + Introduced: "0", Fixed: "2.25.1-3.el9", DistroBackport: true, + }}) + m, err := match.NewMatcher(src) + if err != nil { + t.Fatal(err) + } + probe := match.PackageRecord{ + Collector: cache.CollectorRepoSCA, + Ecosystem: match.EcosystemRPM, + Name: "python3-requests", + Version: "2.25.1-2.el9", + Purl: "pkg:rpm/redhat/python3-requests@2.25.1-2.el9", + ManifestRelPath: "containers/api/rootfs.manifest", + } + hostSame := probe + hostSame.Collector = cache.CollectorHost + hostSame.ManifestRelPath = "" + + results, _, err := m.Match(ctx, []match.PackageRecord{probe, hostSame}) + if err != nil { + t.Fatalf("match: %v", err) + } + if len(results) != 2 { + t.Fatalf("the probe produced %d findings, want 2 (one per collector)", len(results)) + } + var sawRemediable, sawHost bool + for _, r := range results { + switch r.Collector { + case cache.CollectorRepoSCA: + if !r.RemediableByAgent { + t.Error("a repository dependency with a named fixed version is not remediable; the " + + "flag is then false everywhere and \"host findings are never remediable\" is " + + "vacuous") + } + sawRemediable = r.RemediableByAgent + case cache.CollectorHost: + if r.RemediableByAgent { + t.Error("the identical package under the host collector is remediable") + } + sawHost = true + } + } + if !sawRemediable || !sawHost { + t.Fatalf("the probe did not exercise both collectors: remediable=%v host=%v", sawRemediable, sawHost) + } +} + +// advisoryLookup reads the `advisory` row a match was decided on. +// +// IT IS TEST-ONLY AND ITS EXISTENCE IS A FINDING. internal/record/lanea's +// AdvisoryRow mirrors the cache's columns one for one and documents that "the +// caller reads the row and fills this in" — and no caller exists. See +// linkProductionUp. +func advisoryLookup(t *testing.T, db *sql.DB, feeds config.FeedSet) func(string, string) (lanea.AdvisoryRow, bool) { + t.Helper() + slo := map[string]int{} + for _, f := range feeds.Feeds { + slo[f.ID] = f.FreshnessSLOSeconds + } + return func(source, sourceID string) (lanea.AdvisoryRow, bool) { + const q = ` +SELECT ifnull(cve_id,''), ifnull(license_spdx,''), ifnull(license_manual_note,''), anvil_trust, + as_of, staleness_seconds, parse_degraded, ifnull(data_version,''), + ifnull(cvss_vector,''), cvss_score, epss_score, ifnull(epss_as_of,''), kev +FROM advisory WHERE source = ? AND source_id = ?` + var ( + cveID, spdx, note, trust, asOf, dataVersion, vector, epssAsOf string + staleness, degraded, kev int + cvss, epss sql.NullFloat64 + ) + err := db.QueryRow(q, source, sourceID).Scan(&cveID, &spdx, ¬e, &trust, &asOf, + &staleness, °raded, &dataVersion, &vector, &cvss, &epss, &epssAsOf, &kev) + if err != nil { + return lanea.AdvisoryRow{}, false + } + when, err := time.Parse(time.RFC3339, asOf) + if err != nil { + return lanea.AdvisoryRow{}, false + } + row := lanea.AdvisoryRow{ + Source: source, SourceID: sourceID, CVEID: cveID, + FeedID: source, + SnapshotDigest: cache.SchemaSHA256(), + LicenseSPDX: spdx, + LicenseManualNote: note, + Trust: record.Trust(trust), + AsOf: when, + StalenessSeconds: staleness, + FreshnessSLOSeconds: slo[source], + ParseDegraded: degraded == 1, + DataVersion: dataVersion, + CVSSVector: vector, + KEVMember: kev == 1, + } + if cvss.Valid { + v := cvss.Float64 + row.CVSSScore = &v + } + if epss.Valid { + v := epss.Float64 + row.EPSSScore = &v + } + return row, true + } +} + +// --------------------------------------------------------------------------- +// Collector inputs, and the two bridges the seams force +// --------------------------------------------------------------------------- + +// bridgePurl builds the purl internal/collector/host does not. +// +// IT IS TEST-ONLY. SEAM 2: the emitter refuses a match with no purl and says +// explicitly that it will not synthesise one because the namespace comes from +// os-release. The host collector reads os-release and does not build the purl, +// so this function stands in for a component nobody has written. The namespace +// it picks is deliberately crude — this is not a proposed implementation, it is +// a scaffold holding the seam open long enough to test what is behind it. +func bridgePurl(p host.Package) string { + ns := map[string]string{ + host.EcosystemDeb: "debian", + host.EcosystemRPM: "redhat", + host.EcosystemAPK: "alpine", + }[p.Ecosystem] + u := "pkg:" + p.Ecosystem + "/" + ns + "/" + p.Name + "@" + p.Version + if p.Arch != "" { + u += "?arch=" + p.Arch + } + return u +} + +// bridgeEcosystem maps a collector's ecosystem string onto the comparator's +// vocabulary. +// +// IT IS TEST-ONLY. SEAM 1 and SEAM 3: internal/match's allowlist is exact-match +// over {deb, rpm, apk} and its comment says the ingestion layer owns +// normalisation into that vocabulary. Nothing does. This is the missing owner, +// written here so the links behind it can be exercised — and it handles exactly +// the strings this corpus produces, because a fuller table here would look like +// a fix. +func bridgeEcosystem(s string) (string, bool) { + switch s { + case match.EcosystemDeb, match.EcosystemRPM, match.EcosystemAPK: + return s, true + case "redhat", "Red Hat", "rocky", "almalinux": + return match.EcosystemRPM, true + case "debian", "Debian:11", "ubuntu": + return match.EcosystemDeb, true + case "alpine", "Alpine:v3.19": + return match.EcosystemAPK, true + } + return "", false +} + +// --------------------------------------------------------------------------- +// The host inventory: real output where it exists, a probe where it does not +// --------------------------------------------------------------------------- +// +// TWO DIFFERENT THINGS USED TO LIVE IN ONE FIXTURE FILE, AND CONFLATING THEM +// IS WHAT THE TRIPWIRE CAUGHT. +// +// (a) EVIDENCE FOR THE HOST-INVENTORY LINK — that internal/collector/host +// produces an inventory this chain can consume. A hand-written file is +// the WORST possible evidence for that and is superseded the moment a +// package manager exists, which is exactly what decideHostLink's +// predecessor fired about on CI. +// +// (b) HOST-SHAPED PACKAGES THE SYNTHETIC ADVISORY CORPUS CAN DECIDE. The +// corpus holds CVE-2026-1001..1005 against openssl 3.1.3-r0 (apk) and +// python3-requests 2.25.1-3.el9 (rpm). No real machine has those +// versions, and no real machine ever will, so a REAL inventory cannot +// serve this purpose — not on CI, not anywhere. Deleting the file +// would delete the comparator's and the emitter's only decidable input +// and turn two PROVEN links into vacuous ones. +// +// So the two are now separate. (a) is host.Collect, wherever it runs. (b) is +// fixtures/inventory/corpus-probe-packages.json, which is named for what it is +// and is NEVER cited as evidence for the host link. + +// hostInventorySource is which host packages this run submitted, and where +// each group came from. Nothing else in this file decides that question. +type hostInventorySource struct { + // real is true when host.Collect ran to completion on this machine. + real bool + // inv is what host.Collect returned. Non-nil on both paths — the + // collector returns an inventory alongside ErrNoPackageManager — but + // Packages is empty when real is false. + inv *host.Inventory + // collected is host.Collect's own output, empty when real is false. It + // is the ONLY thing that may be cited as evidence for linkHostCollect. + collected []host.Package + // probe is the corpus probe: host-shaped packages the synthetic advisory + // corpus can decide. Present on BOTH paths, evidence for NEITHER path's + // host-collector claim. + probe []host.Package +} + +// submitted is every host package the chain hands the comparator, real output +// first so the two groups are distinguishable in a failure dump. +func (s hostInventorySource) submitted() []host.Package { + out := make([]host.Package, 0, len(s.collected)+len(s.probe)) + out = append(out, s.collected...) + return append(out, s.probe...) +} + +// label is the one-line "which inventory did this run use" the ledger and the +// determinism comparison both print. A run that proved the collector and a run +// that proved only the probe MUST NOT render the same, because telling those +// two apart is the whole point of the host link. +func (s hostInventorySource) label() string { + if s.real { + return fmt.Sprintf("host.Collect (GOOS=%s): %d real package(s) + %d corpus probe package(s)", + runtime.GOOS, len(s.collected), len(s.probe)) + } + return fmt.Sprintf("corpus probe ONLY (GOOS=%s, no package manager): %d package(s)", + runtime.GOOS, len(s.probe)) +} + +// rows renders every submitted package for the two-run byte comparison. +func (s hostInventorySource) rows() []string { + out := make([]string, 0, len(s.collected)+len(s.probe)) + render := func(origin string, pkgs []host.Package) { + for _, p := range pkgs { + out = append(out, fmt.Sprintf("origin=%s ecosystem=%s package=%s version=%s arch=%s", + origin, p.Ecosystem, p.Name, p.Version, p.Arch)) + } + } + render("collected", s.collected) + render("probe", s.probe) + return out +} + +// hostInventoryForChain decides which host inventory the chain runs on. It is +// the ONLY place that decides, and the rule is one sentence: +// +// host.Collect's REAL OUTPUT whenever a supported package manager is present +// on this machine, and the corpus probe alone when none is. +// +// The branch is the error return, not a build tag, an environment variable or +// a flag: host.Collect answers "is there a package manager here" by trying, +// and a harness that asked any other way could disagree with the collector +// about the machine it is running on. +// +// THERE IS NO SKIP ON EITHER BRANCH. Both run the full chain; what differs is +// what the ledger is entitled to claim afterwards. +func hostInventoryForChain(t *testing.T) hostInventorySource { + t.Helper() + src := hostInventorySource{probe: corpusProbePackages(t)} + + inv, err := host.Collect(context.Background(), host.Options{ + // The same instant as everything else in this run, so CollectedAt + // and AsOf cannot make two runs differ. + Now: func() time.Time { return fixtureClock }, + }) + switch { + case err == nil: + // A supported package manager exists. Its output is better evidence + // than anything written by hand, so the chain takes it. + src.real = true + src.inv = inv + src.collected = inv.Packages + case errors.Is(err, host.ErrNoPackageManager): + src.inv = inv + default: + t.Fatalf("host.Collect failed for a reason this harness does not understand: %v", err) + } + return src +} + +// corpusProbePackages loads (b) above: host-shaped packages chosen to +// intersect the synthetic advisory corpus. +// +// IT IS NOT A RECORDING AND MUST NEVER BE READ AS ONE. It carries no purl, +// because host.Package carries no purl — that absence is SEAM 2 and is +// asserted in emitAll rather than described. +func corpusProbePackages(t *testing.T) []host.Package { + t.Helper() + var doc struct { + Packages []host.Package `json:"packages"` + } + if err := json.Unmarshal(readFixture(t, "fixtures/inventory/corpus-probe-packages.json"), &doc); err != nil { + t.Fatalf("reading the corpus probe packages: %v", err) + } + if len(doc.Packages) == 0 { + t.Fatal("the corpus probe is empty, so the comparator and the emitter have no decidable " + + "host input and both links would pass vacuously") + } + for _, p := range doc.Packages { + if strings.TrimSpace(p.Ecosystem) == "" || strings.TrimSpace(p.Name) == "" || + strings.TrimSpace(p.Version) == "" { + t.Fatalf("corpus probe package %+v is missing a field host.Package requires; the file "+ + "is keyed on host.Package's JSON tags so that a struct change breaks it here", p) + } + } + return doc.Packages +} + +// hostPackageRecords applies the field mapping internal/match documents on +// PackageRecord — plus bridgePurl, which the mapping has no column for. +func hostPackageRecords(pkgs []host.Package) []match.PackageRecord { + out := make([]match.PackageRecord, 0, len(pkgs)) + for _, p := range pkgs { + eco, ok := bridgeEcosystem(p.Ecosystem) + if !ok { + continue + } + out = append(out, match.PackageRecord{ + Collector: host.Collector, + Ecosystem: eco, + Name: p.Name, + Version: p.Version, + Arch: p.Arch, + Purl: bridgePurl(p), + }) + } + return out +} + +func repoScan(t *testing.T) repo.ScanResult { + t.Helper() + res, err := repo.ParseReport(readFixture(t, "fixtures/trivy/report.json")) + if err != nil { + t.Fatalf("parsing the Trivy report fixture: %v", err) + } + if len(res.Findings) == 0 { + t.Fatal("the Trivy report fixture parsed to zero findings") + } + if err := res.AssertNotSilentlyEmpty(); err != nil { + t.Fatalf("the parsed scan reports itself as silently empty: %v", err) + } + return res +} + +// repoPackageRecords applies internal/match's documented repo.Finding mapping +// AND NOTHING ELSE — no bridgeEcosystem here, deliberately. +// +// SEAM 3 is only visible if the comparator is allowed to see what the collector +// actually reports. Trivy's `Type` for a lockfile is "npm"; the comparator +// implements deb, rpm and apk. Bridging it here would hide the refusal inside +// the harness, and the point is to make the comparator COUNT it — a refused +// package is a countable gap in CoverageReport rather than a silent zero. +func repoPackageRecords(scan repo.ScanResult) []match.PackageRecord { + out := make([]match.PackageRecord, 0, len(scan.Findings)) + for _, f := range scan.Findings { + out = append(out, match.PackageRecord{ + Collector: f.Collector, + Ecosystem: f.Ecosystem, + Name: f.PackageName, + Version: f.InstalledVersion, + Purl: f.Purl, + ManifestRelPath: f.ManifestRelPath, + }) + } + return out +} + +// --------------------------------------------------------------------------- +// The cache as an advisory source +// --------------------------------------------------------------------------- + +// cacheSource is internal/match's AdvisorySource read straight off the A.2 +// cache. +// +// IT IS TEST-ONLY AND ITS EXISTENCE IS A FINDING. internal/match defines the +// interface precisely so the comparator opens no database, and NOTHING under +// internal/ implements it against the cache — the comparator and the cache +// have never been connected outside this file. See linkProductionUp. +// +// It excludes tombstoned advisories, which is exit criterion 22 seen from the +// read side: a withdrawn advisory keeps its row and stops deciding findings. +type cacheSource struct{ db *sql.DB } + +func (s *cacheSource) AffectedRanges(ctx context.Context, ecosystem, pkg string) ([]match.AffectedRange, error) { + const q = ` +SELECT a.source, a.source_id, ifnull(a.cve_id,''), af.ecosystem, af.package, ifnull(af.purl,''), + ifnull(af.introduced,''), ifnull(af.fixed,''), af.distro_backport +FROM affected af +JOIN advisory a ON a.source = af.source AND a.source_id = af.source_id +WHERE af.ecosystem = ? AND af.package = ? AND a.state = ? +ORDER BY a.source, a.source_id, af.id` + rows, err := s.db.QueryContext(ctx, q, ecosystem, pkg, cache.AdvisoryPublished) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var out []match.AffectedRange + for rows.Next() { + var r match.AffectedRange + var backport int + if err := rows.Scan(&r.Source, &r.SourceID, &r.CVEID, &r.Ecosystem, &r.Package, + &r.Purl, &r.Introduced, &r.Fixed, &backport); err != nil { + return nil, err + } + r.DistroBackport = backport == 1 + // An unbounded range is refused by the comparator, and an advisory + // that names only a fixed version is unbounded below by construction. + // "0" is the OSV convention for it and is what the corpus's own OSV + // documents carry. + if r.Introduced == "" && r.Fixed == "" && !r.AllVersions { + continue + } + if r.Introduced == "" { + r.Introduced = "0" + } + out = append(out, r) + } + return out, rows.Err() +} + +// --------------------------------------------------------------------------- +// Assertions over the cache +// --------------------------------------------------------------------------- + +func assertStoredStringsAreSanitized(t *testing.T, db *sql.DB) { + t.Helper() + // The queryable columns of `advisory`... + rows, err := db.Query(`SELECT source, source_id, ifnull(severity,''), ifnull(cvss_vector,''), + ifnull(data_version,''), ifnull(epss_as_of,'') FROM advisory`) + if err != nil { + t.Fatalf("reading advisory rows: %v", err) + } + defer func() { _ = rows.Close() }() + n := 0 + for rows.Next() { + var source, id, sev, vector, dataVersion, epssAsOf string + if err := rows.Scan(&source, &id, &sev, &vector, &dataVersion, &epssAsOf); err != nil { + t.Fatalf("scanning: %v", err) + } + for field, v := range map[string]string{ + "severity": sev, "cvss_vector": vector, "data_version": dataVersion, "epss_as_of": epssAsOf, + } { + if err := sanitize.AssertSanitized(v); err != nil { + t.Errorf("%s/%s: the stored %s did not survive AssertSanitized: %v", source, id, field, err) + } + } + n++ + } + if err := rows.Err(); err != nil { + t.Fatalf("reading advisory rows: %v", err) + } + if n == 0 { + t.Fatal("no advisory rows were read, so the sanitiser assertion passed vacuously") + } + + // ...and the indexed prose, which cannot be read back. + // + // advisory_fts is contentless, so the text is only observable through a + // MATCH. The corpus's injected characters are checked by SEARCHING for the + // tokens they were embedded in: a description that still carried a + // zero-width space between two letters would tokenise differently and the + // probe below would miss. + if n := count(t, db, `SELECT count(*) FROM advisory_fts WHERE advisory_fts MATCH ?`, "mishandles"); n != 1 { + t.Errorf("the sanitized description of the injection fixture matches %d rows for a word it "+ + "contains, want 1; a zero-width character left inside a word breaks tokenisation, which "+ + "is the retrieval failure spine S7's ingest-time sanitisation prevents", n) + } + if n := count(t, db, `SELECT count(*) FROM advisory_fts WHERE advisory_fts MATCH ?`, "anvilinjectionprobe"); n != 0 { + t.Errorf("the HTML comment carrying an injected instruction is still searchable in %d rows; "+ + "it should have been removed before the text reached the index", n) + } +} + +// assertRawJSONIsVerbatim is the counterpart control. raw_json is the +// publisher's bytes and must NOT be sanitised: CVE-TOU requires records be +// stored byte-verbatim, and two importers that re-render a document store two +// different digests of one advisory. +func assertRawJSONIsVerbatim(t *testing.T, db *sql.DB) { + t.Helper() + var raw []byte + err := db.QueryRow(`SELECT raw_json FROM advisory WHERE source_id = ?`, "CVE-2026-1001").Scan(&raw) + if err != nil { + t.Fatalf("reading raw_json for the injection fixture: %v", err) + } + if !bytes.Contains(raw, []byte("\u200b")) { + t.Error("raw_json no longer carries the zero-width space the fixture ships. The column is " + + "the publisher's bytes verbatim; sanitising it would change the stored document and " + + "break byte-for-byte agreement between the two importers.") + } +} + +func assertCacheInvariants(t *testing.T, db *sql.DB) { + t.Helper() + if err := cache.CheckFTS5(context.Background(), db); err != nil { + t.Fatalf("exit criterion 2: FTS5 is not active on the migrated cache: %v", err) + } + // Exit criterion 11: no advisory row may have BOTH licence columns null. + if n := count(t, db, `SELECT count(*) FROM advisory WHERE license_spdx IS NULL AND license_manual_note IS NULL`); n != 0 { + t.Errorf("exit criterion 11: %d advisory rows declare no licence at all", n) + } + // Exit criterion 23: the unknown dataVersion is persisted and flagged. + if n := count(t, db, `SELECT count(*) FROM advisory WHERE source_id = ? AND parse_degraded = 1 AND data_version = ?`, + "CVE-2026-1002", "5.9"); n != 1 { + t.Errorf("exit criterion 23: the dataVersion 5.9 record is not persisted with parse_degraded=1 (%d rows)", n) + } + // Exit criterion 22: the REJECTED record is tombstoned, kept, and out of + // the index. + if n := count(t, db, `SELECT count(*) FROM advisory WHERE source_id = ? AND state = ? AND tombstoned_at IS NOT NULL`, + "CVE-2026-1003", cache.AdvisoryRejected); n != 1 { + t.Errorf("exit criterion 22: the REJECTED record is not tombstoned (%d rows)", n) + } + if n := count(t, db, `SELECT count(*) FROM advisory_fts WHERE advisory_fts MATCH ?`, "withdrawn"); n != 0 { + t.Errorf("a tombstoned advisory still matches %d rows in the FTS index", n) + } + // Every advisory carries the gate's tier, never the feed row's claim. + if n := count(t, db, `SELECT count(*) FROM advisory WHERE license_tier NOT IN (0,1,2,3)`); n != 0 { + t.Errorf("%d advisory rows carry a tier outside the enum", n) + } +} + +func assertComparatorSeams(t *testing.T, db *sql.DB, cov match.CoverageReport, results []match.MatchResult) { + t.Helper() + + // Exit criterion 20: coverage is populated on every run. + if cov.PackagesSubmitted == 0 || len(cov.SchemesImplemented) == 0 { + t.Errorf("exit criterion 20: the CoverageReport is not populated: %+v", cov) + } + if err := cov.AssertNotSilentlyClean(results); err != nil && len(results) == 0 { + t.Errorf("a zero-finding run was not reported as an absence: %v", err) + } + + // SEAM 1, asserted. The Debian OSV export IS in the cache, and it is NOT + // reachable for a deb host package, because the decoder wrote the + // publisher's ecosystem spelling into a column the comparator matches + // exactly. + stored := count(t, db, `SELECT count(*) FROM affected WHERE ecosystem = ?`, "Debian:11") + if stored == 0 { + t.Fatal("the Debian OSV fixture did not land in `affected`, so SEAM 1 cannot be asserted") + } + reachable := count(t, db, `SELECT count(*) FROM affected WHERE ecosystem = ?`, match.EcosystemDeb) + if reachable != 0 { + t.Errorf("SEAM 1 HAS CLOSED: %d rows now carry the comparator's own ecosystem vocabulary. "+ + "Something normalises. Delete bridgeEcosystem, point the harness at the real column, "+ + "and re-read this assertion.", reachable) + } + for _, r := range results { + if r.Ecosystem == match.EcosystemDeb { + t.Errorf("a deb finding was produced (%s/%s); with no normalisation owner this is "+ + "impossible and the assertion above is wrong", r.Package, r.InstalledVersion) + } + } + + // SEAM 3, asserted. The repository collector reports lang-pkgs findings — + // it skips os-pkgs as A.9's territory — and the comparator implements OS + // package schemes only. The two halves of the SCA path have DISJOINT + // DOMAINS, so no repository finding can ever become a record. What the + // comparator does right is refuse it COUNTABLY rather than returning a + // quiet zero. + if cov.PackagesRefusedScheme == 0 { + t.Error("SEAM 3 HAS CLOSED, or the repo fixture stopped reaching the comparator: no package " + + "was refused for an unimplemented scheme, though Trivy reported an npm dependency and " + + "internal/match implements deb, rpm and apk only") + } + // The refusal must be findable BY NAME somewhere, or "1 package refused" + // is a number an operator cannot act on. + named := false + for _, r := range cov.Refusals { + if strings.Contains(r.Detail, "npm") || r.Ecosystem == "npm" { + named = true + } + } + if !named { + t.Errorf("no entry in CoverageReport.Refusals names the ecosystem that was refused: %+v", + cov.Refusals) + } + // THIS WAS A LOGGED FINDING AND IS NOW AN ASSERTION. A.21 reported that + // EcosystemsRefused — documented as "the list an operator uses to decide + // what to implement next" — was populated only from + // RefusalUnsupportedEcosystem, so a record carrying a purl (every repo-SCA + // finding, and what every collector is encouraged to supply) was refused + // as RefusalUnsupportedPurlType and vanished from the list. The count was + // visible; the thing to implement next was not. internal/match now feeds + // both routes into the list (see Refusal.refusedIdentityToken), and this + // is the end-to-end guard against it regressing. + if cov.PackagesRefusedScheme > 0 && len(cov.EcosystemsRefused) == 0 { + t.Errorf("%d package(s) were refused for an unimplemented scheme and "+ + "CoverageReport.EcosystemsRefused is EMPTY. The count is visible and the thing to "+ + "implement next is not, which is the exact defect A.21 reported and internal/match "+ + "fixed: a refusal arriving by the purl route (RefusalUnsupportedPurlType) is the same "+ + "fact about coverage as one arriving by the ecosystem route. Refusals: %+v", + cov.PackagesRefusedScheme, cov.Refusals) + } + // And specifically: this corpus's refusal arrives by the PURL route, so + // the assertion above cannot be satisfied by the ecosystem route alone. + if !containsString(cov.EcosystemsRefused, "npm") { + t.Errorf("the repository collector's npm dependency was refused, but %q is not in "+ + "EcosystemsRefused (%v). The refusal reaches the comparator carrying a purl, so this "+ + "is the purl route specifically — an operator reading the list would not learn that "+ + "npm is what Anvil cannot yet handle.", "npm", cov.EcosystemsRefused) + } + if cov.Complete { + t.Error("the run refused a package and still reported Complete; Complete must be false " + + "whenever something was refused, or \"nothing was found\" cannot be read as an answer") + } + + // The vendor range IS consulted where the vocabulary happens to line up: + // the Red Hat CSAF decoder writes "rpm" and the Alpine secdb decoder + // writes "apk", so those two reach the comparator. + if count(t, db, `SELECT count(*) FROM affected WHERE ecosystem = ? AND distro_backport = 1`, match.EcosystemRPM) == 0 { + t.Error("the CSAF fixture produced no backported rpm range, so the vendor-first path is untested here") + } + + // The rpm host package sits exactly ON the vendor's fixed version and must + // NOT be flagged: `fixed` is an EXCLUSIVE upper bound. + for _, r := range results { + if r.Collector == cache.CollectorHost && r.Package == "python3-requests" { + t.Errorf("the host's python3-requests %s was flagged against a vendor advisory that "+ + "names it as the fixed version; this is the backport false-positive class", + r.InstalledVersion) + } + } +} + +// --------------------------------------------------------------------------- +// The links this machine cannot prove +// --------------------------------------------------------------------------- + +// decideHostLink writes the host-inventory link's ledger entry, and it is +// called FROM INSIDE runChain, at the point the packages were actually +// submitted — not afterwards from a second host.Collect whose output nothing +// consumed. A ledger entry written somewhere other than where the evidence was +// produced can claim something the chain did not do; that is precisely the +// failure this file exists to prevent, and it is how the fixture inventory came +// to be cited for a link it never exercised. +// +// The two branches say DIFFERENT THINGS on purpose. A reader must be able to +// tell a run that proved the collector from a run that proved only the probe, +// and both the verdict (PROVEN / UNPROVEN) and the text differ accordingly. +func decideHostLink(t *testing.T, led *ledger, src hostInventorySource, collectedSubmitted int, results []match.MatchResult) { + t.Helper() + + for _, p := range hostSourceProblems(src, collectedSubmitted) { + t.Error(p) + } + + if !src.real { + led.unproven(linkHostCollect, + "no dpkg, rpm or apk exists on this host (GOOS="+runtime.GOOS+"), so host.Collect "+ + "cannot produce an inventory here and the chain's host side is the CORPUS PROBE "+ + "alone (fixtures/inventory/corpus-probe-packages.json, "+ + strconv.Itoa(len(src.probe))+" hand-written packages in host.Package's own JSON "+ + "shape). The collector's REFUSAL is proven — it ran to completion and returned "+ + "ErrNoPackageManager rather than hanging or inventing rows — but its OUTPUT is "+ + "not, and no probe package is evidence about the collector.", + "run this harness on a host with one of the three package managers installed; "+ + "hostInventoryForChain then takes host.Collect's real output automatically and "+ + "this entry becomes PROVEN with the collected package count. The probe file stays "+ + "either way: it is the only host input the synthetic advisory corpus can decide, "+ + "and a real inventory cannot replace it") + return + } + + var families []string + for _, c := range src.inv.Coverage { + families = append(families, fmt.Sprintf("%s=%s(%d)", c.Ecosystem, c.Status, c.Packages)) + } + hostFindings := 0 + for _, r := range results { + if r.Collector == cache.CollectorHost { + hostFindings++ + } + } + + led.proven(linkHostCollect, fmt.Sprintf( + "host.Collect ran to completion on this machine (GOOS=%s) and returned %d package(s); "+ + "families %v; parse_degraded=%v. ALL %d were mapped through the documented "+ + "PackageRecord field mapping and submitted to the comparator IN THIS RUN, so this "+ + "entry rests on the collector's own output and not on any file in fixtures/. "+ + "NOTE WHAT THIS DOES NOT SAY: none of those packages produced a finding, and none can "+ + "— this corpus holds only synthetic CVE-2026-xxxx advisories against versions no real "+ + "machine has. The %d host finding(s) below come from the corpus probe, which is "+ + "submitted alongside and is NOT evidence about the collector. What is proven here is "+ + "that a real inventory traverses the chain: collect -> ecosystem mapping -> identity "+ + "-> comparator, at real size, byte-identically across two runs", + runtime.GOOS, len(src.collected), families, src.inv.ParseDegraded, + collectedSubmitted, hostFindings)) +} + +func decideRepoCollector(t *testing.T, led *ledger) { + t.Helper() + scan := repoScan(t) + if bin, err := repo.ResolveBinary(repo.BinaryName); err == nil { + led.proven(linkRepoCollect, "trivy resolved at "+bin+" and the scan ran end to end") + t.Errorf("trivy is installed at %s, so the recorded-report fixture is no longer the best "+ + "available evidence. Run repo.ScanRepo against a fixture tree and prove the collector "+ + "itself, not only its parser. The Trivy E2E job's finding — that the collector cannot "+ + "run without a pre-seeded database — is exactly what a recorded report cannot show.", bin) + return + } + led.unproven(linkRepoCollect, + fmt.Sprintf("the trivy binary is not on PATH, so repo.ScanRepo cannot run here. What IS "+ + "proven is the report path: repo.ParseReport turned a recorded-shape Trivy report into "+ + "%d finding(s) with populated Coverage, and AssertNotSilentlyEmpty accepted it. The "+ + "SCAN is not proven, and a recorded report presupposes the successful run it is "+ + "standing in for — the precise gap the Trivy E2E job found.", len(scan.Findings)), + "install the pinned trivy release and its vulnerability database on the CI host, then call "+ + "repo.ScanRepo against a fixture repository; only that exercises binary resolution, "+ + "argument construction, the exit-code contract and the DB-absent failure mode") +} + +// decideNotWired records the links that exist as packages with no caller. +func decideNotWired(t *testing.T, led *ledger) { + t.Helper() + led.unproven(linkSelfHeal, + "internal/ingest/reconcile's weekly baseline self-heal is not exercised by this harness: "+ + "it re-pulls the full bulk artifact, and proving that it RESTORES a silently-dropped "+ + "record needs a second, later archive plus a deliberate deletion between the two runs", + "A.15's own TestSelfHealRestoresDroppedRecords covers it against a synthetic "+ + "missing-records fixture; an end-to-end proof needs this harness to serve two "+ + "generations of the same archive and delete rows between them") + led.unproven(linkAccelerator, + "internal/mirror/accelerator is not in this chain at all. It is a warm-start optimisation "+ + "that pulls a compiled Trivy-DB/Grype-DB artifact, and pulling one requires a registry "+ + "this harness must not contact", + "A.11/A.13's own tests cover the version gate and the consume-only write refusal against "+ + "synthetic artifacts; an end-to-end proof needs a local OCI registry fixture") + led.unproven(linkProductionUp, + "NOTHING UNDER internal/ WIRES THIS CHAIN TOGETHER. Two components this harness needed do "+ + "not exist in production form: an internal/match.AdvisorySource backed by the A.2 cache "+ + "(the comparator has never been connected to the cache outside this file) and a reader "+ + "that fills internal/record/lanea.AdvisoryRow from an `advisory` row (its doc says "+ + "\"the caller reads the row and fills this in\"; there is no caller). cacheSource and "+ + "advisoryLookup in this file are stand-ins written for the test", + "implement both in internal/ — a cache-backed AdvisorySource and an AdvisoryRow reader — "+ + "and have this harness call them instead of its own copies; the diff between the two "+ + "implementations is then the thing that gets reviewed") +} + +// --------------------------------------------------------------------------- +// Two-run comparison +// --------------------------------------------------------------------------- + +func diffChains(a, b chainOutput) []string { + var out []string + cmp := func(name string, l, r []string) { + if len(l) != len(r) { + out = append(out, fmt.Sprintf("%s: %d rows then %d rows", name, len(l), len(r))) + return + } + for i := range l { + if l[i] != r[i] { + out = append(out, fmt.Sprintf("%s row %d:\n run 1: %s\n run 2: %s", name, i, l[i], r[i])) + } + } + } + cmp("advisory", a.advisory, b.advisory) + cmp("affected", a.affected, b.affected) + cmp("cve_alias", a.alias, b.alias) + cmp("advisory_fts", a.fts, b.fts) + cmp("feed_state", a.feedState, b.feedState) + + // The host INPUT. Two runs that read the host's package database twice + // must read the same thing, and on a real host that is a claim worth + // making at 1000+ rows: host.Collect is documented read-only and sorts + // its output, so a diff here is either a package that was installed or + // removed mid-test or a collector that is not deterministic. Either one + // is a finding, and the diff below is its report. DO NOT relax this to + // "compare counts" — a swap of two versions keeps the count. + if a.hostSourceLabel != b.hostSourceLabel { + out = append(out, fmt.Sprintf("host inventory source: %q then %q", + a.hostSourceLabel, b.hostSourceLabel)) + } + cmp("host inventory", a.hostRows, b.hostRows) + + if len(a.matches) != len(b.matches) { + out = append(out, fmt.Sprintf("match results: %d then %d", len(a.matches), len(b.matches))) + } else { + for i := range a.matches { + if a.matches[i] != b.matches[i] { + out = append(out, fmt.Sprintf("match result %d:\n run 1: %+v\n run 2: %+v", + i, a.matches[i], b.matches[i])) + } + } + } + if !bytes.Equal(a.emissionJSON, b.emissionJSON) { + out = append(out, "emitted records differ:\n run 1:\n"+string(a.emissionJSON)+ + "\n run 2:\n"+string(b.emissionJSON)) + } + return out +} + +// TestTheDiffDetectorSeesADifference is diffChains' negative control. +// +// The determinism claim in the ledger rests entirely on diffChains returning +// nothing. A comparator that returns nothing for two DIFFERENT inputs would +// make that claim unfalsifiable, and "byte-identical across two runs" would be +// a sentence about the detector rather than about the chain. +func TestTheDiffDetectorSeesADifference(t *testing.T) { + base := chainOutput{ + advisory: []string{"source=a source_id=1"}, + affected: []string{"source=a source_id=1 package=openssl"}, + alias: []string{"cve_id=CVE-2026-1001"}, + fts: []string{"source=a source_id=1 indexed"}, + feedState: []string{"feed_id=a"}, + hostSourceLabel: "corpus probe ONLY", + hostRows: []string{"origin=probe ecosystem=apk package=openssl version=3.1.3-r0 arch="}, + matches: []match.MatchResult{{Source: "a", SourceID: "1", Package: "openssl"}}, + emissionJSON: []byte("{}"), + } + if d := diffChains(base, base); len(d) != 0 { + t.Fatalf("two identical outputs were reported as differing: %v", d) + } + + cases := map[string]func(*chainOutput){ + "an advisory column": func(c *chainOutput) { c.advisory = []string{"source=a source_id=2"} }, + "an affected row": func(c *chainOutput) { c.affected = append(c.affected, "extra") }, + "an alias row": func(c *chainOutput) { c.alias = nil }, + "the FTS index": func(c *chainOutput) { c.fts = []string{"source=a source_id=1 not-indexed"} }, + "feed state": func(c *chainOutput) { c.feedState = []string{"feed_id=b"} }, + "a match result": func(c *chainOutput) { c.matches[0].Package = "busybox" }, + "an emitted record": func(c *chainOutput) { c.emissionJSON = []byte("{ }") }, + "the number of rows": func(c *chainOutput) { c.advisory = nil }, + // The host input, both halves: WHICH inventory was used, and what it + // contained. Without these two the determinism claim would say + // nothing about a real inventory, because a real inventory produces + // no findings against this corpus and would leave every other field + // untouched. + "the host inventory source": func(c *chainOutput) { c.hostSourceLabel = "host.Collect" }, + "a host package version": func(c *chainOutput) { + c.hostRows = []string{"origin=probe ecosystem=apk package=openssl version=3.1.4-r0 arch="} + }, + "the number of host packages": func(c *chainOutput) { c.hostRows = nil }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + other := base + other.advisory = append([]string(nil), base.advisory...) + other.affected = append([]string(nil), base.affected...) + other.alias = append([]string(nil), base.alias...) + other.fts = append([]string(nil), base.fts...) + other.feedState = append([]string(nil), base.feedState...) + other.hostRows = append([]string(nil), base.hostRows...) + other.matches = append([]match.MatchResult(nil), base.matches...) + other.emissionJSON = append([]byte(nil), base.emissionJSON...) + mutate(&other) + if d := diffChains(base, other); len(d) == 0 { + t.Errorf("diffChains missed a change to %s, so the determinism claim cannot fail", name) + } + }) + } +} + +// --------------------------------------------------------------------------- +// Exit criteria checked against the working tree rather than the corpus +// --------------------------------------------------------------------------- + +// urlLiteralAllowlist is exit criterion 1 as an ALLOWLIST, because a denylist +// of feed hostnames loses: the next feed is the one nobody listed. +// +// The rule: no string literal under internal/ingest, internal/collector or +// internal/mirror may contain "://" unless it appears here EXACTLY, with a +// reason. Exact and not substring — a substring rule keyed on "https://" would +// admit every URL in the tree, which is the shape of an allowlist that has +// stopped being one. +// +// READ THE LAST THREE ENTRIES. Exit criterion 1 is written about +// internal/ingest and it holds there: no endpoint of any kind is compiled into +// the ingestion tree. internal/mirror/accelerator is OUTSIDE that scope and +// does compile in two endpoints. They are recorded here rather than quietly +// tolerated, and they are reported as an open item — the criterion's own words +// are "no hard-coded feed URL", and an operator who has to point the +// accelerator at a self-hosted mirror (which research/06 S23 says they should, +// after the GHCR TOOMANYREQUESTS incident) has to recompile today. +var urlLiteralAllowlist = map[string]string{ + "anvil-ingest/1 (+https://github.com/Susquehanna-Syntax/Anvil)": "Anvil's own project URL " + + "inside the HTTP User-Agent. It is this process's name, not a feed.", + "https://": "a scheme PREFIX used for validation (a feed URL must be https), not an endpoint", + "://": "a scheme SEPARATOR used for parsing, not an endpoint", + "https://github.com/aquasecurity/trivy/releases (Apache-2.0), put it on PATH, ": "the operator " + + "install hint printed when the trivy binary is absent. It is documentation in an error " + + "message, not a fetch target: nothing in internal/collector/repo downloads it.", + "https://grype.anchore.io/databases/v6/latest.json": "OPEN ITEM: internal/mirror/accelerator " + + "compiles in the Grype v6 listing endpoint as its default. Outside exit criterion 1's " + + "stated scope (internal/ingest), but the same argument applies and research/06 S23 says " + + "operators should self-host. Reported by A.21, not fixed by it.", + "https://ghcr.io": "OPEN ITEM: the same, for the Trivy-DB OCI registry default.", +} + +func TestNoFeedURLLiteralInLaneASource(t *testing.T) { + roots := []string{ + filepath.Join(repoRoot, "internal", "ingest"), + filepath.Join(repoRoot, "internal", "collector"), + filepath.Join(repoRoot, "internal", "mirror"), + } + used := map[string]bool{} + scanned := 0 + for _, root := range roots { + err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") { + return nil + } + scanned++ + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, p, nil, parser.SkipObjectResolution) + if err != nil { + t.Errorf("parsing %s: %v", p, err) + return nil + } + ingest := strings.Contains(filepath.ToSlash(p), "/internal/ingest/") + ast.Inspect(file, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + s, err := strconv.Unquote(lit.Value) + if err != nil || !strings.Contains(s, "://") { + return true + } + if _, ok := urlLiteralAllowlist[s]; ok { + used[s] = true + return true + } + where := "Lane A source" + if ingest { + where = "the internal/ingest tree, which exit criterion 1 names by hand" + } + t.Errorf("%s: the URL literal %q is compiled into %s. Feed URLs live in feeds.yaml. "+ + "If this is not a feed endpoint, add it to urlLiteralAllowlist with a reason.", + fset.Position(lit.Pos()), s, where) + return true + }) + return nil + }) + if err != nil { + t.Fatalf("walking %s: %v", root, err) + } + } + if scanned == 0 { + t.Fatal("the scan indexed no files; the guard is inert and would pass anything") + } + for allowed, reason := range urlLiteralAllowlist { + if !used[allowed] { + t.Errorf("urlLiteralAllowlist names %q (%q), which no longer appears. Delete the entry: "+ + "a stale exemption is how the next real one gets waved through.", allowed, reason) + } + } + t.Logf("exit criterion 1: scanned %d non-test Go files under internal/{ingest,collector,mirror}; "+ + "%d allowlisted literals, no unlisted endpoint. Two of the allowlisted entries are OPEN "+ + "ITEMS in internal/mirror/accelerator, not clean results — read the map.", scanned, len(used)) +} + +// TestTheLicenceGateAdmitsNothingInThisWorkingTree is the honest statement +// about the licence link. +// +// The chain above admits its feeds against a SYNTHETIC pinned body. In this +// repository no publisher licence body has been acquired, so on a fresh clone +// the gate admits NOTHING and Lane A ingests nothing at all. That is the +// production state and it must be visible in the report rather than implied by +// its absence. +func TestTheLicenceGateAdmitsNothingInThisWorkingTree(t *testing.T) { + mirrorFS := os.DirFS(repoRoot) + statuses, err := license.MirrorStatus(mirrorFS) + if err != nil { + t.Fatalf("reading the real mirror manifest: %v", err) + } + if len(statuses) == 0 { + t.Fatal("the real mirror manifest pins nothing, so this test proves nothing") + } + acquired := 0 + for _, s := range statuses { + if s.State == license.BodyVerified { + acquired++ + } + } + if acquired > 0 { + t.Errorf("%d licence bodies are now acquired and pinned in mirror/. The Lane A chain can "+ + "be run against REAL feed licences, and the harness must stop reporting the licence "+ + "link as proven only against a synthetic body. Statuses: %v", acquired, statuses) + } + t.Logf("licence gate, PRODUCTION STATE: %d pins, %d acquired. On a fresh clone the gate admits "+ + "no feed at all, so nothing is fetched and nothing is written. The chain test above proves "+ + "the gate ADMITS correctly given evidence; it does not and cannot prove that any real feed "+ + "is admissible today.", len(statuses), acquired) +} + +// TestTierTwoQuarantineIsOnDiskAndEnforced is exit criterion 9 against the +// working tree: the three share-alike directories exist with non-empty LICENSE +// files, and the write-path gate refuses to put their content anywhere else. +func TestTierTwoQuarantineIsOnDiskAndEnforced(t *testing.T) { + for _, dir := range []string{"ubuntu", "alpine", "osv"} { + p := filepath.Join(repoRoot, "mirror", "tier2", dir, "LICENSE") + info, err := os.Stat(p) + if err != nil { + t.Errorf("exit criterion 9: %s: %v", p, err) + continue + } + if info.Size() == 0 { + t.Errorf("exit criterion 9: %s is empty", p) + } + } + for _, bad := range []string{"mirror/tier0/ubuntu/data.json", "mirror/tier1/alpine/data.json"} { + if err := license.CheckWritePath(config.LicenseTier2, bad); err == nil { + t.Errorf("exit criterion 9: the gate allowed tier 2 content to be written to %s", bad) + } + } + // The negative control: the quarantine directory itself must be allowed, + // or the check above would pass by refusing everything. + if err := license.CheckWritePath(config.LicenseTier2, "mirror/tier2/ubuntu/data.json"); err != nil { + t.Errorf("the gate refused a tier 2 write into its own quarantine directory: %v", err) + } +} + +// TestTheLedgerRefusesAnUnsupportedClaim is the ledger's negative control. +// +// checkLedger flags nothing on a well-formed run, and a validator that has +// never rejected anything has not been tested. +func TestTheLedgerRefusesAnUnsupportedClaim(t *testing.T) { + full := newLedger() + for _, id := range chainLinks { + full.proven(id, "evidence") + } + if p := checkLedger(full); len(p) != 0 { + t.Fatalf("a complete ledger was flagged: %v", p) + } + + cases := []struct { + name string + mut func(*ledger) + want string + }{ + {"missing link", func(l *ledger) { delete(l.v, linkCache) }, "never decided"}, + {"proven with no evidence", func(l *ledger) { l.v[linkCache] = verdict{proven: true} }, "no evidence"}, + {"unproven with no reason", func(l *ledger) { l.v[linkCache] = verdict{settles: "x"} }, "what is missing"}, + {"unproven with no settlement", func(l *ledger) { l.v[linkCache] = verdict{missing: "x"} }, "settlement condition"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + l := newLedger() + for _, id := range chainLinks { + l.proven(id, "evidence") + } + tc.mut(l) + problems := checkLedger(l) + if len(problems) == 0 { + t.Fatalf("checkLedger accepted %q", tc.name) + } + if !strings.Contains(strings.Join(problems, "\n"), tc.want) { + t.Errorf("checkLedger flagged %q but not for the expected reason: %v", tc.name, problems) + } + }) + } +} + +// --------------------------------------------------------------------------- +// Fixture plumbing +// --------------------------------------------------------------------------- + +func readFixture(t *testing.T, p string) []byte { + t.Helper() + b, err := fixtures.ReadFile(p) + if err != nil { + t.Fatalf("reading fixture %s: %v", p, err) + } + return b +} + +// fixturePathFor maps a feed id onto the single document the delta path is +// handed for it. It is a TEST mapping over TEST fixtures, not a feed table. +func fixturePathFor(feedID string) string { + switch feedID { + case "cisa-kev-fixture": + return "fixtures/kev/known_exploited_vulnerabilities.json" + case "osv-debian-fixture": + return "fixtures/osv/GHSA-deb-2026-0001.json" + } + panic("no delta fixture for feed " + feedID) +} + +// metadataSPDX is what a registry would report for a feed. Only KEV has one, +// and it disagrees with the row: that is exit criterion 10. +func metadataSPDX(feedID string) string { + if feedID == "cisa-kev-fixture" { + return config.LicenseNoAssertion + } + return "" +} + +func loadFeeds(t *testing.T, base string) config.FeedSet { + t.Helper() + doc := strings.ReplaceAll(string(readFixture(t, "fixtures/feeds.yaml")), "${BASE}", base) + set, err := config.Parse([]byte(doc)) + if err != nil { + t.Fatalf("parsing the fixture feed table: %v", err) + } + return set +} + +// fixtureServer serves each bulk feed as a zip and each incremental feed as +// its raw document, over an in-process TLS listener. Nothing leaves the +// process and no name here resolves. +func fixtureServer(t *testing.T) *httptest.Server { + t.Helper() + archives := map[string][]byte{ + "/cvelistv5/all.zip": buildZip(t, []string{ + "fixtures/cvelistv5/CVE-2026-1001.json", + "fixtures/cvelistv5/CVE-2026-1002.json", + "fixtures/cvelistv5/CVE-2026-1003.json", + }), + "/alpine/all.zip": buildZip(t, []string{"fixtures/alpine/v3.19-main.json"}), + "/redhat/all.zip": buildZip(t, []string{"fixtures/redhat/RHSA-2026-0001.json"}), + } + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if body, ok := archives[r.URL.Path]; ok { + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("ETag", `"`+shortDigest(body)+`"`) + w.Header().Set("Last-Modified", fixtureClock.Format(http.TimeFormat)) + _, _ = w.Write(body) + return + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv +} + +// buildZip builds a zip with no timestamps and a fixed member order, so the +// archive bytes are identical between runs. +func buildZip(t *testing.T, paths []string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, p := range paths { + w, err := zw.CreateHeader(&zip.FileHeader{Name: path.Base(p), Method: zip.Deflate}) + if err != nil { + t.Fatalf("zip header for %s: %v", p, err) + } + if _, err := w.Write(readFixture(t, p)); err != nil { + t.Fatalf("writing %s into the zip: %v", p, err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("closing the zip: %v", err) + } + return buf.Bytes() +} + +// admittingMirror builds the mirror filesystem the licence gate reads: one +// acquired publisher body per feed, its digest pinned in the manifest, and +// Anvil's own record beside it. +// +// The BODY is a fixture file. The manifest is generated because it carries a +// digest that can only be computed at run time — pinning a number written down +// by hand is the failure mode mirror/README.md is about. +func admittingMirror(t *testing.T, feeds config.FeedSet) fs.FS { + t.Helper() + body := readFixture(t, "fixtures/licence/cc0-1.0-excerpt.txt") + preamble := readFixture(t, "fixtures/licence/tier0-notes.md") + digest := digestOf(body) + + fsys := fstest.MapFS{} + var man strings.Builder + man.WriteString("# synthetic manifest, Lane A conformance harness\n") + man.WriteString("schema_version = 1\n") + man.WriteString("generated_utc = \"2026-02-06\"\n") + man.WriteString("generated_by = \"test/conformance/lanea\"\n") + + notes := map[config.LicenseTier]*strings.Builder{} + for _, f := range feeds.Feeds { + dir := f.MirrorDir + if dir == "" { + dir = f.ID + } + fmt.Fprintf(&man, "\n[[body]]\nfeed_id = %q\ntier = %d\ndir = %q\nspdx_id = %q\n"+ + "text_url = \"https://example.invalid/LICENSE\"\nsha256 = %q\n"+ + "claim_source = \"Lane A conformance harness fixture\"\n", + f.ID, f.LicenseTier.Int(), dir, f.LicenseSPDX, digest) + fsys[path.Join(license.TierDir(f.LicenseTier), dir, license.VerbatimFileName)] = + &fstest.MapFile{Data: body} + + b, ok := notes[f.LicenseTier] + if !ok { + b = &strings.Builder{} + b.Write(preamble) + notes[f.LicenseTier] = b + } + fmt.Fprintf(b, "\n%s\nSPDX-License-Identifier: %s\n\nAnvil's record: this synthetic source "+ + "is public domain and carries no obligation.\n%s\n", + license.BodyBeginMarker(f.ID), f.LicenseSPDX, license.BodyEndMarker(f.ID)) + } + for tier, b := range notes { + fsys[path.Join(license.TierDir(tier), license.NotesFileName)] = &fstest.MapFile{Data: []byte(b.String())} + } + fsys[license.ManifestFileName] = &fstest.MapFile{Data: []byte(man.String())} + return fsys +} + +func openCache(t *testing.T) *sql.DB { + t.Helper() + p := filepath.Join(t.TempDir(), "anvil-cache.sqlite") + db, err := cache.Open(context.Background(), p) + if err != nil { + t.Fatalf("opening the cache: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, err := cache.Migrate(context.Background(), db); err != nil { + t.Fatalf("migrating the cache: %v", err) + } + // Exit criterion 2: migrations are idempotent on a file already at the + // latest version. + if _, err := cache.Migrate(context.Background(), db); err != nil { + t.Fatalf("re-running migrations on an up-to-date cache: %v", err) + } + return db +} + +// --------------------------------------------------------------------------- +// Small helpers +// --------------------------------------------------------------------------- + +const ( + advisoryDumpSQL = ` +SELECT source, source_id, ifnull(cve_id,''), ifnull(published,''), ifnull(modified,''), state, + ifnull(tombstoned_at,''), ifnull(severity,''), ifnull(cvss_vector,''), ifnull(cvss_score,-1), + ifnull(epss_score,-1), ifnull(epss_as_of,''), kev, ifnull(license_spdx,''), + ifnull(license_manual_note,''), license_tier, anvil_trust, as_of, staleness_seconds, + parse_degraded, ifnull(data_version,''), hex(raw_json) +FROM advisory ORDER BY source, source_id` + + affectedDumpSQL = ` +SELECT source, source_id, ecosystem, package, ifnull(purl,''), ifnull(introduced,''), + ifnull(fixed,''), distro_backport +FROM affected ORDER BY source, source_id, ecosystem, package, ifnull(introduced,''), ifnull(fixed,'')` + + aliasDumpSQL = `SELECT cve_id, source, source_id FROM cve_alias ORDER BY cve_id, source, source_id` + + // advisory_fts is EXTERNAL-CONTENT: selecting its columns returns NULL by + // design, so the dump asks whether an index ROW EXISTS for each advisory. + // Comparing column values would compare NULL to NULL and pass over any + // divergence at all — which is how the A.8/A.14 tombstone divergence this + // harness found stayed invisible. + ftsDumpSQL = ` +SELECT a.source, a.source_id, a.state, + CASE WHEN f.rowid IS NULL THEN 'not-indexed' ELSE 'indexed' END AS indexed +FROM advisory a LEFT JOIN advisory_fts f ON f.rowid = a.rowid +ORDER BY a.source, a.source_id` + + feedStateDumpSQL = ` +SELECT feed_id, ifnull(etag,''), ifnull(last_modified,''), ifnull(watermark,''), + ifnull(last_ok_at,''), consecutive_failures, license_tier +FROM feed_state ORDER BY feed_id` +) + +func dump(t *testing.T, db *sql.DB, q string) []string { + t.Helper() + rows, err := db.Query(q) + if err != nil { + t.Fatalf("dump query: %v", err) + } + defer func() { _ = rows.Close() }() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("dump columns: %v", err) + } + var out []string + for rows.Next() { + cells := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range cells { + ptrs[i] = &cells[i] + } + if err := rows.Scan(ptrs...); err != nil { + t.Fatalf("dump scan: %v", err) + } + parts := make([]string, len(cols)) + for i, c := range cells { + parts[i] = cols[i] + "=" + fmt.Sprintf("%v", c) + } + out = append(out, strings.Join(parts, " ")) + } + if err := rows.Err(); err != nil { + t.Fatalf("dump: %v", err) + } + sort.Strings(out) + return out +} + +func count(t *testing.T, db *sql.DB, q string, args ...any) int { + t.Helper() + var n int + if err := db.QueryRow(q, args...).Scan(&n); err != nil { + t.Fatalf("counting with %q: %v", q, err) + } + return n +} + +func mergeCounts(dst map[string]int, src map[string]int) { + for k, v := range src { + dst[k] += v + } +} + +func sumCounts(m map[string]int) int { + n := 0 + for _, v := range m { + n += v + } + return n +} + +func digestOf(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func shortDigest(b []byte) string { return digestOf(b)[:16] } + +func containsString(ss []string, want string) bool { + for _, s := range ss { + if s == want { + return true + } + } + return false +} + +// hostSourceProblems is decideHostLink's tripwire set, written as a PURE +// FUNCTION over the source so that its negative control can call the shipping +// code instead of a re-implementation of it. Same argument as checkLedger. +// +// EVERY ENTRY POINTS THE SAME WAY: better evidence was available and the run +// did not use all of it. The tripwire that used to live here pointed the other +// way — it failed BECAUSE a package manager existed — and the answer to that +// one was to wire the collector in, not to keep firing. What must still fail +// is the collector's output being acquired and then quietly dropped, because +// that produces a ledger entry claiming a proof over nothing. +func hostSourceProblems(src hostInventorySource, collectedSubmitted int) []string { + var problems []string + if !src.real { + if len(src.collected) > 0 { + problems = append(problems, fmt.Sprintf( + "the host source says no package manager is present and carries %d collected "+ + "package(s). One of the two is wrong, and the ledger is about to describe "+ + "whichever the reader trusts less.", len(src.collected))) + } + return problems + } + if len(src.collected) == 0 { + problems = append(problems, "host.Collect returned no error on a host that HAS a package "+ + "manager, and ZERO packages. That is the silent-clean reading exit criterion 20 "+ + "forbids: a present package manager that enumerates nothing is a failed collection, "+ + "not a clean host. Read Inventory.Coverage for which family failed.") + } + if collectedSubmitted != len(src.collected) { + problems = append(problems, fmt.Sprintf( + "host.Collect returned %d package(s) but %d of them became comparator records; the "+ + "rest were dropped between the collector and the comparator (bridgeEcosystem "+ + "recognises only deb, rpm and apk). The ledger entry would then be a claim about "+ + "output the chain did not consume.", len(src.collected), collectedSubmitted)) + } + return problems +} + +// TestTheHostLedgerLineNamesWhichInventoryItUsed is the control for the one +// thing the host link exists to communicate. +// +// THE REAL PATH CANNOT RUN ON A WINDOWS DEVELOPMENT HOST — there is no dpkg, +// rpm or apk, so host.Collect refuses and the chain above takes the probe +// branch every time. Everything DOWNSTREAM of that call is host-independent +// and is exercised here against two synthetic sources: a run that collected +// real packages and a run that did not. They must not render the same, in the +// verdict or in the words, because a reader's only way to tell "this proved +// the collector" from "this proved a file" is the line the ledger prints. +func TestTheHostLedgerLineNamesWhichInventoryItUsed(t *testing.T) { + probe := []host.Package{{Ecosystem: host.EcosystemAPK, Name: "openssl", Version: "3.1.3-r0"}} + collected := []host.Package{ + {Ecosystem: host.EcosystemDeb, Name: "libc6", Version: "2.35-0ubuntu3.6", Arch: "amd64"}, + {Ecosystem: host.EcosystemDeb, Name: "openssl", Version: "3.0.2-0ubuntu1.15", Arch: "amd64"}, + } + collectedSrc := hostInventorySource{ + real: true, + collected: collected, + probe: probe, + inv: &host.Inventory{ + Packages: collected, + Coverage: []host.FamilyCoverage{ + {Ecosystem: host.EcosystemDeb, Status: host.FamilyCollected, Packages: 2}, + {Ecosystem: host.EcosystemRPM, Status: host.FamilyAbsent}, + {Ecosystem: host.EcosystemAPK, Status: host.FamilyAbsent}, + }, + }, + } + probeOnly := hostInventorySource{probe: probe, inv: &host.Inventory{}} + + collectedLed, probeLed := newLedger(), newLedger() + decideHostLink(t, collectedLed, collectedSrc, len(collectedSrc.collected), nil) + decideHostLink(t, probeLed, probeOnly, 0, nil) + + rv, ok := collectedLed.v[linkHostCollect] + if !ok || !rv.proven { + t.Fatalf("a run that collected %d real packages did not prove the host link: %+v", + len(collected), rv) + } + if !strings.Contains(rv.evidence, "host.Collect") { + t.Errorf("the PROVEN entry does not name host.Collect as its source: %q", rv.evidence) + } + pv := probeLed.v[linkHostCollect] + if pv.proven { + t.Fatalf("a run with no package manager claimed the host link PROVEN: %+v", pv) + } + if !strings.Contains(pv.missing, "CORPUS PROBE") { + t.Errorf("the UNPROVEN entry does not name the corpus probe as what it ran on: %q", pv.missing) + } + if rv.evidence == pv.missing { + t.Error("the two branches render identically, so the ledger cannot distinguish a run that " + + "proved the collector from one that proved only a file") + } + if p := checkLedger(newLedgerWith(linkHostCollect, rv)); len(p) != 0 { + t.Errorf("the PROVEN host entry does not satisfy the ledger's own validator: %v", p) + } + if p := checkLedger(newLedgerWith(linkHostCollect, pv)); len(p) != 0 { + t.Errorf("the UNPROVEN host entry does not satisfy the ledger's own validator: %v", p) + } + + // The determinism comparison reads label() and rows(); both must carry the + // distinction too, or two runs on different sources would compare equal. + if collectedSrc.label() == probeOnly.label() { + t.Errorf("both sources render the same label %q", collectedSrc.label()) + } + if d := diffChains( + chainOutput{hostSourceLabel: collectedSrc.label(), hostRows: collectedSrc.rows()}, + chainOutput{hostSourceLabel: probeOnly.label(), hostRows: probeOnly.rows()}, + ); len(d) == 0 { + t.Error("diffChains sees no difference between a real inventory and the probe alone, so " + + "the two-run comparison would not notice the chain switching sources mid-test") + } + if len(collectedSrc.submitted()) != len(collected)+len(probe) { + t.Errorf("the real path submitted %d packages, want %d collected + %d probe", + len(collectedSrc.submitted()), len(collected), len(probe)) + } +} + +// newLedgerWith builds a full ledger with one entry replaced, so a single +// verdict can be put through checkLedger. +func newLedgerWith(id linkID, v verdict) *ledger { + l := newLedger() + for _, other := range chainLinks { + l.proven(other, "evidence") + } + l.v[id] = v + return l +} + +// TestTheHostTripwiresFireOnDroppedEvidence is hostSourceProblems' negative +// control. A guard that has never rejected anything has not been tested, and +// these are the shapes in which better evidence is collected and then not used. +func TestTheHostTripwiresFireOnDroppedEvidence(t *testing.T) { + pkgs := []host.Package{ + {Ecosystem: host.EcosystemDeb, Name: "libc6", Version: "2.35-0ubuntu3.6"}, + {Ecosystem: host.EcosystemDeb, Name: "openssl", Version: "3.0.2-0ubuntu1.15"}, + } + cases := []struct { + name string + src hostInventorySource + submitted int + want string + }{ + { + name: "a real inventory fully submitted", + src: hostInventorySource{real: true, collected: pkgs}, + submitted: 2, + }, + { + name: "no package manager, probe only", + src: hostInventorySource{probe: pkgs[:1]}, + }, + { + name: "a present package manager that enumerated nothing", + src: hostInventorySource{real: true}, + submitted: 0, + want: "ZERO packages", + }, + { + name: "real packages collected and then dropped", + src: hostInventorySource{real: true, collected: pkgs}, + submitted: 1, + want: "the chain did not consume", + }, + { + name: "a probe-only source carrying collected packages", + src: hostInventorySource{collected: pkgs}, + submitted: 0, + want: "One of the two is wrong", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := hostSourceProblems(tc.src, tc.submitted) + if tc.want == "" { + if len(got) != 0 { + t.Fatalf("a well-formed source was flagged: %v", got) + } + return + } + if len(got) == 0 { + t.Fatalf("hostSourceProblems accepted %q", tc.name) + } + if !strings.Contains(strings.Join(got, "\n"), tc.want) { + t.Errorf("flagged %q but not for the expected reason: %v", tc.name, got) + } + }) + } +} + +// TestTheChainUsesTheBestAvailableHostEvidence is the tripwire that replaced +// the one CI fired, pointed the right way round. +// +// The old guard failed the run when a package manager appeared, because the +// chain was reading a hand-written file and the file had stopped being the +// best available evidence. Wiring host.Collect in answers that — but only for +// as long as the wiring stays. This test re-asks the machine DIRECTLY, without +// going through hostInventoryForChain, and fails if the chain's host input is +// not the best thing this machine can offer. +// +// It is deliberately not a statement about which branch is correct HERE: on a +// Windows development host it asserts the probe branch, on a Linux CI host it +// asserts the collected branch, and either way it asserts that the chain and +// the machine agree. +func TestTheChainUsesTheBestAvailableHostEvidence(t *testing.T) { + _, err := host.Collect(context.Background(), host.Options{ + Now: func() time.Time { return fixtureClock }, + }) + switch { + case err == nil, errors.Is(err, host.ErrNoPackageManager): + default: + t.Fatalf("host.Collect failed for a reason this harness does not understand: %v", err) + } + managerPresent := err == nil + + src := hostInventoryForChain(t) + if managerPresent != src.real { + t.Fatalf("this machine %s a supported package manager, and the chain's host source says "+ + "real=%v. The chain must run on host.Collect's output wherever it exists and on the "+ + "corpus probe only where it does not; a disagreement here means the ledger's host "+ + "line describes an inventory the chain did not use.", + map[bool]string{true: "HAS", false: "does not have"}[managerPresent], src.real) + } + if managerPresent && len(src.collected) == 0 { + t.Error("the chain took the real branch and collected nothing; see hostSourceProblems") + } + if !managerPresent && len(src.collected) != 0 { + t.Errorf("the chain took the probe branch and still carries %d collected package(s)", + len(src.collected)) + } + // The probe is present on BOTH branches and is never evidence about the + // collector. If it ever went missing the comparator and emission links + // would lose their only decidable host input and pass vacuously. + if len(src.probe) == 0 { + t.Error("the corpus probe is empty on this branch") + } + t.Logf("host evidence: %s", src.label()) +}