test(engine): strengthen vulnerability consolidation assurance (#299)

Exercise matcher-order independence, conservative evidence merging, package isolation, and deterministic tie-breaking. Add a bounded native fuzz target and scaled benchmark, and include the target in the nightly fuzz matrix.

Author: bomly-guy

dev-docs/CI.md

@@ -110,6 +110,7 @@ Bomly runs native Go fuzzing nightly and on demand through the `Fuzz` workflow.
 
 - Node lockfile parsers for npm, pnpm, and Yarn
 - SPDX/CycloneDX SBOM JSON detection and decoding
+- Cross-matcher vulnerability alias consolidation and evidence preservation
 - SDK package URL canonicalization and transport JSON
 - Managed plugin archive-name and relative-path sanitizers
 

internal/engine/vulnerability_consolidation_test.go

@@ -1,6 +1,11 @@
 package engine
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strings"
 	"testing"
 
 	"github.com/bomly-dev/bomly-cli/sdk"
@@ -76,6 +81,341 @@ func TestConsolidateVulnerabilitiesUsesTransitiveAliasesButNotRelatedIDs(t *test
 	}
 }
 
+func TestEngineMatchConsolidationIsIndependentOfMatcherOrder(t *testing.T) {
+	const purl = "pkg:golang/example.test/module@v1.0.0"
+	contributions := []sdk.Vulnerability{
+		{
+			ID: "GHSA-aaaa-bbbb-cccc", Aliases: []string{" CVE-2026-0001 "},
+			Source: "first", FixedVersions: []string{"1.0.1"},
+		},
+		{
+			ID: "GO-2026-0001", Aliases: []string{"cve-2026-0001", "GHSA-aaaa-bbbb-cccc"},
+			Source: "second", Title: "Detailed advisory", Summary: "A detailed summary",
+			Details:        "A detailed description that makes this the canonical record.",
+			ParsedSeverity: sdk.SeverityHigh, FixState: sdk.FixStateNotFixed,
+		},
+		{
+			ID: "CVE-2026-0001", Aliases: []string{"GO-2026-0001"},
+			Source: "third", References: []sdk.Reference{{URL: "https://example.test/advisory"}},
+			KEVExploited: true,
+		},
+	}
+
+	var canonical []byte
+	for _, order := range permutations(len(contributions)) {
+		components := newTestRegistry()
+		for matcherIndex, contributionIndex := range order {
+			contribution := contributions[contributionIndex].Clone()
+			components.registerMatcher(fakeMatcher{
+				name: fmt.Sprintf("matcher-%d", matcherIndex),
+				run: func(registry *sdk.PackageRegistry) {
+					pkg := registry.Ensure(purl)
+					pkg.Vulnerabilities = append(pkg.Vulnerabilities, contribution.Clone())
+				},
+			})
+		}
+		result, err := NewEngine(components).Match(context.Background(), MatchRequest{
+			Graph: sdk.New(), Registry: sdk.NewPackageRegistry(),
+		})
+		if err != nil {
+			t.Fatalf("order %v: Match() error = %v", order, err)
+		}
+		pkg, ok := result.Registry.Get(purl)
+		if !ok || len(pkg.Vulnerabilities) != 1 {
+			t.Fatalf("order %v: package vulnerabilities = %#v", order, pkg)
+		}
+		if result.VulnerabilitiesConsolidated != 2 {
+			t.Fatalf("order %v: consolidated count = %d, want 2", order, result.VulnerabilitiesConsolidated)
+		}
+		encoded, err := json.Marshal(pkg.Vulnerabilities)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if canonical == nil {
+			canonical = encoded
+			continue
+		}
+		if string(encoded) != string(canonical) {
+			t.Fatalf("matcher order changed consolidation:\nfirst: %s\norder %v: %s", canonical, order, encoded)
+		}
+	}
+}
+
+func TestConsolidateVulnerabilitiesPreservesAllEvidenceConservatively(t *testing.T) {
+	hops := 1
+	values := []sdk.Vulnerability{
+		{
+			ID:         "GO-2026-0002",
+			Aliases:    []string{"CVE-2026-0002"},
+			Related:    []string{"CVE-2026-9999"},
+			Summary:    "rich summary",
+			Details:    "rich details",
+			Published:  "2026-01-01",
+			Modified:   "2026-02-01",
+			Source:     "osv",
+			DataSource: "https://example.test/osv",
+			Namespace:  "go",
+			Title:      "rich title",
+			Reasons:    []string{"reason one"},
+			Severity: []sdk.Severity{{
+				Type: sdk.SeverityTypeCVSSV3, Score: "CVSS:3.1/example",
+			}},
+			Affected: []sdk.Affected{{
+				Versions: []string{"1.0.0"},
+				Ranges: []sdk.VersionRange{{
+					Type:   sdk.VersionRangeTypeSemver,
+					Events: []sdk.RangeEvent{{Introduced: "0"}, {Fixed: "1.0.1"}},
+				}},
+				DatabaseSpecific: map[string]any{"range": "primary"},
+			}},
+			References:           []sdk.Reference{{URL: "https://example.test/one", Type: sdk.ReferenceTypeAdvisory}},
+			ParsedSeverity:       sdk.SeverityHigh,
+			SeveritySource:       "osv",
+			CVSS:                 []sdk.CVSSScore{{Vector: "vector-one", Score: 8.1}},
+			EPSS:                 []sdk.EPSSScore{{CVE: "CVE-2026-0002", EPSS: 0.4}},
+			CWEs:                 []sdk.CWE{{ID: "CWE-20"}},
+			KnownExploited:       []sdk.KnownExploited{{CVE: "CVE-2026-0002", URLs: []string{"https://example.test/kev"}}},
+			RiskScore:            70,
+			FixState:             sdk.FixStateFixed,
+			FixedIn:              "1.0.1",
+			FixedVersions:        []string{"1.0.1"},
+			FixAvailable:         []sdk.FixAvailable{{Version: "1.0.1", Kind: sdk.FixAvailableFirstObserved}},
+			AffectedVersionRange: "< 1.0.1",
+			CPEs:                 []string{"cpe:/a:example:module"},
+			AffectedSymbols: []sdk.AffectedSymbol{{
+				Symbol: "Parse", Kind: sdk.SymbolKindFunction,
+				Definition: &sdk.SourcePosition{File: "parse.go", Line: 10},
+			}},
+			DatabaseSpecific: map[string]any{"primary": true},
+			Reachability: &sdk.Reachability{
+				Status: sdk.ReachabilityUnreachable, Tier: sdk.TierSymbol,
+				Analyzer: "go", Hops: &hops,
+			},
+		},
+		{
+			ID:             "GHSA-dddd-eeee-ffff",
+			Aliases:        []string{"cve-2026-0002"},
+			Related:        []string{"CVE-2026-8888"},
+			Withdrawn:      "2026-03-01",
+			Reasons:        []string{"reason two"},
+			References:     []sdk.Reference{{URL: "https://example.test/two"}},
+			ParsedSeverity: sdk.SeverityCritical,
+			CVSS:           []sdk.CVSSScore{{Vector: "vector-two", Score: 9.8}},
+			EPSS:           []sdk.EPSSScore{{CVE: "CVE-2026-0002", EPSS: 0.9}},
+			CWEs:           []sdk.CWE{{ID: "CWE-787"}},
+			KEVExploited:   true,
+			RiskScore:      95,
+			FixState:       sdk.FixStateWontFix,
+			FixedVersions:  []string{"1.0.2"},
+			FixAvailable:   []sdk.FixAvailable{{Version: "1.0.2"}},
+			CPEs:           []string{"cpe:/a:example:other"},
+			DatabaseSpecific: map[string]any{
+				"secondary": true,
+				"primary":   false,
+			},
+			Reachability: &sdk.Reachability{Status: sdk.ReachabilityReachable, Tier: sdk.TierPackage},
+		},
+	}
+	before, err := json.Marshal(values)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got := consolidateVulnerabilities(values)
+	if len(got) != 1 {
+		t.Fatalf("consolidated vulnerabilities = %d, want 1", len(got))
+	}
+	merged := got[0]
+	if merged.ID != "GO-2026-0002" || merged.Source != "osv" {
+		t.Fatalf("canonical record = %#v", merged)
+	}
+	if merged.Withdrawn != "2026-03-01" {
+		t.Fatalf("missing supplementary scalar evidence: %#v", merged)
+	}
+	if merged.ParsedSeverity != sdk.SeverityCritical || merged.RiskScore != 95 ||
+		merged.FixState != sdk.FixStateWontFix || !merged.KEVExploited {
+		t.Fatalf("non-conservative conflict result: %#v", merged)
+	}
+	if merged.Reachability == nil || merged.Reachability.Status != sdk.ReachabilityReachable {
+		t.Fatalf("reachability = %#v, want reachable", merged.Reachability)
+	}
+	for name, count := range map[string]int{
+		"related": len(merged.Related), "reasons": len(merged.Reasons),
+		"references": len(merged.References), "cvss": len(merged.CVSS),
+		"epss": len(merged.EPSS), "cwes": len(merged.CWEs),
+		"fixed_versions": len(merged.FixedVersions), "fix_available": len(merged.FixAvailable),
+		"cpes": len(merged.CPEs),
+	} {
+		if count != 2 {
+			t.Errorf("%s count = %d, want 2", name, count)
+		}
+	}
+	if len(merged.Severity) != 1 || len(merged.Affected) != 1 ||
+		len(merged.KnownExploited) != 1 || len(merged.AffectedSymbols) != 1 {
+		t.Fatalf("structured evidence was lost: %#v", merged)
+	}
+	if merged.DatabaseSpecific["primary"] != true || merged.DatabaseSpecific["secondary"] != true {
+		t.Fatalf("database_specific merge = %#v", merged.DatabaseSpecific)
+	}
+
+	merged.Aliases[0] = "mutated"
+	merged.Affected[0].Versions[0] = "mutated"
+	merged.KnownExploited[0].URLs[0] = "mutated"
+	merged.AffectedSymbols[0].Definition.File = "mutated"
+	merged.DatabaseSpecific["primary"] = "mutated"
+	merged.Reachability.Status = sdk.ReachabilityUnknown
+	after, err := json.Marshal(values)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(after) != string(before) {
+		t.Fatalf("consolidation output aliases input:\nbefore: %s\nafter:  %s", before, after)
+	}
+}
+
+func TestConsolidateRegistryVulnerabilitiesKeepsPackagesAndVersionsIsolated(t *testing.T) {
+	registry := sdk.NewPackageRegistry()
+	for _, purl := range []string{
+		"pkg:npm/example@1.0.0",
+		"pkg:npm/example@2.0.0",
+		"pkg:npm/other@1.0.0",
+	} {
+		registry.Ensure(purl).Vulnerabilities = []sdk.Vulnerability{
+			{ID: "CVE-2026-0003", Aliases: []string{"GHSA-1111-2222-3333"}},
+			{ID: "GHSA-1111-2222-3333"},
+		}
+	}
+
+	before, after := consolidateRegistryVulnerabilities(registry)
+	if before != 6 || after != 3 {
+		t.Fatalf("registry counts = %d -> %d, want 6 -> 3", before, after)
+	}
+	for _, pkg := range registry.All() {
+		if len(pkg.Vulnerabilities) != 1 {
+			t.Fatalf("%s vulnerabilities = %#v", pkg.ID, pkg.Vulnerabilities)
+		}
+	}
+}
+
+func TestConsolidateVulnerabilitiesUsesDeterministicRichnessTieBreakers(t *testing.T) {
+	values := []sdk.Vulnerability{
+		{ID: "Z-ADVISORY", Aliases: []string{"SHARED"}, Source: "z-source", Summary: "z"},
+		{ID: "A-ADVISORY", Aliases: []string{"shared"}, Source: "a-source", Summary: "a"},
+	}
+	for _, order := range [][]sdk.Vulnerability{values, {values[1], values[0]}} {
+		got := consolidateVulnerabilities(order)
+		if len(got) != 1 || got[0].ID != "A-ADVISORY" || got[0].Summary != "a" {
+			t.Fatalf("tie-break result = %#v", got)
+		}
+	}
+}
+
+func FuzzConsolidateVulnerabilities(f *testing.F) {
+	for _, seed := range [][]sdk.Vulnerability{
+		{{ID: "ADV-1", Aliases: []string{"ALIAS-1"}}, {ID: "ADV-2", Aliases: []string{"ADV-1"}}},
+		{{ID: "ADV-1", Related: []string{"ADV-2"}}, {ID: "ADV-2"}},
+		{{ID: " A "}, {ID: "a", Aliases: []string{"B"}}, {ID: "B"}},
+	} {
+		data, err := json.Marshal(seed)
+		if err != nil {
+			f.Fatal(err)
+		}
+		f.Add(data)
+	}
+
+	f.Fuzz(func(t *testing.T, data []byte) {
+		if len(data) > 64<<10 {
+			return
+		}
+		var values []sdk.Vulnerability
+		if err := json.Unmarshal(data, &values); err != nil || len(values) > 64 {
+			return
+		}
+		before, err := json.Marshal(values)
+		if err != nil {
+			return
+		}
+		got := consolidateVulnerabilities(values)
+		if len(got) > len(values) {
+			t.Fatalf("consolidation grew input from %d to %d", len(values), len(got))
+		}
+		for left := range got {
+			for right := left + 1; right < len(got); right++ {
+				if identitySetsOverlap(vulnerabilityIdentities(got[left]), vulnerabilityIdentities(got[right])) {
+					t.Fatalf("output groups overlap: %#v and %#v", got[left], got[right])
+				}
+			}
+		}
+		again := consolidateVulnerabilities(got)
+		firstJSON, firstErr := json.Marshal(got)
+		secondJSON, secondErr := json.Marshal(again)
+		if firstErr != nil || secondErr != nil || string(firstJSON) != string(secondJSON) {
+			t.Fatalf("consolidation is not idempotent:\nfirst:  %s\nsecond: %s", firstJSON, secondJSON)
+		}
+		after, err := json.Marshal(values)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if string(after) != string(before) {
+			t.Fatal("consolidation mutated its input")
+		}
+	})
+}
+
+func BenchmarkConsolidateVulnerabilities(b *testing.B) {
+	for _, size := range []int{10, 100, 1000} {
+		values := benchmarkVulnerabilities(size)
+		b.Run(fmt.Sprintf("records-%d", size), func(b *testing.B) {
+			b.ReportAllocs()
+			for range b.N {
+				result := consolidateVulnerabilities(values)
+				if len(result) != (size+1)/2 {
+					b.Fatalf("result count = %d, want %d", len(result), (size+1)/2)
+				}
+			}
+		})
+	}
+}
+
+func benchmarkVulnerabilities(size int) []sdk.Vulnerability {
+	values := make([]sdk.Vulnerability, 0, size)
+	for idx := range size {
+		component := idx / 2
+		id := fmt.Sprintf("ADV-%05d-%d", component, idx%2)
+		alias := fmt.Sprintf("CVE-2026-%05d", component)
+		values = append(values, sdk.Vulnerability{
+			ID: id, Aliases: []string{alias}, Source: fmt.Sprintf("source-%d", idx%3),
+			Summary: strings.Repeat("x", idx%17),
+		})
+	}
+	return values
+}
+
+func permutations(size int) [][]int {
+	if size == 0 {
+		return [][]int{{}}
+	}
+	var out [][]int
+	var visit func([]int, []int)
+	visit = func(prefix, remaining []int) {
+		if len(remaining) == 0 {
+			out = append(out, slices.Clone(prefix))
+			return
+		}
+		for idx, value := range remaining {
+			next := append(slices.Clone(prefix), value)
+			tail := append(slices.Clone(remaining[:idx]), remaining[idx+1:]...)
+			visit(next, tail)
+		}
+	}
+	remaining := make([]int, size)
+	for idx := range size {
+		remaining[idx] = idx
+	}
+	visit(nil, remaining)
+	return out
+}
+
 func sameStrings(left, right []string) bool {
 	if len(left) != len(right) {
 		return false

scripts/run-fuzz.sh

@@ -8,6 +8,7 @@ targets=(
   "github.com/bomly-dev/bomly-cli/internal/detectors/node/pnpm FuzzDepGraphFromPNPMLockfile"
   "github.com/bomly-dev/bomly-cli/internal/detectors/node/yarn FuzzDepGraphFromYarnLockfile"
   "github.com/bomly-dev/bomly-cli/internal/sbom FuzzUnmarshalAutoJSON"
+  "github.com/bomly-dev/bomly-cli/internal/engine FuzzConsolidateVulnerabilities"
   "github.com/bomly-dev/bomly-cli/sdk FuzzCanonicalizePackageURL"
   "github.com/bomly-dev/bomly-cli/sdk FuzzGraphJSON"
   "github.com/bomly-dev/bomly-cli/sdk FuzzPackageRegistryJSON"