diff --git a/docker/lib/enrich-github-cpe.py b/docker/lib/enrich-github-cpe.py index 4dcbe026..9bddd1bb 100644 --- a/docker/lib/enrich-github-cpe.py +++ b/docker/lib/enrich-github-cpe.py @@ -33,15 +33,27 @@ import re import sys -# Curated (owner/repo) -> (vendor, product). Verified against NVD; each entry's -# CPE was confirmed to carry real, version-ranged NVD vulnerabilities. Keep this -# list short and hand-checked — do not add an entry by pattern-matching a repo -# name against its apparent vendor. +# Curated (owner/repo) -> (vendor, product) or (vendor, product, strip_prefix). +# Verified against NVD; each entry's CPE was confirmed to carry real, +# version-ranged NVD vulnerabilities. Keep this list short and hand-checked — +# do not add an entry by pattern-matching a repo name against its apparent +# vendor. +# +# strip_prefix, when present, is a literal prefix removed from the purl +# version before it goes into the cpe (e.g. golang/go tags its releases +# "go1.24.2", but NVD's version field is "1.24.2"). Confirmed with a direct +# grype CPE lookup: feeding the raw "go1.24.2" string in, grype's version +# comparator fails to parse it and treats the component as vulnerable to +# essentially every Go CVE ever filed regardless of version -- a false-positive +# flood, not a missing-match problem, so this is a correctness requirement, +# not cosmetic. GITHUB_CPE_MAP = { ("chromium", "chromium"): ("google", "chrome"), ("boostorg", "boost"): ("boost", "boost"), ("hunter-packages", "boost"): ("boost", "boost"), ("open5gs", "open5gs"): ("open5gs", "open5gs"), + ("golang", "go"): ("golang", "go", "go"), + ("davegamble", "cjson"): ("davegamble", "cjson"), ("torproject", "tor"): ("torproject", "tor"), } @@ -85,8 +97,16 @@ def derive_cpe(purl): entry = GITHUB_CPE_MAP.get((owner.lower(), repo.lower())) if not entry: return None - vendor, product = entry - version = _strip_repo_prefix(version, repo) + vendor, product = entry[0], entry[1] + if len(entry) > 2: + # Explicit strip_prefix (e.g. golang/go's "go" tag prefix, which has no + # separator _strip_repo_prefix could key off). + if version.startswith(entry[2]): + version = version[len(entry[2]):] + if not version or not _CPE_SAFE_VERSION.match(version): + return None + else: + version = _strip_repo_prefix(version, repo) return f"cpe:2.3:a:{vendor}:{product}:{version}:*:*:*:*:*:*:*" diff --git a/docker/lib/enrich-maven-cpe.py b/docker/lib/enrich-maven-cpe.py index 9c7f9084..d965b23c 100644 --- a/docker/lib/enrich-maven-cpe.py +++ b/docker/lib/enrich-maven-cpe.py @@ -62,10 +62,32 @@ "com.fasterxml.jackson.dataformat": ("fasterxml", "@artifact"), "org.springframework": ("vmware", "spring_framework"), "org.springframework.security": ("vmware", "spring_security"), + "org.springframework.boot": ("vmware", "spring_boot"), "commons-beanutils": ("apache", "commons_beanutils"), "commons-fileupload": ("apache", "commons_fileupload"), "commons-collections": ("apache", "commons_collections"), "org.json": ("stleary", "json-java"), + # The generic org.apache.* rule (vendor=2nd segment, product=last segment) + # only holds when NVD's own CPE reuses the groupId's tail. These groups + # need an explicit override because NVD's product (or vendor) differs from + # what the rule would derive — each verified against NVD's own cpeMatch + # data, not guessed. + "log4j": ("apache", "log4j"), # single-segment group: the generic rule needs 2+ segments, so this old-style Log4j 1.x groupId gets no CPE without an override + "org.apache.sshd": ("apache", "mina_sshd"), # rule would derive apache:sshd; NVD files it under mina_sshd + "org.apache.xmlgraphics": ("apache", "batik"), # rule would derive apache:xmlgraphics; this groupId is Batik's, and NVD's product is batik + "com.h2database": ("h2database", "h2"), # rule would derive h2database:h2database; NVD's product is h2 (the artifactId, not the groupId tail) + "rhino": ("mozilla", "rhino"), # single-segment group, map-only like log4j above + "net.sourceforge.nekohtml": ("cyberneko_html_project", "cyberneko_html"), # rule would derive sourceforge:nekohtml + "org.owasp.antisamy": ("antisamy_project", "antisamy"), # rule would derive owasp:antisamy + "org.postgresql": ("postgresql", "postgresql_jdbc_driver"), # rule would derive postgresql:postgresql + "org.quartz-scheduler": ("softwareag", "quartz"), # rule would derive quartz-scheduler:quartz-scheduler + "org.codehaus.woodstox": ("fasterxml", "woodstox"), # the pre-rename groupId; NVD files Woodstox CVEs under the current fasterxml vendor regardless of which groupId a given release used + "com.mchange": ("mchange", "c3p0"), # rule would derive mchange:mchange; NVD's product is the artifactId c3p0 + "io.opentelemetry.instrumentation": ("linuxfoundation", "opentelemetry_instrumentation_for_java"), # rule would derive opentelemetry:instrumentation + "io.undertow": ("redhat", "undertow"), # rule would derive undertow:undertow; NVD files it under redhat + "org.eclipse.angus": ("eclipse", "angus_mail"), # rule would derive eclipse:angus (missing the _mail suffix NVD's product carries) + "org.bouncycastle": ("bouncycastle", "bc-java"), # the modern (post-rename) Bouncy Castle groupId; rule would derive bouncycastle:bouncycastle + "bouncycastle": ("bouncycastle", "bouncy-castle-crypto-package"), # the legacy (pre-rename) groupId, single-segment so map-only; covers releases up to 1.35, distinct from bc-java above } # Versions with a CPE-unsafe shape are left alone: a ':' (cpe field separator), diff --git a/tests/test-postprocess.sh b/tests/test-postprocess.sh index 02ee64e0..8aa2e14c 100755 --- a/tests/test-postprocess.sh +++ b/tests/test-postprocess.sh @@ -1313,6 +1313,9 @@ cat > "$WORK/gh.json" <<'JSON' {"type":"library","name":"chromium","version":"133.0.6937.1","purl":"pkg:github/chromium/chromium@133.0.6937.1"}, {"type":"library","name":"boost","version":"v1.69.0-p0","purl":"pkg:github/hunter-packages/boost@v1.69.0-p0"}, {"type":"library","name":"open5gs","version":"2.6.5","purl":"pkg:github/open5gs/open5gs@2.6.5"}, + {"type":"library","name":"go","version":"go1.24.2","purl":"pkg:github/golang/go@go1.24.2"}, + {"type":"library","name":"go-bare-version","version":"1.24.2","purl":"pkg:github/golang/go@1.24.2"}, + {"type":"library","name":"cjson","version":"v1.7.16","purl":"pkg:github/davegamble/cjson@v1.7.16"}, {"type":"library","name":"tor","version":"tor-0.2.4.8-alpha","purl":"pkg:github/torproject/tor@tor-0.2.4.8-alpha"}, {"type":"library","name":"random-tool","version":"1.0.0","purl":"pkg:github/some-org/random-tool@1.0.0"}, {"type":"library","name":"has-cpe","version":"1.0","purl":"pkg:github/chromium/chromium@1.0","cpe":"cpe:2.3:a:preset:preset:1.0:*:*:*:*:*:*:*"}, @@ -1328,7 +1331,16 @@ gh_cpe_of() { jq -r --arg n "$1" '[.components[]|select(.name==$n)]|.[0].cpe // [ "$(gh_cpe_of boost)" = "cpe:2.3:a:boost:boost:v1.69.0-p0:*:*:*:*:*:*:*" ] && pass "hunter-packages/boost -> boost:boost (curated)" || fail "boost cpe='$(gh_cpe_of boost)'" # (c) curated map: open5gs/open5gs -> open5gs:open5gs. [ "$(gh_cpe_of open5gs)" = "cpe:2.3:a:open5gs:open5gs:2.6.5:*:*:*:*:*:*:*" ] && pass "open5gs/open5gs -> open5gs:open5gs (curated)" || fail "open5gs cpe='$(gh_cpe_of open5gs)'" -# (c2) curated map: torproject/tor -> torproject:tor, AND the release-tag prefix +# (c2) curated map with a strip_prefix: golang/go tags releases "go1.24.2"; +# NVD's version field has no "go" prefix, so it must be stripped before the cpe +# is built (feeding the raw tag in floods every Go CVE ever, see F-1c4i below). +[ "$(gh_cpe_of go)" = "cpe:2.3:a:golang:go:1.24.2:*:*:*:*:*:*:*" ] && pass "golang/go strips its \"go\" version-tag prefix before the cpe" || fail "go cpe='$(gh_cpe_of go)'" +# (c3) the strip is conditional -- a version that never had the prefix is left as-is. +[ "$(gh_cpe_of go-bare-version)" = "cpe:2.3:a:golang:go:1.24.2:*:*:*:*:*:*:*" ] && pass "golang/go with an already-bare version is untouched by the strip" || fail "go-bare-version cpe='$(gh_cpe_of go-bare-version)'" +# (c4) curated map, no strip_prefix needed: davegamble/cjson keeps its "v" tag +# prefix as-is (grype's comparator handles it fine, confirmed in F-1c4i). +[ "$(gh_cpe_of cjson)" = "cpe:2.3:a:davegamble:cjson:v1.7.16:*:*:*:*:*:*:*" ] && pass "davegamble/cjson -> davegamble:cjson (curated, no strip needed)" || fail "cjson cpe='$(gh_cpe_of cjson)'" +# (c5) curated map: torproject/tor -> torproject:tor, AND the release-tag prefix # ('tor-0.2.4.8-alpha') is stripped from the embedded version. Left in, that # prefix defeats grype's version-range comparison against NVD (verified: it # drops CVE recovery for this component from 30 to 2 — see enrich-github-cpe.py). @@ -1362,6 +1374,22 @@ if command -v grype >/dev/null 2>&1; then *,CVE-2016-9840,*) pass "grype CPE matcher recovers CVE-2016-9840 for boost (github-curated cpe)" ;; *) fail "CVE-2016-9840 not found in grype nvd:cpe results for boost" ;; esac + case ",$gh_cves," in + *,CVE-2025-4674,*) pass "grype CPE matcher recovers CVE-2025-4674 for golang/go (\"go\" prefix stripped)" ;; + *) fail "CVE-2025-4674 not found in grype nvd:cpe results for go" ;; + esac + # the false-positive-flood this strip prevents: every Go CVE ever, because + # grype's comparator can't parse "go1.24.2" as a version at all. + go_raw_n=$(GRYPE_BIN=grype python3 -c " +import subprocess, json +p = subprocess.run(['grype', 'cpe:2.3:a:golang:go:go1.24.2:*:*:*:*:*:*:*', '-o', 'json'], capture_output=True, text=True, timeout=60) +print(len(json.loads(p.stdout).get('matches', []))) +" 2>/dev/null) + [ "${go_raw_n:-0}" -gt 50 ] && pass "unstripped \"go1.24.2\" confirmed to flood matches (${go_raw_n}), motivating the strip_prefix fix" || echo " SKIP: could not reproduce the unstripped-version flood (got ${go_raw_n:-0} matches); not a failure, just unconfirmed on this grype DB build" + case ",$gh_cves," in + *,CVE-2023-50471,*) pass "grype CPE matcher recovers CVE-2023-50471 for davegamble/cjson" ;; + *) fail "CVE-2023-50471 not found in grype nvd:cpe results for cjson" ;; + esac case ",$gh_cves," in *,CVE-2013-7295,*) pass "grype CPE matcher recovers CVE-2013-7295 for tor (github-curated cpe, prefix stripped)" ;; *) fail "CVE-2013-7295 not found in grype nvd:cpe results for tor" ;; @@ -1441,6 +1469,74 @@ else echo " SKIP: grype not installed; skipping CVE-recovery regression (F-1c5)" fi +echo "== F-1c6: maven CPE enrichment — expanded curated map (groups where the generic rule derives the wrong product) ==" +# These groupIds all pass the generic org.apache.* (or 2-segment) rule and get +# SOME cpe, but the wrong one -- NVD's actual product differs from what the +# rule would derive (e.g. org.apache.sshd -> apache:sshd, but NVD's product is +# mina_sshd). Each entry below is verified against NVD's own cpeMatch data +# (docker/lib/enrich-maven-cpe.py's MAVEN_CPE_MAP comment has the per-entry +# rationale), not guessed. +cat > "$WORK/mvn-expanded.json" <<'JSON' +{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"type":"library","name":"log4j","version":"1.2.17","purl":"pkg:maven/log4j/log4j@1.2.17"}, + {"type":"library","name":"sshd-core","version":"2.12.1","purl":"pkg:maven/org.apache.sshd/sshd-core@2.12.1"}, + {"type":"library","name":"batik-css","version":"1.7","purl":"pkg:maven/org.apache.xmlgraphics/batik-css@1.7"}, + {"type":"library","name":"h2","version":"1.3.157","purl":"pkg:maven/com.h2database/h2@1.3.157"}, + {"type":"library","name":"js","version":"1.7R2","purl":"pkg:maven/rhino/js@1.7R2"}, + {"type":"library","name":"nekohtml","version":"1.9.12","purl":"pkg:maven/net.sourceforge.nekohtml/nekohtml@1.9.12"}, + {"type":"library","name":"antisamy","version":"1.4.3","purl":"pkg:maven/org.owasp.antisamy/antisamy@1.4.3"}, + {"type":"library","name":"postgresql","version":"42.1.4","purl":"pkg:maven/org.postgresql/postgresql@42.1.4"}, + {"type":"library","name":"quartz","version":"1.5.2","purl":"pkg:maven/org.quartz-scheduler/quartz@1.5.2"}, + {"type":"library","name":"spring-boot","version":"1.5.6.RELEASE","purl":"pkg:maven/org.springframework.boot/spring-boot@1.5.6.RELEASE"}, + {"type":"library","name":"woodstox-core-asl","version":"4.1.2","purl":"pkg:maven/org.codehaus.woodstox/woodstox-core-asl@4.1.2"}, + {"type":"library","name":"c3p0","version":"0.9.1.1","purl":"pkg:maven/com.mchange/c3p0@0.9.1.1"}, + {"type":"library","name":"opentelemetry-instrumentation-api","version":"2.10.0","purl":"pkg:maven/io.opentelemetry.instrumentation/opentelemetry-instrumentation-api@2.10.0"}, + {"type":"library","name":"undertow-core","version":"2.3.17.Final","purl":"pkg:maven/io.undertow/undertow-core@2.3.17.Final"}, + {"type":"library","name":"angus-mail","version":"2.0.3","purl":"pkg:maven/org.eclipse.angus/angus-mail@2.0.3"}, + {"type":"library","name":"bcprov-jdk15on","version":"1.36","purl":"pkg:maven/org.bouncycastle/bcprov-jdk15on@1.36"}, + {"type":"library","name":"bcmail-jdk14","version":"1.35","purl":"pkg:maven/bouncycastle/bcmail-jdk14@1.35"}]} +JSON +python3 "$MVNCPE" "$WORK/mvn-expanded.json" >/dev/null 2>&1 +exp_cpe_of() { jq -r --arg n "$1" '[.components[]|select(.name==$n)]|.[0].cpe // "NONE"' "$WORK/mvn-expanded.json"; } +[ "$(exp_cpe_of log4j)" = "cpe:2.3:a:apache:log4j:1.2.17:*:*:*:*:*:*:*" ] && pass "single-segment log4j groupId curated (apache:log4j)" || fail "log4j cpe='$(exp_cpe_of log4j)'" +[ "$(exp_cpe_of sshd-core)" = "cpe:2.3:a:apache:mina_sshd:2.12.1:*:*:*:*:*:*:*" ] && pass "org.apache.sshd curated (apache:mina_sshd, not apache:sshd)" || fail "sshd-core cpe='$(exp_cpe_of sshd-core)'" +[ "$(exp_cpe_of batik-css)" = "cpe:2.3:a:apache:batik:1.7:*:*:*:*:*:*:*" ] && pass "org.apache.xmlgraphics curated (apache:batik, not apache:xmlgraphics)" || fail "batik-css cpe='$(exp_cpe_of batik-css)'" +[ "$(exp_cpe_of h2)" = "cpe:2.3:a:h2database:h2:1.3.157:*:*:*:*:*:*:*" ] && pass "com.h2database curated (h2database:h2, not h2database:h2database)" || fail "h2 cpe='$(exp_cpe_of h2)'" +[ "$(exp_cpe_of js)" = "cpe:2.3:a:mozilla:rhino:1.7R2:*:*:*:*:*:*:*" ] && pass "single-segment rhino groupId curated (mozilla:rhino)" || fail "js cpe='$(exp_cpe_of js)'" +[ "$(exp_cpe_of nekohtml)" = "cpe:2.3:a:cyberneko_html_project:cyberneko_html:1.9.12:*:*:*:*:*:*:*" ] && pass "net.sourceforge.nekohtml curated (not sourceforge:nekohtml)" || fail "nekohtml cpe='$(exp_cpe_of nekohtml)'" +[ "$(exp_cpe_of antisamy)" = "cpe:2.3:a:antisamy_project:antisamy:1.4.3:*:*:*:*:*:*:*" ] && pass "org.owasp.antisamy curated (not owasp:antisamy)" || fail "antisamy cpe='$(exp_cpe_of antisamy)'" +[ "$(exp_cpe_of postgresql)" = "cpe:2.3:a:postgresql:postgresql_jdbc_driver:42.1.4:*:*:*:*:*:*:*" ] && pass "org.postgresql curated (postgresql_jdbc_driver, not postgresql)" || fail "postgresql cpe='$(exp_cpe_of postgresql)'" +[ "$(exp_cpe_of quartz)" = "cpe:2.3:a:softwareag:quartz:1.5.2:*:*:*:*:*:*:*" ] && pass "org.quartz-scheduler curated (softwareag:quartz)" || fail "quartz cpe='$(exp_cpe_of quartz)'" +[ "$(exp_cpe_of spring-boot)" = "cpe:2.3:a:vmware:spring_boot:1.5.6.RELEASE:*:*:*:*:*:*:*" ] && pass "org.springframework.boot curated (vmware:spring_boot)" || fail "spring-boot cpe='$(exp_cpe_of spring-boot)'" +[ "$(exp_cpe_of woodstox-core-asl)" = "cpe:2.3:a:fasterxml:woodstox:4.1.2:*:*:*:*:*:*:*" ] && pass "org.codehaus.woodstox curated to the post-rename vendor (fasterxml:woodstox)" || fail "woodstox-core-asl cpe='$(exp_cpe_of woodstox-core-asl)'" +[ "$(exp_cpe_of c3p0)" = "cpe:2.3:a:mchange:c3p0:0.9.1.1:*:*:*:*:*:*:*" ] && pass "com.mchange curated (mchange:c3p0, not mchange:mchange)" || fail "c3p0 cpe='$(exp_cpe_of c3p0)'" +[ "$(exp_cpe_of opentelemetry-instrumentation-api)" = "cpe:2.3:a:linuxfoundation:opentelemetry_instrumentation_for_java:2.10.0:*:*:*:*:*:*:*" ] && pass "io.opentelemetry.instrumentation curated" || fail "opentelemetry-instrumentation-api cpe='$(exp_cpe_of opentelemetry-instrumentation-api)'" +[ "$(exp_cpe_of undertow-core)" = "cpe:2.3:a:redhat:undertow:2.3.17.Final:*:*:*:*:*:*:*" ] && pass "io.undertow curated (redhat:undertow, not undertow:undertow)" || fail "undertow-core cpe='$(exp_cpe_of undertow-core)'" +[ "$(exp_cpe_of angus-mail)" = "cpe:2.3:a:eclipse:angus_mail:2.0.3:*:*:*:*:*:*:*" ] && pass "org.eclipse.angus curated (angus_mail, not angus)" || fail "angus-mail cpe='$(exp_cpe_of angus-mail)'" +[ "$(exp_cpe_of bcprov-jdk15on)" = "cpe:2.3:a:bouncycastle:bc-java:1.36:*:*:*:*:*:*:*" ] && pass "org.bouncycastle curated (bc-java, not bouncycastle:bouncycastle)" || fail "bcprov-jdk15on cpe='$(exp_cpe_of bcprov-jdk15on)'" +[ "$(exp_cpe_of bcmail-jdk14)" = "cpe:2.3:a:bouncycastle:bouncy-castle-crypto-package:1.35:*:*:*:*:*:*:*" ] && pass "legacy bouncycastle groupId curated (bouncy-castle-crypto-package)" || fail "bcmail-jdk14 cpe='$(exp_cpe_of bcmail-jdk14)'" +# idempotent. +cp "$WORK/mvn-expanded.json" "$WORK/mvn-expanded2.json"; python3 "$MVNCPE" "$WORK/mvn-expanded2.json" >/dev/null 2>&1 +diff -q "$WORK/mvn-expanded.json" "$WORK/mvn-expanded2.json" >/dev/null 2>&1 && pass "F-1c6 enrichment is idempotent" || fail "second run changed the SBOM" +# regression: feeding a couple of these through grype's CPE matcher actually +# recovers the real CVE, not just a syntactically-correct cpe string. +if command -v grype >/dev/null 2>&1; then + cat > "$WORK/exp-log4j.json" <<'JSON' +{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"type":"library","name":"log4j","version":"1.2.17","purl":"pkg:maven/log4j/log4j@1.2.17"}]} +JSON + python3 "$MVNCPE" "$WORK/exp-log4j.json" >/dev/null 2>&1 + python3 "$LIB/scan-nvd-cpe.py" "$WORK/exp-log4j.json" "$WORK/exp-log4j-out" >/dev/null 2>&1 + if [ -f "$WORK/exp-log4j-out_security_grype.json" ]; then + exp_log4j_n=$(jq '[.Results[0].Vulnerabilities[]] | length' "$WORK/exp-log4j-out_security_grype.json") + [ "${exp_log4j_n:-0}" -gt 0 ] && pass "grype CPE matcher recovers CVEs for log4j@1.2.17 (curated cpe)" || fail "no CVEs recovered for log4j@1.2.17" + else + echo " SKIP: grype produced no sidecar (offline DB unavailable?); skipping log4j CVE-recovery assertion" + fi +else + echo " SKIP: grype not installed; skipping CVE-recovery regression (F-1c6)" +fi + echo "== F-1d: NVD version filter (scan-nvd-cpe) — drops loose-range false positives ==" # The filter is what removes grype's over-broad nvd:cpe matches (a fixed-in-9.0.104 # Tomcat CVE that grype's DB matches to 7.0.50 because it dropped the >= 9.0.0 lower