diff --git a/README.md b/README.md index c2470cc..e8631e6 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,28 @@ A python script that translates machine-readable SBOMs into a format suitable fo This script merges SBOMs generated from Github's Dependabot tool and outputs it as a human readable excel file. +Supported SBOM formats: +- SPDX JSON v2.3 +- CycloneDX JSON v1.6 + +This tool can also process SPDX SBOMs generated by VCPKG and combine them into a single SBOM with CPE entries. VCPKG is working on natively supporting security references in their SBOMs, but until then this tool can help fill the gap. See https://github.com/package-url/purl-spec/pull/562 and https://github.com/microsoft/vcpkg/discussions/36078 + + # Usage +If [uv](https://docs.astral.sh/uv/) is installed, you can run the tool without setting up an environment. Simply execute the script with the required arguments. + ``` -usage: gen_sbom.py [-h] client_name input_directory output_file +Usage: gen_sbom.py [OPTIONS] INPUT_DIRECTORY_PATH OUTPUT_FILE_PATH -positional arguments: - input_directory Github SBOM json files directory. - output_file Output combined SBOM excel file path. + Generate a combined SBOM from multiple SPDX and CycloneDX SBOMs in the input + directory. -optional arguments: - -h, --help show this help message and exit \ No newline at end of file +Options: + --verbose Enable verbose logging. + --author-name TEXT Override the Author Name. + --vcpkg Combine VCPKG SBOMs. + --spdx-output-file FILE Output combined SPDX SBOM file path (for VCPKG + only). + --help Show this message and exit. +``` diff --git a/gen_sbom.py b/gen_sbom.py old mode 100644 new mode 100755 index cfc3b42..d707c5b --- a/gen_sbom.py +++ b/gen_sbom.py @@ -1,101 +1,435 @@ -import os -import json -import argparse +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "click~=8.1.8", +# "openpyxl~=3.1.5", +# "packaging~=25.0", +# "pydantic~=2.11.9", +# "pyyaml~=6.0.3", +# "types-PyYAML", +# ] +# /// + import logging +from collections.abc import Generator, Sequence +from datetime import datetime +from functools import reduce +from pathlib import Path +from typing import Annotated, Any, Literal, Protocol +import click import openpyxl - +import yaml +from packaging import version +from pydantic import ( + AliasPath, + BaseModel, + ConfigDict, + Field, + computed_field, + field_serializer, +) logger = logging.getLogger(__name__) -def newer(p1, p2): - return p1 if p1["versionInfo"] > p2["versionInfo"] else p2 +class BaseSBOM(BaseModel): + def to_fda_records(self, author: str | None) -> Generator["FDARecord"]: + raise NotImplementedError + + +class SPDXRef(BaseModel): + # referenceCategory: str + referenceType: str + referenceLocator: str + + +class SPDXPackage(BaseModel): + _default_supplier = "Open-source software" + model_config = ConfigDict(populate_by_name=True) + SPDXID: str + name: str + + # The versionInfo field is optional (https://spdx.github.io/spdx-spec/v2.3/package-information/#73-package-version-field). + # VCPKG generated SBOMs sometimes have versionInfo missing + # versionInfo: str | None = None + version: Annotated[str, Field(alias="versionInfo")] = "" + supplier: str = Field(default=_default_supplier) + externalRefs: list[SPDXRef] = Field(default_factory=list) + + @field_serializer("supplier") + def supplier_serializer(self, supplier: str) -> str: + if supplier.startswith("Organization: ") or supplier.startswith("Person: "): + return supplier + return "NOASSERTION" + + @computed_field + def purl(self) -> str | None: + for ref in self.externalRefs: + if ref.referenceType == "purl": + return ref.referenceLocator + return None + + +class SPDXCreationInfo(BaseModel): + creators: list[str] + created: str + + +class SPDX2_3(BaseSBOM): + spdxVersion: Literal["SPDX-2.3", "SPDX-2.2"] + SPDXID: str + name: str + creationInfo: SPDXCreationInfo + packages: list[SPDXPackage] + + _is_vcpkg: bool = False + + def model_post_init(self, __context: Any): + self._is_vcpkg = any( + "tool: vcpkg" in creator.lower() for creator in self.creationInfo.creators + ) + + def to_fda_records(self, author: str | None) -> Generator["FDARecord"]: + """Convert SPDX packages to FDARecords.""" + for p in self.packages: + if not p.version: + logger.warning( + f"Package {p.name} ({p.SPDXID}) is missing versionInfo, skipping" + ) + continue + # Skip binary package provided in vcpkg + if self._is_vcpkg: + if p.SPDXID == "SPDXRef-binary": + continue + + unique_id = p.SPDXID # fallback, prefer purl, then cpe + for ref in p.externalRefs: + if ref.referenceType == "purl": + unique_id = ref.referenceLocator + break + elif ( + ref.referenceType == "SECURITY" + and ref.referenceLocator.startswith("cpe:2.3:") + ): + unique_id = ref.referenceLocator + + yield FDARecord( + author=author if author else ", ".join(self.creationInfo.creators), + timestamp=self.creationInfo.created, + supplier=p.supplier, + name=p.name, + version=p.version, + unique_identifier=unique_id, + ) + + +def enrich_vcpkg(sbom: SPDX2_3): + """Enrich VCPKG SPDX SBOM with cpe and supplier info from vcpkg.yml.""" + vcpkg_yaml_path = Path(__file__).parent / "vcpkg.yml" + if not vcpkg_yaml_path.is_file(): + logger.error(f"vcpkg.yml not found at {vcpkg_yaml_path}, cannot enrich SBOM") + return + + with vcpkg_yaml_path.open() as f: + vcpkg_data = yaml.safe_load(f) + + def follow_link(pkg_name: str) -> str | None: + """Follow links in vcpkg.yml to get the actual package name. (recursive)""" + pkg = vcpkg_data.get(pkg_name) + if not pkg: + return None + if "aka" in pkg: + return follow_link(pkg["aka"]) + return pkg_name + + for p in sbom.packages: + name = follow_link(p.name) + if name != p.name: + logger.info(f"Using {name} for {p.name}") + if not name: + logger.error(f"Package {p.name} not found in vcpkg.yml, cannot enrich") + continue + p.SPDXID = f"SPDXRef-{name}" + p.supplier = "vcpkg" -def create_sbom_packages_dict(sbom): - packages = {} - for p in sbom["packages"]: - name = p["name"] - packages[name] = newer(p, packages[name]) if name in packages else p - return packages + if "cpe" in vcpkg_data[name]: + version = p.version.split("#")[0] # Remove vcpkg revision number + cpe = vcpkg_data[name]["cpe"] + vendor = cpe.split(":")[3:4][0] # Extract vendor from cpe + p.supplier += f", {vendor}" + p.externalRefs.append( + SPDXRef( + referenceType="SECURITY", + referenceLocator=f"{cpe}:{version}", + ) + ) + logger.info(f"Added CPE {cpe} to package {p.name}") + # deduplicate by SPDXID + unique_packages = {} + for p in sbom.packages: + if p.SPDXID not in unique_packages: + unique_packages[p.SPDXID] = p + else: + logger.warning( + f"Duplicate package {p.SPDXID} found, keeping the first occurrence" + ) + sbom.packages = list(unique_packages.values()) -def merge_sboms(sbom1, sbom2): - packages1 = create_sbom_packages_dict(sbom1) - packages2 = create_sbom_packages_dict(sbom2) - merged_packages = [] - for name, p in packages1.items(): - merged_packages.append(newer(p, packages2[name]) if name in packages2 else p) - for name, p in packages2.items(): - merged_packages.extend([] if name in packages1 else [p]) - sbom1["packages"] = merged_packages - sbom1["creationInfo"]["created"] = max( - sbom1["creationInfo"]["created"], sbom2["creationInfo"]["created"] +class CycloneComponent(BaseModel): + name: str + version: str + purl: str | None = None + supplier: str = Field( + validation_alias=AliasPath("supplier", "name"), default="Open-source software" ) - return sbom1 + bom_ref: str = Field(alias="bom-ref") -excel_header = [ - "Author Name", - "Timestamp", - "Supplier Name", - "Component Name", - "Version String", - "Unique Identifier", - "Relationship", -] +class CycloneMetadata(BaseModel): + timestamp: str + tools: list[dict] = Field(validation_alias=AliasPath("tools", "components")) + @computed_field + def author(self) -> str: + return ", ".join( + f"{tool['type']}: {tool['name']}-{tool['version']}" for tool in self.tools + ) -def save_as_xlsx(sbom, output_file_path, author_name=None): + +class Cyclone1_6(BaseSBOM): + bomFormat: Literal["CycloneDX"] + specVersion: Literal["1.6"] + version: int + metadata: CycloneMetadata + components: list[CycloneComponent] | None = None + + def to_fda_records(self, author: str | None) -> Generator["FDARecord"]: + """Convert CycloneDX components to FDARecords.""" + if self.components: + for c in self.components: + yield FDARecord( + author=author if author else self.metadata.author, # type: ignore[arg-type] + timestamp=self.metadata.timestamp, + supplier=c.supplier, + name=c.name, + version=c.version, + unique_identifier=c.purl if c.purl else c.bom_ref, + ) + + +class FDARecord(BaseModel): + """FDA required fields.""" + + author: str + timestamp: str + supplier: str = "Open-source software" + name: str + version: str + unique_identifier: str + relationship: Literal["Is contained by"] = "Is contained by" + + +class CommonRecordProtocol(Protocol): + version: str + name: str + supplier: str + + +def newer[T: CommonRecordProtocol](p1: T, p2: T) -> T: + """Return the package with the newer version using semantic version comparison.""" + if p1.version == p2.version: + return p2 # Arbitrary choice if versions are equal + try: + v1 = version.parse(p1.version) + v2 = version.parse(p2.version) + return p1 if v1 > v2 else p2 + except Exception as e: + # Fallback to string comparison if version parsing fails + logger.warning( + f"Failed to parse versions '{p1.version}' or '{p2.version}': {e}" + ) + return p1 if p1.version > p2.version else p2 + + +def merge_sboms[T: CommonRecordProtocol]( + sbom1: Sequence[T], sbom2: Sequence[T] +) -> list[T]: + """Merge two SBOMs, keeping the newest version of each package.""" + records = {(r.name, r.supplier): r for r in sbom1} + for r in sbom2: + key = (r.name, r.supplier) + if key in records: + records[key] = newer(records[key], r) + else: + records[key] = r + return list(records.values()) + + +def deduplicate(records: list[FDARecord]) -> list[FDARecord]: + """Deduplicate records by unique_identifier, keeping the first occurrence.""" + seen = set() + deduped = [] + for r in records: + key = r.unique_identifier + if key not in seen: + deduped.append(r) + seen.add(key) + else: + logger.warning(f"Duplicate record found for unique_identifier: {key}") + return deduped + + +def save_as_xlsx(bom: list[FDARecord], output_file_path: Path | str): + """Save the BOM as an excel file.""" + excel_header = [ + "Author Name", + "Timestamp", + "Supplier Name", + "Component Name", + "Version String", + "Unique Identifier", + "Relationship", + ] wb = openpyxl.Workbook() ws = wb.active ws.append(excel_header) - for p in sbom["packages"]: + for r in bom: + ws.append( + [ + r.author, + r.timestamp, + r.supplier, + r.name, + r.version, + r.unique_identifier, + r.relationship, + ] + ) + wb.save(output_file_path) + + +@click.command() +@click.argument( + "input_directory_path", + type=click.Path(exists=True, file_okay=False, path_type=Path), +) +@click.argument("output_file_path", type=click.Path(dir_okay=False, path_type=Path)) +@click.option("--verbose", is_flag=True, help="Enable verbose logging.") +@click.option("--author-name", type=str, default=None, help="Override the Author Name.") +@click.option("--vcpkg", is_flag=True, help="Combine VCPKG SBOMs.") +@click.option( + "--spdx-output-file", + type=click.Path(dir_okay=False, path_type=Path), + default=None, + help="Output combined SPDX SBOM file path (for VCPKG only).", +) +def main( + input_directory_path: Path, + output_file_path: Path, + author_name: str | None = None, + vcpkg: bool = False, + spdx_output_file: Path | None = None, + verbose: bool = False, +): + """Generate a combined SBOM from multiple SPDX and CycloneDX SBOMs in the input directory.""" + if verbose: + logging.getLogger().setLevel(logging.DEBUG) + if vcpkg: + return gen_sbom_vcpkg( + input_directory_path, output_file_path, author_name, spdx_output_file + ) + else: + gen_sbom(input_directory_path, output_file_path, author_name) - if "versionInfo" not in p: - # The versionInfo field is optional (https://spdx.github.io/spdx-spec/v2.3/package-information/#73-package-version-field). - # I haven't yet seen a case where it is missing and it should be included in the human-readable SBOM - logger.warning("Skipping '%s' due to no versionInfo field", p["name"]) + +def gen_sbom( + input_directory_path: Path, output_file_path: Path, author_name: str | None = None +): + """Generate a combined SBOM from multiple SPDX and CycloneDX SBOMs in the input directory.""" + bom_parsers: list[type[BaseSBOM]] = [SPDX2_3] # , Cyclone1_6] + boms: list[list[FDARecord]] = [] + + for bom_file in input_directory_path.glob("**/*.json"): + if not bom_file.is_file(): continue + logger.info(f"Processing {bom_file}") + for bom_parser in bom_parsers: + try: + bom = bom_parser.model_validate_json(bom_file.read_text()) + logger.info(f"Parsed {bom_file} as {bom_parser.__name__}") + boms.append(list(bom.to_fda_records(author_name))) + break + except Exception as e: + logger.exception(f"Failed to parse {bom_file} as {bom_parser.__name__}") + logger.debug( + f"Failed to parse {bom_file} as {bom_parser.__name__}: {e}" + ) + else: + logger.error(f"Failed to parse {bom_file} with all known parsers") + raise ValueError(f"Unknown BOM format in {bom_file}") - row = [ - author_name if author_name else ", ".join(sbom["creationInfo"]["creators"]), - sbom["creationInfo"]["created"], - p.get("supplier", "Open-source software"), - p["name"], - p["versionInfo"], - get_purl(p) or p["SPDXID"], - "Is contained by", - ] - ws.append(row) - wb.save(output_file_path) + merged_bom: list[FDARecord] = reduce(merge_sboms, boms, []) + save_as_xlsx(merged_bom, output_file_path) + # Check for duplicates (side effect: log warnings) + deduplicate(merged_bom) -def get_purl(p): - if "externalRefs" in p: - for ref in p["externalRefs"]: - if ref["referenceType"] == "purl": - return ref["referenceLocator"] - return None +def gen_sbom_vcpkg( + input_directory_path: Path, + output_file_path: Path, + author_name: str | None = None, + spdx_output_file: Path | None = None, +): + boms: list[SPDXPackage] = [] + for bom_file in input_directory_path.glob("**/*.json"): + if not bom_file.is_file(): + continue + logger.info(f"Processing {bom_file}") + try: + bom = SPDX2_3.model_validate_json(bom_file.read_text()) + if bom._is_vcpkg: + boms = merge_sboms( + boms, [p for p in bom.packages if p.SPDXID == "SPDXRef-port"] + ) + else: + logger.warning(f"Skipping non-vcpkg SBOM: {bom_file}") + except Exception as e: + logger.exception(f"Failed to parse {bom_file} as SPDX2_3") + logger.debug(f"Failed to parse {bom_file} as SPDX2_3: {e}") + raise ValueError(f"Unknown BOM format in {bom_file}") + final_bom = SPDX2_3( + spdxVersion="SPDX-2.3", + SPDXID="SPDXRef-DOCUMENT", + name="Combined VCPKG", + creationInfo=SPDXCreationInfo.model_validate( + { + "creators": [author_name] + if author_name + else [ + "Tool: github.com/innolitics/fda-readable-sbom", + "Tool: https://github.com/microsoft/vcpkg", + ], + "created": datetime.now().isoformat() + "Z", + } + ), + packages=boms, + ) -def gen_sbom(input_directory_path, output_file_path, author_name=None): - master_sbom = {} - for file_name in os.listdir(input_directory_path): - input_file_path = os.path.join(input_directory_path, file_name) - with open(input_file_path, "r") as f: - sbom = json.load(f) - master_sbom = merge_sboms(master_sbom, sbom) if master_sbom != {} else sbom - save_as_xlsx(master_sbom, output_file_path, author_name) + enrich_vcpkg(final_bom) + if spdx_output_file: + spdx_output_file.write_text( + final_bom.model_dump_json(indent=2, by_alias=True, exclude_none=True) + ) + save_as_xlsx(list(final_bom.to_fda_records(author=author_name)), output_file_path) if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("input_directory", help="Github SPDX SBOM json files directory.") - parser.add_argument("output_file", help="Output combined SBOM excel file path.") - parser.add_argument("--author", help="Override the Author Name.") - args = parser.parse_args() - - gen_sbom(args.input_directory, args.output_file, args.author) + logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s") + main() diff --git a/requirements.txt b/requirements.txt index a717bf1..1baa2fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,81 @@ -openpyxl \ No newline at end of file +# This file was autogenerated by uv via the following command: +# uv export --script gen_sbom.py +annotated-types==0.7.0 \ + --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ + --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 +click==8.1.8 \ + --hash=sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2 \ + --hash=sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a +colorama==0.4.6 ; sys_platform == 'win32' \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 +et-xmlfile==2.0.0 \ + --hash=sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa \ + --hash=sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54 +openpyxl==3.1.5 \ + --hash=sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2 \ + --hash=sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050 +packaging==25.0 \ + --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ + --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f +pydantic==2.11.10 \ + --hash=sha256:802a655709d49bd004c31e865ef37da30b540786a46bfce02333e0e24b5fe29a \ + --hash=sha256:dc280f0982fbda6c38fada4e476dc0a4f3aeaf9c6ad4c28df68a666ec3c61423 +pydantic-core==2.33.2 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 +types-pyyaml==6.0.12.20250915 \ + --hash=sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3 \ + --hash=sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6 +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ + --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 diff --git a/test_gen_sbom.py b/test_gen_sbom.py deleted file mode 100644 index 6b5972d..0000000 --- a/test_gen_sbom.py +++ /dev/null @@ -1,34 +0,0 @@ -from gen_sbom import merge_sboms - - -def test_merge_sboms_packages(): - sbom1 = { - "creationInfo": {"created": "2022-08-03T19:42:37Z"}, - "packages": [ - {"SPDXID": "1", "name": "a", "versionInfo": "1.0.0"}, - {"SPDXID": "2", "name": "b", "versionInfo": "1.0.0"}, - {"SPDXID": "4", "name": "b", "versionInfo": "1.3.0"}, - {"SPDXID": "3", "name": "c", "versionInfo": "1.0.0"}, - {"SPDXID": "5", "name": "d", "versionInfo": "1.0.0"}, - ], - } - sbom2 = { - "creationInfo": {"created": "2023-08-03T19:42:37Z"}, - "packages": [ - {"SPDXID": "11", "name": "a", "versionInfo": "2.0.0"}, - {"SPDXID": "22", "name": "b", "versionInfo": "0.9.0"}, - {"SPDXID": "33", "name": "c", "versionInfo": "2.0.0"}, - {"SPDXID": "44", "name": "e", "versionInfo": "1.0.0"}, - ], - } - expected_sbom = { - "creationInfo": {"created": "2023-08-03T19:42:37Z"}, - "packages": [ - {"SPDXID": "11", "name": "a", "versionInfo": "2.0.0"}, - {"SPDXID": "4", "name": "b", "versionInfo": "1.3.0"}, - {"SPDXID": "33", "name": "c", "versionInfo": "2.0.0"}, - {"SPDXID": "5", "name": "d", "versionInfo": "1.0.0"}, - {"SPDXID": "44", "name": "e", "versionInfo": "1.0.0"}, - ], - } - assert merge_sboms(sbom1, sbom2) == expected_sbom diff --git a/vcpkg.yml b/vcpkg.yml new file mode 100644 index 0000000..d2d4ad7 --- /dev/null +++ b/vcpkg.yml @@ -0,0 +1,502 @@ +# This file contains extra data to link vcpkg packages to their CPE and license. +# lookup cpe at https://cvedetails.com and https://nvd.nist.gov/products/cpe/search +abseil: + cpe: cpe:2.3:a:abseil:common_libraries + license: Apache-2.0 +boost-format: + aka: boost +boost-accumulators: + aka: boost +boost-algorithm: + aka: boost +boost-align: + aka: boost +boost-any: + aka: boost +boost-array: + aka: boost +boost-asio: + aka: boost +boost-assert: + aka: boost +boost-assign: + aka: boost +boost-atomic: + aka: boost +boost-beast: + aka: boost +boost-bimap: + aka: boost +boost-bind: + aka: boost +boost-callable-traits: + aka: boost +boost-charconv: + aka: boost +boost-chrono: + aka: boost +boost-circular-buffer: + aka: boost +boost-cmake: + aka: boost +boost-cobalt: + aka: boost +boost-compat: + aka: boost +boost-compute: + aka: boost +boost-concept-check: + aka: boost +boost-config: + aka: boost +boost-container-hash: + aka: boost +boost-container: + aka: boost +boost-context: + aka: boost +boost-contract: + aka: boost +boost-conversion: + aka: boost +boost-convert: + aka: boost +boost-core: + aka: boost +boost-coroutine2: + aka: boost +boost-coroutine: + aka: boost +boost-crc: + aka: boost +boost-date-time: + aka: boost +boost-describe: + aka: boost +boost-detail: + aka: boost +boost-dll: + aka: boost +boost-dynamic-bitset: + aka: boost +boost-endian: + aka: boost +boost-exception: + aka: boost +boost-fiber: + aka: boost +boost-filesystem: + aka: boost +boost-flyweight: + aka: boost +boost-foreach: + aka: boost +boost-format: + aka: boost +boost-function-types: + aka: boost +boost-functional: + aka: boost +boost-function: + aka: boost +boost-fusion: + aka: boost +boost-geometry: + aka: boost +boost-gil: + aka: boost +boost-graph: + aka: boost +boost-hana: + aka: boost +boost-hash2: + aka: boost +boost-headers: + aka: boost +boost-heap: + aka: boost +boost-histogram: + aka: boost +boost-hof: + aka: boost +boost-icl: + aka: boost +boost-integer: + aka: boost +boost-interprocess: + aka: boost +boost-interval: + aka: boost +boost-intrusive: + aka: boost +boost-io: + aka: boost +boost-iostreams: + aka: boost +boost-iterator: + aka: boost +boost-json: + aka: boost +boost-lambda2: + aka: boost +boost-lambda: + aka: boost +boost-leaf: + aka: boost +boost-lexical-cast: + aka: boost +boost-local-function: + aka: boost +boost-locale: + aka: boost +boost-lockfree: + aka: boost +boost-log: + aka: boost +boost-logic: + aka: boost +boost-math: + aka: boost +boost-metaparse: + aka: boost +boost-move: + aka: boost +boost-mp11: + aka: boost +boost-mpl: + aka: boost +boost-mqtt5: + aka: boost +boost-msm: + aka: boost +boost-multi-array: + aka: boost +boost-multi-index: + aka: boost +boost-multiprecision: + aka: boost +boost-mysql: + aka: boost +boost-nowide: + aka: boost +boost-numeric-conversion: + aka: boost +boost-odeint: + aka: boost +boost-optional: + aka: boost +boost-outcome: + aka: boost +boost-parameter-python: + aka: boost +boost-parameter: + aka: boost +boost-parser: + aka: boost +boost-pfr: + aka: boost +boost-phoenix: + aka: boost +boost-poly-collection: + aka: boost +boost-polygon: + aka: boost +boost-pool: + aka: boost +boost-predef: + aka: boost +boost-preprocessor: + aka: boost +boost-process: + aka: boost +boost-program-options: + aka: boost +boost-property-map: + aka: boost +boost-property-tree: + aka: boost +boost-proto: + aka: boost +boost-ptr-container: + aka: boost +boost-python: + aka: boost +boost-qvm: + aka: boost +boost-random: + aka: boost +boost-range: + aka: boost +boost-ratio: + aka: boost +boost-rational: + aka: boost +boost-redis: + aka: boost +boost-regex: + aka: boost +boost-safe-numerics: + aka: boost +boost-scope-exit: + aka: boost +boost-scope: + aka: boost +boost-serialization: + aka: boost +boost-signals2: + aka: boost +boost-smart-ptr: + aka: boost +boost-sort: + aka: boost +boost-spirit: + aka: boost +boost-stacktrace: + aka: boost +boost-statechart: + aka: boost +boost-static-assert: + aka: boost +boost-static-string: + aka: boost +boost-stl-interfaces: + aka: boost +boost-system: + aka: boost +boost-test: + aka: boost +boost-thread: + aka: boost +boost-throw-exception: + aka: boost +boost-timer: + aka: boost +boost-tokenizer: + aka: boost +boost-tti: + aka: boost +boost-tuple: + aka: boost +boost-type-erasure: + aka: boost +boost-type-index: + aka: boost +boost-type-traits: + aka: boost +boost-typeof: + aka: boost +boost-ublas: + aka: boost +boost-uninstall: + aka: boost +boost-units: + aka: boost +boost-unordered: + aka: boost +boost-url: + aka: boost +boost-utility: + aka: boost +boost-uuid: + aka: boost +boost-variant2: + aka: boost +boost-variant: + aka: boost +boost-vmd: + aka: boost +boost-wave: + aka: boost +boost-winapi: + aka: boost +boost-xpressive: + aka: boost +boost-yap: + aka: boost +boost: + cpe: cpe:2.3:a:boost:boost + license: BSL-1.0 +brotli: + cpe: cpe:2.3:a:google:brotli + license: MIT +bzip2: + cpe: cpe:2.3:a:bzip:bzip2 + license: BSD-style +dbus: + cpe: cpe:2.3:a:freedesktop:dbus + license: AFL-2.1 OR GPL-2.0-or-later +dcmtk: + cpe: cpe:2.3:a:offis:dcmtk + license: BSD-3-Clause +double-conversion: + cpe: cpe:2.3:a:google:double_conversion # does not exist in NVD 2025-10-24 + license: BSD-3-Clause +egl-registry: + cpe: cpe:2.3:a:khronos:egl # does not exist in NVD 2025-10-24 + license: Apache-2.0 +eigen3: + cpe: cpe:2.3:a:eigen:eigen # does not exist in NVD 2025-10-24 + license: MPL-2.0 +expat: + cpe: cpe:2.3:a:libexpat_project:libexpat + license: MIT +flann: + cpe: cpe:2.3:a:mariusmuja:flann # does not exist in NVD 2025-10-24 + license: BSD-3-Clause +flatbuffers: + cpe: cpe:2.3:a:google:flatbuffers + license: Apache-2.0 +freetype: + cpe: cpe:2.3:a:freetype:freetype + license: FTL +gtest: + cpe: cpe:2.3:a:google:googletest # does not exist in NVD 2025-10-24 + license: BSD-3-Clause +harfbuzz: + cpe: cpe:2.3:a:harfbuzz_project:harfbuzz + license: MIT +hwloc: + cpe: cpe:2.3:a:open-mpi:hwloc # does not exist in NVD 2025-10-24 + license: BSD-3-Clause +icu: + cpe: cpe:2.3:a:icu-project:international_components_for_unicode + license: ICU +intel-mkl: + cpe: cpe:2.3:a:intel:math_kernel_library + license: Intel Simplified Software License +libffi: + cpe: cpe:2.3:a:libffi_project:libffi + license: MIT +libjpeg-turbo: + cpe: cpe:2.3:a:libjpeg-turbo:libjpeg-turbo + license: BSD-3-Clause +liblzma: + cpe: cpe:2.3:a:tukaani:xz + license: Public Domain +libpng: + cpe: cpe:2.3:a:libpng:libpng + license: PNG +libpq: + cpe: cpe:2.3:a:postgresql:postgresql + license: PostgreSQL +libwebp: + cpe: cpe:2.3:a:webmproject:libwebp + license: BSD-3-Clause +lz4: + cpe: cpe:2.3:a:lz4_project:lz4 + license: BSD-2-Clause +mimalloc: + cpe: cpe:2.3:a:microsoft:mimalloc + license: MIT +nanoflann: + cpe: cpe:2.3:a:jlblancoc:nanoflann # does not exist in NVD 2025-10-24 + license: BSD +nifticlib: + cpe: cpe:2.3:a:neuroimaging_informatics_technology_initiative:nifticlib # does not exist in NVD 2025-10-24 + license: Public Domain +opencv4: + cpe: cpe:2.3:a:opencv:opencv + license: Apache-2.0 +opengl-registry: + cpe: cpe:2.3:a:khronos:opengl_registry # does not exist in NVD 2025-10-24 + license: Apache-2.0 +opengl: + cpe: cpe:2.3:a:khronos:opengl # does not exist in NVD 2025-10-24 + license: MIT +openssl: + cpe: cpe:2.3:a:openssl:openssl + license: Apache-2.0 +pcl: + cpe: cpe:2.3:a:pointclouds:point_cloud_library + license: BSD-3-Clause +pcre2: + cpe: cpe:2.3:a:pcre:pcre2 + license: BSD-3-Clause +pkgconf: + cpe: cpe:2.3:a:pkgconf:pkgconf + license: ISC +protobuf: + cpe: cpe:2.3:a:google:protobuf + license: BSD-3-Clause +python3: + cpe: cpe:2.3:a:python:python + license: Python Software Foundation License +qhull: + cpe: cpe:2.3:a:qhull:qhull # does not exist in NVD 2025-10-24 + license: Qhull +qt: + cpe: cpe:2.3:a:qt:qt + license: LGPL-3.0 +qtbase: + aka: qt +qtcharts: + aka: qt +qtmultimedia: + aka: qt +qtserialport: + aka: qt +qtshadertools: + aka: qt +qtsvg: + aka: qt +quirc: + cpe: cpe:2.3:a:dlbeer:quirc # does not exist in NVD 2025-10-24 + license: ISC +sqlcipher: + cpe: cpe:2.3:a:zetetic:sqlcipher + license: Commercial +sqlite3: + cpe: cpe:2.3:a:sqlite:sqlite + license: Public Domain +tbb: + cpe: cpe:2.3:a:intel:threading_building_blocks + license: Apache-2.0 +tcl: + cpe: cpe:2.3:a:tcl:tcl + license: Tcl/Tk License +tiff: + cpe: cpe:2.3:a:libtiff:libtiff + license: BSD-style +utf8-range: + cpe: cpe:2.3:a:google:utf8_range # does not exist in NVD 2025-10-24 + license: MIT +vcpkg-boost: + cpe: cpe:2.3:a:microsoft:vcpkg_boost + license: MIT +vcpkg-cmake-config: + cpe: cpe:2.3:a:microsoft:vcpkg_cmake_config + license: MIT +vcpkg-cmake-get-vars: + cpe: cpe:2.3:a:microsoft:vcpkg_cmake_get_vars + license: MIT +vcpkg-cmake: + cpe: cpe:2.3:a:microsoft:vcpkg_cmake + license: MIT +vcpkg-get-python-packages: + cpe: cpe:2.3:a:microsoft:vcpkg_get_python_packages + license: MIT +vcpkg-get-python: + cpe: cpe:2.3:a:microsoft:vcpkg_get_python + license: MIT +vcpkg-make: + cpe: cpe:2.3:a:microsoft:vcpkg_make + license: MIT +vcpkg-msbuild: + cpe: cpe:2.3:a:microsoft:vcpkg_msbuild + license: MIT +vcpkg-pkgconfig-get-modules: + cpe: cpe:2.3:a:microsoft:vcpkg_pkgconfig_get_modules + license: MIT +vcpkg-tool-lessmsi: + cpe: cpe:2.3:a:microsoft:vcpkg_tool_lessmsi + license: MIT +vcpkg-tool-meson: + cpe: cpe:2.3:a:microsoft:vcpkg_tool_meson + license: MIT +zlib: + cpe: cpe:2.3:a:zlib:zlib + license: Zlib +zstd: + cpe: cpe:2.3:a:facebook:zstandard + license: BSD-3-Clause + diff --git a/verify_vcpkg_cpe.py b/verify_vcpkg_cpe.py new file mode 100755 index 0000000..16ed86e --- /dev/null +++ b/verify_vcpkg_cpe.py @@ -0,0 +1,82 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.13" +# dependencies = ["nvdlib~=0.8.3", "pyyaml~=6.0.3"] +# /// +import sys +import time + +import nvdlib +import yaml + + +def verify_cpes(filepath="vcpkg.yml"): + """ + Parses a YAML file, extracts CPE strings, and verifies them against the NVD. + """ + try: + with open(filepath, "r") as f: + data = yaml.safe_load(f) + except FileNotFoundError: + print(f"Error: File not found at '{filepath}'") + sys.exit(1) + except yaml.YAMLError as e: + print(f"Error parsing YAML file: {e}") + sys.exit(1) + + if not isinstance(data, dict): + print("Error: YAML file is not a dictionary of packages.") + sys.exit(1) + + print(f"Verifying CPEs from '{filepath}' against NVD...\n") + + not_found = [] + found_count = 0 + + packages_to_check = [] + for package_name, details in data.items(): + if isinstance(details, dict) and "cpe" in details: + packages_to_check.append((package_name, details["cpe"])) + + total_count = len(packages_to_check) + print(f"Found {total_count} packages with CPEs to verify.") + + for i, (package_name, cpe_string) in enumerate(packages_to_check): + try: + # Use cpeMatchString for an exact search. + # The NVD API may return multiple minor versions for a base CPE string, + # so we check if we get at least one result. + results = nvdlib.searchCPE(cpeMatchString=cpe_string, limit=1) + if results: + print(f"✅ Found: {package_name} ({cpe_string})") + found_count += 1 + else: + print(f"❌ Not Found: {package_name} ({cpe_string})") + not_found.append((package_name, cpe_string)) + except Exception as e: + print(f"ERROR searching for {package_name} ({cpe_string}): {e}") + not_found.append((package_name, cpe_string)) + + # The public NVD API has a rate limit. A delay prevents hitting it. + # 5 requests per 30 seconds without an API key. + if (i + 1) % 5 == 0 and i < total_count - 1: + print("\n--- Pausing for 30 seconds to respect NVD API rate limit ---\n") + time.sleep(30) + + print("\n--- Verification Summary ---") + print(f"Total Packages with CPEs: {total_count}") + print(f"Found in NVD: {found_count}") + print(f"Not Found in NVD: {len(not_found)}") + + if not_found: + print("\nPackages not found in NVD:") + for package, cpe in not_found: + print(f" - {package}: {cpe}") + + +if __name__ == "__main__": + # Assumes vcpkg.yml is in the same directory as the script. + # You can pass a different path as an argument. + file_to_check = sys.argv[1] if len(sys.argv) > 1 else "vcpkg.yml" + verify_cpes(file_to_check) diff --git a/vuln-report.py b/vuln-report.py new file mode 100755 index 0000000..aeedf84 --- /dev/null +++ b/vuln-report.py @@ -0,0 +1,147 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.13" +# dependencies = ["nvdlib~=0.8.3"] +# /// +import json +import argparse +import nvdlib # type: ignore +from datetime import datetime + + +def parse_spdx(file_path): + """Parses the SPDX JSON file to extract package information.""" + with open(file_path, "r") as f: + spdx_data = json.load(f) + return spdx_data.get("packages", []) + + +def get_cpe_from_package(package): + """Extracts the CPE string from a package's external references.""" + for ref in package.get("externalRefs", []): + if ref.get("referenceType") == "SECURITY" and ref.get( + "referenceLocator", "" + ).startswith("cpe:"): + return ref["referenceLocator"] + return None + + +def find_vulnerabilities(cpe_string): + """Finds vulnerabilities for a given CPE string using nvdlib.""" + if not cpe_string: + return [] + try: + # The free NVD API has rate limits, so this may be slow. + # nvdlib handles waiting to respect the rate limit. + print(f"Searching for vulnerabilities for: {cpe_string}") + # We search for CVEs that match the CPE string. + # The 'limit' parameter can be adjusted if needed. + results = nvdlib.searchCVE(cpeName=cpe_string, limit=2000) + return results + except Exception as e: + print(f"Could not fetch vulnerabilities for {cpe_string}. Error: {e}") + return [] + + +def generate_markdown_report(packages_with_vulns, spdx_file_name): + """Generates a Markdown report from the vulnerability data.""" + report_lines = [ + f"# Vulnerability Report for {spdx_file_name}", + f"Report generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + "\n---", + ] + + if not packages_with_vulns: + report_lines.append( + "\n**No vulnerabilities found for the components in this SBOM.**" + ) + return "\n".join(report_lines) + + for package_info in packages_with_vulns: + report_lines.append( + f"\n## 📦 Package: {package_info['name']} ({package_info['version']})" + ) + report_lines.append(f"**CPE:** `{package_info['cpe']}`") + + vulns = package_info["vulnerabilities"] + if not vulns: + report_lines.append("\n*No vulnerabilities found for this package.*") + continue + + report_lines.append("\n### Found Vulnerabilities:") + for cve in vulns: + report_lines.append( + f"\n#### 🚨 [{cve.id}](https://nvd.nist.gov/vuln/detail/{cve.id})" + ) + + # Get CVSS V3 score if available, otherwise V2 + severity = "N/A" + if hasattr(cve.metrics, "cvssMetricV31") and cve.metrics.cvssMetricV31: + severity = f"{cve.metrics.cvssMetricV31[0].cvssData.baseScore} ({cve.metrics.cvssMetricV31[0].cvssData.baseSeverity})" + elif hasattr(cve.metrics, "cvssMetricV2") and cve.metrics.cvssMetricV2: + severity = f"{cve.metrics.cvssMetricV2[0].cvssData.baseScore} ({cve.metrics.cvssMetricV2[0].baseSeverity})" + + report_lines.append(f"- **Severity:** {severity}") + + description = "No description available." + if cve.descriptions: + description = cve.descriptions[0].value + + report_lines.append(f"- **Description:** {description}") + + return "\n".join(report_lines) + + +def main(): + parser = argparse.ArgumentParser( + description="Generate a vulnerability report from an SPDX JSON file." + ) + parser.add_argument("spdx_file", help="Path to the SPDX JSON file.") + parser.add_argument( + "-o", + "--output", + help="Path to the output Markdown file. If not provided, prints to console.", + ) + + args = parser.parse_args() + + packages = parse_spdx(args.spdx_file) + packages_with_vulns = [] + + for package in packages: + name = package.get("name", "N/A") + version = package.get("versionInfo", "N/A") + cpe = get_cpe_from_package(package) + + if not cpe: + continue + + vulnerabilities = find_vulnerabilities(cpe) + + # We only add packages with vulnerabilities to the report + if vulnerabilities: + packages_with_vulns.append( + { + "name": name, + "version": version, + "cpe": cpe, + "vulnerabilities": vulnerabilities, + } + ) + + markdown_report = generate_markdown_report(packages_with_vulns, args.spdx_file) + + if args.output: + with open(args.output, "w") as f: + f.write(markdown_report) + print(f"Report successfully generated at {args.output}") + else: + print("\n" + "=" * 20 + " REPORT " + "=" * 20 + "\n") + print(markdown_report) + + +if __name__ == "__main__": + print("This is a last resort tool to map SPDX SBOM with CPE string to vulnerabilities in NVD.") + print("Consider using Trivy or Syft/Grype for better results.") + main()