From 62869e7ba4d5886dfdcfb05ea849df483719634b Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Wed, 15 Jul 2026 11:42:02 -0600 Subject: [PATCH 1/5] ci: check dependency licenses Add a locked runtime dependency license scan with an explicit permissive allowlist and package-specific reviewed exceptions. Fail on unknown licenses, exception drift, and stale exceptions. Closes #822 Signed-off-by: Nabin Mulepati --- .github/workflows/ci.yml | 22 +++ Makefile | 9 +- dependency-license-policy.toml | 53 ++++++ scripts/check_dependency_licenses.py | 196 +++++++++++++++++++++ tests_e2e/test_dependency_license_check.py | 92 ++++++++++ 5 files changed, 370 insertions(+), 2 deletions(-) create mode 100644 dependency-license-policy.toml create mode 100644 scripts/check_dependency_licenses.py create mode 100644 tests_e2e/test_dependency_license_check.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36ebd0dd1..5e900bfd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,6 +249,28 @@ jobs: - name: Check license headers run: make check-license-headers + dependency-licenses: + name: Check Dependency Licenses + needs: validate-dispatch + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + version: "latest" + python-version: "3.11" + enable-cache: true + + - name: Install locked runtime dependencies + run: uv sync --all-packages --no-dev --locked + + - name: Check dependency licenses + run: make check-dependency-licenses + # =========================================================================== # Summary Job for Branch Protection # This job creates status checks matching the old job naming convention diff --git a/Makefile b/Makefile index e83917146..7dadcb7b4 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,7 @@ help: @echo " check-fern-docs-locally - Install deps, generate Fern artifacts, and run fern check" @echo " serve-fern-docs-locally - Generate local Fern artifacts and serve Fern docs" @echo " check-license-headers - Check if all files have license headers" + @echo " check-dependency-licenses - Check runtime dependency license compatibility" @echo " update-license-headers - Add license headers to all files" @echo "" @echo "⚡ Performance:" @@ -449,9 +450,13 @@ show-versions: @uv run python -c "from data_designer.interface._version import __version__; print(f' data-designer: {__version__}')" 2>/dev/null || echo " data-designer: (not installed)" # ============================================================================== -# LICENSE HEADERS +# LICENSE CHECKS # ============================================================================== +check-dependency-licenses: + @echo "🔍 Checking runtime dependency licenses..." + uv run --no-sync python $(REPO_PATH)/scripts/check_dependency_licenses.py + check-license-headers: @echo "🔍 Checking license headers in all files..." uv run python $(REPO_PATH)/scripts/update_license_headers.py --check @@ -746,7 +751,7 @@ clean-test-coverage: .PHONY: bench-cli-startup bench-cli-startup-verbose \ build build-config build-engine build-interface \ check-all check-all-fix check-config check-engine check-interface \ - check-fern-docs check-fern-docs-locally check-fern-release-version check-fern-theme-access check-license-headers \ + check-dependency-licenses check-fern-docs check-fern-docs-locally check-fern-release-version check-fern-theme-access check-license-headers \ clean clean-dist clean-notebooks clean-pycache clean-test-coverage \ convert-execute-notebooks \ coverage coverage-config coverage-engine coverage-interface \ diff --git a/dependency-license-policy.toml b/dependency-license-policy.toml new file mode 100644 index 000000000..ce9be8332 --- /dev/null +++ b/dependency-license-policy.toml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Permissive licenses that the Apache Software Foundation classifies as +# "Apache-like" (Category A). Values use SPDX identifiers after normalization. +# https://www.apache.org/legal/resolved.html#category-a +allowed_licenses = [ + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "MIT", + "Zlib", +] + +# Existing dependencies outside the global whitelist are reviewed by package +# and exact reported license. A license change or a new package with one of +# these licenses must be reviewed explicitly. +[exceptions.certifi] +license = "Mozilla Public License 2.0 (MPL 2.0)" +reason = "Unmodified TLS certificate bundle installed as a separate transitive dependency." + +[exceptions.chardet] +license = "GNU Lesser General Public License v2 or later (LGPLv2+)" +reason = "Unmodified character-detection library installed as a separate runtime dependency." + +[exceptions.email-validator] +license = "The Unlicense (Unlicense)" +reason = "Existing unmodified validation library installed as a separate runtime dependency." + +[exceptions.numpy] +license = "BSD-3-Clause AND 0BSD AND MIT AND Zlib AND CC0-1.0" +reason = "Core numerical dependency with bundled components under additional permissive licenses." + +[exceptions.pathspec] +license = "Mozilla Public License 2.0 (MPL 2.0)" +reason = "Unmodified transitive dependency of sqlfluff installed as a separate package." + +[exceptions.pillow] +license = "MIT-CMU" +reason = "Core image dependency under the permissive MIT-CMU license." + +[exceptions.regex] +license = "Apache-2.0 AND CNRI-Python" +reason = "Existing regex dependency containing code under the CNRI Python license." + +[exceptions.tqdm] +license = "MPL-2.0 AND MIT" +reason = "Unmodified progress library installed as a separate transitive dependency." + +[exceptions.typing_extensions] +license = "PSF-2.0" +reason = "Python typing compatibility dependency under the PSF license." diff --git a/scripts/check_dependency_licenses.py b/scripts/check_dependency_licenses.py new file mode 100644 index 000000000..643774133 --- /dev/null +++ b/scripts/check_dependency_licenses.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +PIP_LICENSES_VERSION = "5.5.5" +POLICY_PATH = Path(__file__).resolve().parents[1] / "dependency-license-policy.toml" + +LICENSE_ALIASES = { + "Apache Software License": {"Apache-2.0"}, + "BSD License": {"BSD-3-Clause"}, + "ISC License (ISCL)": {"ISC"}, + "MIT License": {"MIT"}, + "Python Software Foundation License": {"PSF-2.0"}, + "The Unlicense (Unlicense)": {"Unlicense"}, +} +EXPRESSION_OPERATOR = re.compile(r"\s+(?:AND|OR|WITH)\s+|[()]", re.IGNORECASE) + + +@dataclass(frozen=True) +class ExceptionPolicy: + license: str + reason: str + + +@dataclass(frozen=True) +class LicensePolicy: + allowed_licenses: frozenset[str] + exceptions: dict[str, ExceptionPolicy] + + +@dataclass(frozen=True) +class PackageLicense: + name: str + version: str + license: str + license_text: str + + +def load_policy(path: Path = POLICY_PATH) -> LicensePolicy: + """Load the dependency-license policy from TOML.""" + data = tomllib.loads(path.read_text(encoding="utf-8")) + exceptions = { + name.casefold(): ExceptionPolicy(license=value["license"], reason=value["reason"]) + for name, value in data.get("exceptions", {}).items() + } + return LicensePolicy(allowed_licenses=frozenset(data["allowed_licenses"]), exceptions=exceptions) + + +def looks_like_mit_license(license_text: str) -> bool: + normalized = " ".join(license_text.split()).casefold() + return normalized.startswith("mit license ") and "permission is hereby granted, free of charge" in normalized + + +def normalized_license_ids(package: PackageLicense) -> frozenset[str]: + """Normalize pip-licenses output into SPDX-like license identifiers.""" + reported = package.license.strip() + if reported.casefold() == "unknown": + return frozenset({"MIT"}) if looks_like_mit_license(package.license_text) else frozenset() + + if reported.startswith("MIT License\n") and looks_like_mit_license(reported): + return frozenset({"MIT"}) + + aliases = LICENSE_ALIASES.get(reported) + if aliases is not None: + return frozenset(aliases) + + identifiers: set[str] = set() + for part in reported.split(";"): + stripped = part.strip() + part_aliases = LICENSE_ALIASES.get(stripped) + if part_aliases is not None: + identifiers.update(part_aliases) + continue + + tokens = [token.strip() for token in EXPRESSION_OPERATOR.split(stripped) if token.strip()] + if not tokens: + return frozenset() + identifiers.update(tokens) + + return frozenset(identifiers) + + +def parse_report(data: list[dict[str, Any]]) -> list[PackageLicense]: + """Parse the structured pip-licenses report.""" + return [ + PackageLicense( + name=str(item["Name"]), + version=str(item["Version"]), + license=str(item["License"]), + license_text=str(item.get("LicenseText", "")), + ) + for item in data + ] + + +def evaluate_report(packages: list[PackageLicense], policy: LicensePolicy) -> tuple[list[str], list[str]]: + """Return policy violations and reviewed-exception descriptions.""" + violations: list[str] = [] + reviewed: list[str] = [] + seen_exceptions: set[str] = set() + + for package in sorted(packages, key=lambda item: item.name.casefold()): + package_key = package.name.casefold() + exception = policy.exceptions.get(package_key) + if exception is not None: + seen_exceptions.add(package_key) + if package.license != exception.license: + violations.append( + f"{package.name}=={package.version}: exception expected {exception.license!r}, " + f"but package reports {package.license!r}" + ) + else: + reviewed.append(f"{package.name}=={package.version}: {package.license} ({exception.reason})") + continue + + identifiers = normalized_license_ids(package) + if not identifiers: + violations.append(f"{package.name}=={package.version}: unknown or unrecognized license {package.license!r}") + elif not identifiers.issubset(policy.allowed_licenses): + disallowed = ", ".join(sorted(identifiers - policy.allowed_licenses)) + violations.append(f"{package.name}=={package.version}: disallowed license(s): {disallowed}") + + for package_key in sorted(policy.exceptions.keys() - seen_exceptions): + violations.append(f"{package_key}: stale exception; package is not present in the scanned environment") + + return violations, reviewed + + +def collect_report() -> list[PackageLicense]: + """Run the pinned scanner against the active Python environment.""" + command = [ + "uv", + "tool", + "run", + "--from", + f"pip-licenses=={PIP_LICENSES_VERSION}", + "pip-licenses", + "--python", + sys.executable, + "--from=mixed", + "--format=json", + "--with-license-file", + "--no-license-path", + ] + result = subprocess.run(command, capture_output=True, check=False, text=True) + if result.returncode != 0: + print(result.stderr, file=sys.stderr) + raise RuntimeError(f"pip-licenses exited with status {result.returncode}") + return parse_report(json.loads(result.stdout)) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Check runtime dependencies against the Apache-2.0 license policy") + parser.add_argument("--report", type=Path, help="Read a pip-licenses JSON report instead of running the scanner") + args = parser.parse_args() + + if args.report is None: + packages = collect_report() + else: + packages = parse_report(json.loads(args.report.read_text(encoding="utf-8"))) + + violations, reviewed = evaluate_report(packages, load_policy()) + print(f"Checked {len(packages)} installed runtime packages.") + + if reviewed: + print("\nReviewed package-specific exceptions:") + for item in reviewed: + print(f" - {item}") + + if violations: + print("\nDependency license policy violations:", file=sys.stderr) + for violation in violations: + print(f" - {violation}", file=sys.stderr) + return 1 + + print("\nAll dependency licenses satisfy the Apache-2.0 compatibility policy.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests_e2e/test_dependency_license_check.py b/tests_e2e/test_dependency_license_check.py new file mode 100644 index 000000000..388004752 --- /dev/null +++ b/tests_e2e/test_dependency_license_check.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "check_dependency_licenses.py" + + +def load_script() -> ModuleType: + spec = importlib.util.spec_from_file_location("check_dependency_licenses", SCRIPT_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_evaluate_report_accepts_permissive_and_composite_licenses() -> None: + module = load_script() + policy = module.LicensePolicy( + allowed_licenses=frozenset({"Apache-2.0", "BSD-3-Clause", "MIT"}), + exceptions={}, + ) + packages = [ + module.PackageLicense("apache", "1", "Apache-2.0", ""), + module.PackageLicense("dual", "1", "Apache-2.0 OR BSD-3-Clause", ""), + module.PackageLicense("mit", "1", "MIT License", ""), + ] + + violations, reviewed = module.evaluate_report(packages, policy) + + assert violations == [] + assert reviewed == [] + + +def test_evaluate_report_rejects_copyleft_and_unknown_licenses() -> None: + module = load_script() + policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={}) + packages = [ + module.PackageLicense("copyleft", "1", "LGPL-2.1-or-later", ""), + module.PackageLicense("unknown", "1", "UNKNOWN", ""), + ] + + violations, _ = module.evaluate_report(packages, policy) + + assert violations == [ + "copyleft==1: disallowed license(s): LGPL-2.1-or-later", + "unknown==1: unknown or unrecognized license 'UNKNOWN'", + ] + + +def test_evaluate_report_recognizes_bundled_mit_license_text() -> None: + module = load_script() + policy = module.LicensePolicy(allowed_licenses=frozenset({"MIT"}), exceptions={}) + package = module.PackageLicense( + "missing-metadata", + "1", + "UNKNOWN", + "MIT License\n\nCopyright example\n\nPermission is hereby granted, free of charge, to any person obtaining a copy", + ) + + violations, _ = module.evaluate_report([package], policy) + + assert violations == [] + + +def test_evaluate_report_requires_exception_license_to_remain_unchanged() -> None: + module = load_script() + exception = module.ExceptionPolicy(license="MPL-2.0", reason="Reviewed separately.") + policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={"example": exception}) + package = module.PackageLicense("example", "2", "GPL-3.0-only", "") + + violations, reviewed = module.evaluate_report([package], policy) + + assert violations == ["example==2: exception expected 'MPL-2.0', but package reports 'GPL-3.0-only'"] + assert reviewed == [] + + +def test_evaluate_report_rejects_stale_exceptions() -> None: + module = load_script() + exception = module.ExceptionPolicy(license="MPL-2.0", reason="Reviewed separately.") + policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={"removed": exception}) + + violations, _ = module.evaluate_report([], policy) + + assert violations == ["removed: stale exception; package is not present in the scanned environment"] From e33bfdd9976cdb90af422c126c2fa6fb8d0ab67a Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Wed, 15 Jul 2026 13:00:03 -0600 Subject: [PATCH 2/5] fix: handle dependency license expressions Preserve SPDX OR, AND, and WITH semantics during policy evaluation. Cover malformed and delimited metadata, recognize the alternate MIT heading, and declare the Python 3.10 TOML fallback. Signed-off-by: Nabin Mulepati --- pyproject.toml | 1 + scripts/check_dependency_licenses.py | 108 ++++++++++++++++----- tests_e2e/test_dependency_license_check.py | 84 ++++++++++++++-- uv.lock | 2 + 4 files changed, 166 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4e6bb2b46..a84db2b0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dev = [ "pytest-env>=1.2.0,<2", "pytest-httpx>=0.36.0,<1", "ruff>=0.14.10,<1", + "tomli>=2.0.1,<3; python_version < '3.11'", ] docs = [ "jupytext>=1.16.0,<2", diff --git a/scripts/check_dependency_licenses.py b/scripts/check_dependency_licenses.py index 643774133..513cc189b 100644 --- a/scripts/check_dependency_licenses.py +++ b/scripts/check_dependency_licenses.py @@ -28,7 +28,7 @@ "Python Software Foundation License": {"PSF-2.0"}, "The Unlicense (Unlicense)": {"Unlicense"}, } -EXPRESSION_OPERATOR = re.compile(r"\s+(?:AND|OR|WITH)\s+|[()]", re.IGNORECASE) +EXPRESSION_TOKEN = re.compile(r"(\(|\)|\bAND\b|\bOR\b|\bWITH\b)") @dataclass(frozen=True) @@ -63,36 +63,99 @@ def load_policy(path: Path = POLICY_PATH) -> LicensePolicy: def looks_like_mit_license(license_text: str) -> bool: normalized = " ".join(license_text.split()).casefold() - return normalized.startswith("mit license ") and "permission is hereby granted, free of charge" in normalized - - -def normalized_license_ids(package: PackageLicense) -> frozenset[str]: - """Normalize pip-licenses output into SPDX-like license identifiers.""" + has_mit_heading = normalized.startswith(("mit license ", "the mit license (mit) ")) + return has_mit_heading and "permission is hereby granted, free of charge" in normalized + + +def _and_alternatives( + left: tuple[frozenset[str], ...], right: tuple[frozenset[str], ...] +) -> tuple[frozenset[str], ...]: + return tuple(left_ids | right_ids for left_ids in left for right_ids in right) + + +class LicenseExpressionParser: + """Parse the SPDX operators needed by package license metadata.""" + + def __init__(self, expression: str) -> None: + self.tokens = [token.strip() for token in EXPRESSION_TOKEN.split(expression) if token.strip()] + self.position = 0 + + def parse(self) -> tuple[frozenset[str], ...]: + alternatives = self._parse_or() + if self.position != len(self.tokens): + return () + return alternatives + + def _parse_or(self) -> tuple[frozenset[str], ...]: + alternatives = self._parse_and() + while self._accept("OR"): + right = self._parse_and() + if not right: + return () + alternatives += right + return alternatives + + def _parse_and(self) -> tuple[frozenset[str], ...]: + alternatives = self._parse_primary() + while self._accept("AND"): + alternatives = _and_alternatives(alternatives, self._parse_primary()) + return alternatives + + def _parse_primary(self) -> tuple[frozenset[str], ...]: + if self._accept("("): + alternatives = self._parse_or() + if not self._accept(")"): + return () + return alternatives + + if self.position >= len(self.tokens) or self.tokens[self.position] in {"AND", "OR", "WITH", ")"}: + return () + + reported_id = self.tokens[self.position] + self.position += 1 + identifiers = frozenset(LICENSE_ALIASES.get(reported_id, {reported_id})) + + if self._accept("WITH"): + if self.position >= len(self.tokens) or self.tokens[self.position] in {"(", ")", "AND", "OR", "WITH"}: + return () + self.position += 1 + + return (identifiers,) + + def _accept(self, token: str) -> bool: + if self.position >= len(self.tokens) or self.tokens[self.position] != token: + return False + self.position += 1 + return True + + +def normalized_license_alternatives(package: PackageLicense) -> tuple[frozenset[str], ...]: + """Normalize a reported license into alternatives of required SPDX-like identifiers.""" reported = package.license.strip() if reported.casefold() == "unknown": - return frozenset({"MIT"}) if looks_like_mit_license(package.license_text) else frozenset() + return (frozenset({"MIT"}),) if looks_like_mit_license(package.license_text) else () if reported.startswith("MIT License\n") and looks_like_mit_license(reported): - return frozenset({"MIT"}) + return (frozenset({"MIT"}),) aliases = LICENSE_ALIASES.get(reported) if aliases is not None: - return frozenset(aliases) + return (frozenset(aliases),) - identifiers: set[str] = set() + alternatives: tuple[frozenset[str], ...] = (frozenset(),) + parsed_any = False for part in reported.split(";"): stripped = part.strip() - part_aliases = LICENSE_ALIASES.get(stripped) - if part_aliases is not None: - identifiers.update(part_aliases) + if not stripped: continue - tokens = [token.strip() for token in EXPRESSION_OPERATOR.split(stripped) if token.strip()] - if not tokens: - return frozenset() - identifiers.update(tokens) + parsed = LicenseExpressionParser(stripped).parse() + if not parsed: + return () + alternatives = _and_alternatives(alternatives, parsed) + parsed_any = True - return frozenset(identifiers) + return alternatives if parsed_any else () def parse_report(data: list[dict[str, Any]]) -> list[PackageLicense]: @@ -128,11 +191,12 @@ def evaluate_report(packages: list[PackageLicense], policy: LicensePolicy) -> tu reviewed.append(f"{package.name}=={package.version}: {package.license} ({exception.reason})") continue - identifiers = normalized_license_ids(package) - if not identifiers: + alternatives = normalized_license_alternatives(package) + if not alternatives: violations.append(f"{package.name}=={package.version}: unknown or unrecognized license {package.license!r}") - elif not identifiers.issubset(policy.allowed_licenses): - disallowed = ", ".join(sorted(identifiers - policy.allowed_licenses)) + elif not any(identifiers.issubset(policy.allowed_licenses) for identifiers in alternatives): + disallowed_ids = set().union(*(identifiers - policy.allowed_licenses for identifiers in alternatives)) + disallowed = ", ".join(sorted(disallowed_ids)) violations.append(f"{package.name}=={package.version}: disallowed license(s): {disallowed}") for package_key in sorted(policy.exceptions.keys() - seen_exceptions): diff --git a/tests_e2e/test_dependency_license_check.py b/tests_e2e/test_dependency_license_check.py index 388004752..66fa6fc7a 100644 --- a/tests_e2e/test_dependency_license_check.py +++ b/tests_e2e/test_dependency_license_check.py @@ -39,6 +39,68 @@ def test_evaluate_report_accepts_permissive_and_composite_licenses() -> None: assert reviewed == [] +def test_evaluate_report_accepts_allowed_or_alternative() -> None: + module = load_script() + policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0", "MIT"}), exceptions={}) + packages = [ + module.PackageLicense("dual", "1", "MIT OR GPL-3.0-only", ""), + module.PackageLicense("grouped", "1", "Apache-2.0 AND (MIT OR GPL-3.0-only)", ""), + ] + + violations, _ = module.evaluate_report(packages, policy) + + assert violations == [] + + +def test_evaluate_report_requires_all_and_terms() -> None: + module = load_script() + policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={}) + package = module.PackageLicense("conjunctive", "1", "Apache-2.0 AND GPL-3.0-only", "") + + violations, _ = module.evaluate_report([package], policy) + + assert violations == ["conjunctive==1: disallowed license(s): GPL-3.0-only"] + + +def test_evaluate_report_rejects_disallowed_or_alternatives_and_malformed_expressions() -> None: + module = load_script() + policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={}) + packages = [ + module.PackageLicense("disallowed", "1", "GPL-2.0-only OR GPL-3.0-only", ""), + module.PackageLicense("malformed", "1", "Apache-2.0 OR", ""), + ] + + violations, _ = module.evaluate_report(packages, policy) + + assert violations == [ + "disallowed==1: disallowed license(s): GPL-2.0-only, GPL-3.0-only", + "malformed==1: unknown or unrecognized license 'Apache-2.0 OR'", + ] + + +def test_evaluate_report_uses_base_license_with_exception() -> None: + module = load_script() + policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={}) + packages = [ + module.PackageLicense("allowed", "1", "Apache-2.0 WITH LLVM-exception", ""), + module.PackageLicense("rejected", "1", "GPL-2.0-only WITH Classpath-exception-2.0", ""), + ] + + violations, _ = module.evaluate_report(packages, policy) + + assert violations == ["rejected==1: disallowed license(s): GPL-2.0-only"] + + +def test_evaluate_report_ignores_empty_semicolon_segments() -> None: + module = load_script() + policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0", "MIT"}), exceptions={}) + package = module.PackageLicense("trailing", "1", "Apache Software License;; ", "") + + violations, _ = module.evaluate_report([package], policy) + + assert violations == [] + + def test_evaluate_report_rejects_copyleft_and_unknown_licenses() -> None: module = load_script() policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={}) @@ -58,14 +120,22 @@ def test_evaluate_report_rejects_copyleft_and_unknown_licenses() -> None: def test_evaluate_report_recognizes_bundled_mit_license_text() -> None: module = load_script() policy = module.LicensePolicy(allowed_licenses=frozenset({"MIT"}), exceptions={}) - package = module.PackageLicense( - "missing-metadata", - "1", - "UNKNOWN", - "MIT License\n\nCopyright example\n\nPermission is hereby granted, free of charge, to any person obtaining a copy", - ) + packages = [ + module.PackageLicense( + "missing-metadata", + "1", + "UNKNOWN", + "MIT License\n\nCopyright example\n\nPermission is hereby granted, free of charge, to any person obtaining a copy", + ), + module.PackageLicense( + "alternate-heading", + "1", + "UNKNOWN", + "The MIT License (MIT)\n\nCopyright example\n\nPermission is hereby granted, free of charge, to any person", + ), + ] - violations, _ = module.evaluate_report([package], policy) + violations, _ = module.evaluate_report(packages, policy) assert violations == [] diff --git a/uv.lock b/uv.lock index 23c74449f..49d62b3ba 100644 --- a/uv.lock +++ b/uv.lock @@ -965,6 +965,7 @@ dev = [ { name = "pytest-env" }, { name = "pytest-httpx" }, { name = "ruff" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] docs = [ { name = "jupytext" }, @@ -999,6 +1000,7 @@ dev = [ { name = "pytest-env", specifier = ">=1.2.0,<2" }, { name = "pytest-httpx", specifier = ">=0.36.0,<1" }, { name = "ruff", specifier = ">=0.14.10,<1" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.1,<3" }, ] docs = [{ name = "jupytext", specifier = ">=1.16.0,<2" }] notebooks = [ From 9f237f3a6a09467e683d5ca1fefb1316fee03b5f Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Thu, 16 Jul 2026 10:42:51 -0600 Subject: [PATCH 3/5] fix: validate all license lock slices Scan isolated runtime environments across every supported Python version. Require exact version and license exception reports, preserve SPDX WITH terms, and accept only complete canonical MIT text. Signed-off-by: Nabin Mulepati --- .github/workflows/ci.yml | 13 +-- Makefile | 6 +- dependency-license-policy.toml | 29 +++--- scripts/check_dependency_licenses.py | 64 ++++++++++-- tests_e2e/test_dependency_license_check.py | 108 +++++++++++++++++++-- 5 files changed, 183 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e900bfd9..5ba24ff3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,9 +250,13 @@ jobs: run: make check-license-headers dependency-licenses: - name: Check Dependency Licenses + name: Check Dependency Licenses (Python ${{ matrix.python-version }}) needs: validate-dispatch runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout code @@ -262,14 +266,11 @@ jobs: uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: version: "latest" - python-version: "3.11" + python-version: ${{ matrix.python-version }} enable-cache: true - - name: Install locked runtime dependencies - run: uv sync --all-packages --no-dev --locked - - name: Check dependency licenses - run: make check-dependency-licenses + run: make check-dependency-licenses LICENSE_PYTHON_VERSION=${{ matrix.python-version }} # =========================================================================== # Summary Job for Branch Protection diff --git a/Makefile b/Makefile index 7dadcb7b4..8dc026a26 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ REPO_PATH := $(shell pwd) PRE_COMMIT ?= .venv/bin/pre-commit +LICENSE_PYTHON_VERSION ?= 3.11 # Package directories CONFIG_PKG := packages/data-designer-config @@ -454,8 +455,9 @@ show-versions: # ============================================================================== check-dependency-licenses: - @echo "🔍 Checking runtime dependency licenses..." - uv run --no-sync python $(REPO_PATH)/scripts/check_dependency_licenses.py + @echo "🔍 Checking Python $(LICENSE_PYTHON_VERSION) locked runtime dependency licenses..." + uv run --isolated --python $(LICENSE_PYTHON_VERSION) --all-packages --no-dev --locked \ + python $(REPO_PATH)/scripts/check_dependency_licenses.py check-license-headers: @echo "🔍 Checking license headers in all files..." diff --git a/dependency-license-policy.toml b/dependency-license-policy.toml index ce9be8332..d7a346195 100644 --- a/dependency-license-policy.toml +++ b/dependency-license-policy.toml @@ -13,41 +13,44 @@ allowed_licenses = [ "Zlib", ] -# Existing dependencies outside the global whitelist are reviewed by package -# and exact reported license. A license change or a new package with one of -# these licenses must be reviewed explicitly. +# Existing dependencies outside the global whitelist are reviewed by package, +# version, and exact reported license. A version or license metadata change must +# be reviewed explicitly. [exceptions.certifi] -license = "Mozilla Public License 2.0 (MPL 2.0)" reason = "Unmodified TLS certificate bundle installed as a separate transitive dependency." +reports = [{ version = "2026.2.25", license = "Mozilla Public License 2.0 (MPL 2.0)" }] [exceptions.chardet] -license = "GNU Lesser General Public License v2 or later (LGPLv2+)" reason = "Unmodified character-detection library installed as a separate runtime dependency." +reports = [{ version = "5.2.0", license = "GNU Lesser General Public License v2 or later (LGPLv2+)" }] [exceptions.email-validator] -license = "The Unlicense (Unlicense)" reason = "Existing unmodified validation library installed as a separate runtime dependency." +reports = [{ version = "2.3.0", license = "The Unlicense (Unlicense)" }] [exceptions.numpy] -license = "BSD-3-Clause AND 0BSD AND MIT AND Zlib AND CC0-1.0" -reason = "Core numerical dependency with bundled components under additional permissive licenses." +reason = "Core numerical dependency with reviewed permissive license metadata in each locked Python slice." +reports = [ + { version = "2.2.6", license = "BSD License" }, + { version = "2.4.3", license = "BSD-3-Clause AND 0BSD AND MIT AND Zlib AND CC0-1.0" }, +] [exceptions.pathspec] -license = "Mozilla Public License 2.0 (MPL 2.0)" reason = "Unmodified transitive dependency of sqlfluff installed as a separate package." +reports = [{ version = "1.0.4", license = "Mozilla Public License 2.0 (MPL 2.0)" }] [exceptions.pillow] -license = "MIT-CMU" reason = "Core image dependency under the permissive MIT-CMU license." +reports = [{ version = "12.3.0", license = "MIT-CMU" }] [exceptions.regex] -license = "Apache-2.0 AND CNRI-Python" reason = "Existing regex dependency containing code under the CNRI Python license." +reports = [{ version = "2026.2.28", license = "Apache-2.0 AND CNRI-Python" }] [exceptions.tqdm] -license = "MPL-2.0 AND MIT" reason = "Unmodified progress library installed as a separate transitive dependency." +reports = [{ version = "4.67.3", license = "MPL-2.0 AND MIT" }] [exceptions.typing_extensions] -license = "PSF-2.0" reason = "Python typing compatibility dependency under the PSF license." +reports = [{ version = "4.15.0", license = "PSF-2.0" }] diff --git a/scripts/check_dependency_licenses.py b/scripts/check_dependency_licenses.py index 513cc189b..da7d50ae5 100644 --- a/scripts/check_dependency_licenses.py +++ b/scripts/check_dependency_licenses.py @@ -29,11 +29,37 @@ "The Unlicense (Unlicense)": {"Unlicense"}, } EXPRESSION_TOKEN = re.compile(r"(\(|\)|\bAND\b|\bOR\b|\bWITH\b)") +MIT_LICENSE_HEADINGS = {"mit license", "the mit license (mit)"} +MIT_LICENSE_BODY = " ".join( + """Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the \"Software\"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.""".split() +).casefold() @dataclass(frozen=True) -class ExceptionPolicy: +class ExceptionReport: + version: str license: str + + +@dataclass(frozen=True) +class ExceptionPolicy: + reports: frozenset[ExceptionReport] reason: str @@ -55,16 +81,31 @@ def load_policy(path: Path = POLICY_PATH) -> LicensePolicy: """Load the dependency-license policy from TOML.""" data = tomllib.loads(path.read_text(encoding="utf-8")) exceptions = { - name.casefold(): ExceptionPolicy(license=value["license"], reason=value["reason"]) + name.casefold(): ExceptionPolicy( + reports=frozenset( + ExceptionReport(version=report["version"], license=report["license"]) for report in value["reports"] + ), + reason=value["reason"], + ) for name, value in data.get("exceptions", {}).items() } return LicensePolicy(allowed_licenses=frozenset(data["allowed_licenses"]), exceptions=exceptions) def looks_like_mit_license(license_text: str) -> bool: - normalized = " ".join(license_text.split()).casefold() - has_mit_heading = normalized.startswith(("mit license ", "the mit license (mit) ")) - return has_mit_heading and "permission is hereby granted, free of charge" in normalized + lines = [line.strip() for line in license_text.splitlines() if line.strip()] + if len(lines) < 3 or lines[0].casefold() not in MIT_LICENSE_HEADINGS: + return False + + permission_index = next( + (index for index, line in enumerate(lines) if line.casefold().startswith("permission is hereby granted")), None + ) + if permission_index is None or permission_index < 2: + return False + if not all(line.casefold().startswith("copyright") for line in lines[1:permission_index]): + return False + + return " ".join(lines[permission_index:]).casefold() == MIT_LICENSE_BODY def _and_alternatives( @@ -118,7 +159,9 @@ def _parse_primary(self) -> tuple[frozenset[str], ...]: if self._accept("WITH"): if self.position >= len(self.tokens) or self.tokens[self.position] in {"(", ")", "AND", "OR", "WITH"}: return () + exception_id = self.tokens[self.position] self.position += 1 + identifiers = frozenset(f"{identifier} WITH {exception_id}" for identifier in identifiers) return (identifiers,) @@ -182,10 +225,15 @@ def evaluate_report(packages: list[PackageLicense], policy: LicensePolicy) -> tu exception = policy.exceptions.get(package_key) if exception is not None: seen_exceptions.add(package_key) - if package.license != exception.license: + report = ExceptionReport(version=package.version, license=package.license) + if report not in exception.reports: + expected = ", ".join( + f"{approved.version} with {approved.license!r}" + for approved in sorted(exception.reports, key=lambda item: (item.version, item.license)) + ) violations.append( - f"{package.name}=={package.version}: exception expected {exception.license!r}, " - f"but package reports {package.license!r}" + f"{package.name}=={package.version}: exception does not approve {package.license!r}; " + f"expected one of: {expected}" ) else: reviewed.append(f"{package.name}=={package.version}: {package.license} ({exception.reason})") diff --git a/tests_e2e/test_dependency_license_check.py b/tests_e2e/test_dependency_license_check.py index 66fa6fc7a..081e71595 100644 --- a/tests_e2e/test_dependency_license_check.py +++ b/tests_e2e/test_dependency_license_check.py @@ -9,6 +9,28 @@ from types import ModuleType SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "check_dependency_licenses.py" +MIT_LICENSE_TEXT = """MIT License + +Copyright (c) 2026 Example + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" def load_script() -> ModuleType: @@ -78,17 +100,29 @@ def test_evaluate_report_rejects_disallowed_or_alternatives_and_malformed_expres ] -def test_evaluate_report_uses_base_license_with_exception() -> None: +def test_evaluate_report_requires_with_exception_approval() -> None: module = load_script() policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={}) packages = [ - module.PackageLicense("allowed", "1", "Apache-2.0 WITH LLVM-exception", ""), + module.PackageLicense("unknown", "1", "Apache-2.0 WITH Totally-Unknown-terms", ""), module.PackageLicense("rejected", "1", "GPL-2.0-only WITH Classpath-exception-2.0", ""), ] violations, _ = module.evaluate_report(packages, policy) - assert violations == ["rejected==1: disallowed license(s): GPL-2.0-only"] + assert violations == [ + "rejected==1: disallowed license(s): GPL-2.0-only WITH Classpath-exception-2.0", + "unknown==1: disallowed license(s): Apache-2.0 WITH Totally-Unknown-terms", + ] + + approved_policy = module.LicensePolicy( + allowed_licenses=frozenset({"Apache-2.0", "Apache-2.0 WITH LLVM-exception"}), exceptions={} + ) + approved = module.PackageLicense("approved", "1", "Apache-2.0 WITH LLVM-exception", "") + + approved_violations, _ = module.evaluate_report([approved], approved_policy) + + assert approved_violations == [] def test_evaluate_report_ignores_empty_semicolon_segments() -> None: @@ -125,13 +159,13 @@ def test_evaluate_report_recognizes_bundled_mit_license_text() -> None: "missing-metadata", "1", "UNKNOWN", - "MIT License\n\nCopyright example\n\nPermission is hereby granted, free of charge, to any person obtaining a copy", + MIT_LICENSE_TEXT, ), module.PackageLicense( "alternate-heading", "1", "UNKNOWN", - "The MIT License (MIT)\n\nCopyright example\n\nPermission is hereby granted, free of charge, to any person", + MIT_LICENSE_TEXT.replace("MIT License", "The MIT License (MIT)", 1), ), ] @@ -140,21 +174,79 @@ def test_evaluate_report_recognizes_bundled_mit_license_text() -> None: assert violations == [] +def test_evaluate_report_rejects_incomplete_or_modified_mit_license_text() -> None: + module = load_script() + policy = module.LicensePolicy(allowed_licenses=frozenset({"MIT"}), exceptions={}) + packages = [ + module.PackageLicense( + "additional-restriction", + "1", + "UNKNOWN", + f"{MIT_LICENSE_TEXT}\nUse is prohibited in commercial products.", + ), + module.PackageLicense( + "truncated", + "1", + "UNKNOWN", + "MIT License\n\nCopyright example\n\nPermission is hereby granted, free of charge", + ), + ] + + violations, _ = module.evaluate_report(packages, policy) + + assert violations == [ + "additional-restriction==1: unknown or unrecognized license 'UNKNOWN'", + "truncated==1: unknown or unrecognized license 'UNKNOWN'", + ] + + def test_evaluate_report_requires_exception_license_to_remain_unchanged() -> None: module = load_script() - exception = module.ExceptionPolicy(license="MPL-2.0", reason="Reviewed separately.") + exception = module.ExceptionPolicy( + reports=frozenset({module.ExceptionReport(version="1", license="MPL-2.0")}), + reason="Reviewed separately.", + ) policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={"example": exception}) package = module.PackageLicense("example", "2", "GPL-3.0-only", "") violations, reviewed = module.evaluate_report([package], policy) - assert violations == ["example==2: exception expected 'MPL-2.0', but package reports 'GPL-3.0-only'"] + assert violations == ["example==2: exception does not approve 'GPL-3.0-only'; expected one of: 1 with 'MPL-2.0'"] assert reviewed == [] +def test_evaluate_report_accepts_each_exact_exception_report() -> None: + module = load_script() + exception = module.ExceptionPolicy( + reports=frozenset( + { + module.ExceptionReport(version="1", license="Legacy license"), + module.ExceptionReport(version="2", license="Current license"), + } + ), + reason="Reviewed per locked slice.", + ) + policy = module.LicensePolicy(allowed_licenses=frozenset(), exceptions={"example": exception}) + packages = [ + module.PackageLicense("example", "1", "Legacy license", ""), + module.PackageLicense("example", "2", "Current license", ""), + ] + + violations, reviewed = module.evaluate_report(packages, policy) + + assert violations == [] + assert reviewed == [ + "example==1: Legacy license (Reviewed per locked slice.)", + "example==2: Current license (Reviewed per locked slice.)", + ] + + def test_evaluate_report_rejects_stale_exceptions() -> None: module = load_script() - exception = module.ExceptionPolicy(license="MPL-2.0", reason="Reviewed separately.") + exception = module.ExceptionPolicy( + reports=frozenset({module.ExceptionReport(version="1", license="MPL-2.0")}), + reason="Reviewed separately.", + ) policy = module.LicensePolicy(allowed_licenses=frozenset({"Apache-2.0"}), exceptions={"removed": exception}) violations, _ = module.evaluate_report([], policy) From b048ee444f3d9ca8952be7cd44b2a3679f6e1209 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Thu, 16 Jul 2026 10:57:00 -0600 Subject: [PATCH 4/5] chore: remove email-validator license exception Remove the reviewed exception after PR #824 dropped email-validator from the locked runtime dependency graph. Signed-off-by: Nabin Mulepati --- dependency-license-policy.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dependency-license-policy.toml b/dependency-license-policy.toml index d7a346195..373a981a8 100644 --- a/dependency-license-policy.toml +++ b/dependency-license-policy.toml @@ -24,10 +24,6 @@ reports = [{ version = "2026.2.25", license = "Mozilla Public License 2.0 (MPL 2 reason = "Unmodified character-detection library installed as a separate runtime dependency." reports = [{ version = "5.2.0", license = "GNU Lesser General Public License v2 or later (LGPLv2+)" }] -[exceptions.email-validator] -reason = "Existing unmodified validation library installed as a separate runtime dependency." -reports = [{ version = "2.3.0", license = "The Unlicense (Unlicense)" }] - [exceptions.numpy] reason = "Core numerical dependency with reviewed permissive license metadata in each locked Python slice." reports = [ From 322f72aec9c398dae7219e24983ace41824ed07b Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Thu, 16 Jul 2026 10:59:11 -0600 Subject: [PATCH 5/5] fix: declare license checker fallback Move the Python 3.10 tomli fallback into a dedicated dependency group and request it explicitly from the isolated license-check target. Signed-off-by: Nabin Mulepati --- Makefile | 2 +- pyproject.toml | 2 ++ uv.lock | 6 ++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8dc026a26..40869ae18 100644 --- a/Makefile +++ b/Makefile @@ -456,7 +456,7 @@ show-versions: check-dependency-licenses: @echo "🔍 Checking Python $(LICENSE_PYTHON_VERSION) locked runtime dependency licenses..." - uv run --isolated --python $(LICENSE_PYTHON_VERSION) --all-packages --no-dev --locked \ + uv run --isolated --python $(LICENSE_PYTHON_VERSION) --all-packages --no-dev --group license-check --locked \ python $(REPO_PATH)/scripts/check_dependency_licenses.py check-license-headers: diff --git a/pyproject.toml b/pyproject.toml index a84db2b0e..cba354e04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,8 @@ dev = [ "pytest-env>=1.2.0,<2", "pytest-httpx>=0.36.0,<1", "ruff>=0.14.10,<1", +] +license-check = [ "tomli>=2.0.1,<3; python_version < '3.11'", ] docs = [ diff --git a/uv.lock b/uv.lock index 49d62b3ba..3420cd1c5 100644 --- a/uv.lock +++ b/uv.lock @@ -965,11 +965,13 @@ dev = [ { name = "pytest-env" }, { name = "pytest-httpx" }, { name = "ruff" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, ] docs = [ { name = "jupytext" }, ] +license-check = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] notebooks = [ { name = "aiohttp" }, { name = "bleach" }, @@ -1000,9 +1002,9 @@ dev = [ { name = "pytest-env", specifier = ">=1.2.0,<2" }, { name = "pytest-httpx", specifier = ">=0.36.0,<1" }, { name = "ruff", specifier = ">=0.14.10,<1" }, - { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.1,<3" }, ] docs = [{ name = "jupytext", specifier = ">=1.16.0,<2" }] +license-check = [{ name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.1,<3" }] notebooks = [ { name = "aiohttp", specifier = ">=3.14.1,<4" }, { name = "bleach", specifier = ">=6.4.0,<7" },