From e2a272e7638a5a6ecb7751cac78a2dbefd0b3313 Mon Sep 17 00:00:00 2001 From: stacknil Date: Fri, 12 Jun 2026 12:04:31 +0800 Subject: [PATCH] test(docs): cover reviewer surface ci paths --- scripts/validate-reviewer-routes.py | 76 +++++++++++++++---- .../sbom-diff-and-risk/docs/reviewer-path.md | 31 +++++++- 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/scripts/validate-reviewer-routes.py b/scripts/validate-reviewer-routes.py index ebc30a0..34ed49f 100644 --- a/scripts/validate-reviewer-routes.py +++ b/scripts/validate-reviewer-routes.py @@ -1,6 +1,8 @@ from __future__ import annotations +from fnmatch import fnmatchcase import re +import subprocess import sys from pathlib import Path from urllib.parse import unquote @@ -129,8 +131,13 @@ Path("tools/sbom-diff-and-risk/docs/reviewer-path.md"): ( "Artifact evidence map", "Reviewer route contract", + "Reviewer outcome statements", + "What can I safely say in review?", + "Your summary separates verified evidence from non-claims.", + "Do not write:", "Markdown links across the reviewer surface resolve", "workflow path filters cover reviewer-surface changes", + "every tracked reviewer-surface Markdown file is covered", "python scripts/validate-reviewer-routes.py", "No network", "not current PyPI package truth", @@ -280,20 +287,37 @@ def iter_reviewer_surface_markdown(errors: list[str]) -> tuple[Path, ...]: absolute_root = REPO_ROOT / root if not absolute_root.exists(): errors.append(f"missing reviewer surface root: {root.as_posix()}") - continue - if absolute_root.is_file(): - candidates = (absolute_root,) if absolute_root.suffix.lower() == ".md" else () - else: - candidates = sorted(absolute_root.rglob("*.md")) + tracked_paths = subprocess.run( + [ + "git", + "-C", + str(REPO_ROOT), + "ls-files", + "--", + *(root.as_posix() for root in REVIEWER_SURFACE_ROOTS), + ], + capture_output=True, + text=True, + check=False, + ) + if tracked_paths.returncode != 0: + errors.append( + "failed to list tracked reviewer surface files: " + f"{tracked_paths.stderr.strip()}" + ) + return tuple() + + for raw_path in tracked_paths.stdout.splitlines(): + relative_path = Path(raw_path) + if relative_path.suffix.lower() != ".md": + continue - for absolute_path in candidates: - relative_path = absolute_path.relative_to(REPO_ROOT) - if relative_path in seen: - continue + if relative_path in seen: + continue - seen.add(relative_path) - markdown_paths.append(relative_path) + seen.add(relative_path) + markdown_paths.append(relative_path) return tuple(markdown_paths) @@ -338,6 +362,21 @@ def workflow_path_filters(workflow_path: Path, event_name: str, errors: list[str return filters +def path_filter_matches(path_filter: str, path: Path) -> bool: + path_text = path.as_posix() + filter_text = path_filter.strip("/") + + if filter_text.endswith("/**"): + prefix = filter_text.removesuffix("/**") + return path_text == prefix or path_text.startswith(f"{prefix}/") + + parent_filter, separator, name_filter = filter_text.rpartition("/") + if separator and any(character in name_filter for character in "*?[]"): + return path.parent.as_posix() == parent_filter and fnmatchcase(path.name, name_filter) + + return path_text == filter_text + + def iter_local_links(markdown_path: Path) -> set[str]: text = read_markdown(markdown_path) raw_targets = INLINE_LINK_RE.findall(text) @@ -422,7 +461,9 @@ def validate_required_paths(errors: list[str]) -> None: errors.append(f"missing supporting-project boundary file: {path.as_posix()}") -def validate_workflow_path_filters(errors: list[str]) -> None: +def validate_workflow_path_filters( + reviewer_surface_markdown: tuple[Path, ...], errors: list[str] +) -> None: for event_name in WORKFLOW_EVENTS_WITH_PATH_FILTERS: filters = workflow_path_filters(WORKFLOW_PATH, event_name, errors) if not filters: @@ -436,6 +477,13 @@ def validate_workflow_path_filters(errors: list[str]) -> None: f"{required_filter!r}" ) + for markdown_path in reviewer_surface_markdown: + if not any(path_filter_matches(path_filter, markdown_path) for path_filter in filters): + errors.append( + f"{WORKFLOW_PATH}: {event_name} path filters do not cover " + f"{markdown_path.as_posix()}" + ) + def main() -> int: errors: list[str] = [] @@ -453,7 +501,7 @@ def main() -> int: validate_required_text(markdown_path, errors) validate_required_paths(errors) - validate_workflow_path_filters(errors) + validate_workflow_path_filters(reviewer_surface_markdown, errors) if errors: print("Reviewer route validation failed:", file=sys.stderr) @@ -466,7 +514,7 @@ def main() -> int: f"{len(DOCS_TO_VALIDATE)} documents and " f"{len(REQUIRED_REVIEWER_PATHS)} reviewer paths checked; " f"{len(reviewer_surface_markdown)} reviewer-surface markdown files " - "link-checked; workflow path filters checked." + "link-checked; workflow path filters and coverage checked." ) return 0 diff --git a/tools/sbom-diff-and-risk/docs/reviewer-path.md b/tools/sbom-diff-and-risk/docs/reviewer-path.md index e885288..dc8f63b 100644 --- a/tools/sbom-diff-and-risk/docs/reviewer-path.md +++ b/tools/sbom-diff-and-risk/docs/reviewer-path.md @@ -13,6 +13,7 @@ where to find it, and what it does not prove. | Can the examples be reproduced locally? | [15-minute reproduction check](#15-minute-reproduction-check) | `regenerate-example-artifacts.py --check` passes without enrichment. | | Can the released tool artifacts be verified? | [Release evidence](#release-evidence) | You can choose the correct GitHub release, checksum, or attestation path. | | Are the reviewer routes still valid? | [Reviewer route contract](#reviewer-route-contract) | `python scripts/validate-reviewer-routes.py` passes from the repository root. | +| What can I safely say in review? | [Reviewer outcome statements](#reviewer-outcome-statements) | Your summary separates verified evidence from non-claims. | | Is this enough for a full review? | [Deep review](#deep-review) | You have followed the reproducible checklist in the evidence pack. | ## 30-second orientation @@ -135,7 +136,8 @@ This checks that the repository reviewer route still has the expected local links, markdown anchors, reviewer-path documents, supporting-project boundary files, and required non-claim phrases. It also checks that Markdown links across the reviewer surface resolve and that workflow path filters cover -reviewer-surface changes. +reviewer-surface changes, including whether every tracked reviewer-surface +Markdown file is covered by those filters. Use this when you change reviewer-facing docs, examples, or supporting project entry points. The contract lives in @@ -149,12 +151,39 @@ Expected result: - local markdown anchors resolve - Markdown links across the reviewer surface resolve - workflow path filters cover reviewer-surface changes +- every tracked reviewer-surface Markdown file is covered by the workflow path filters - supporting project reviewer paths and boundary files still exist - required non-claims remain present in reviewer-facing docs Stop here if your review question is whether the reviewer route itself is still coherent after documentation changes. +## Reviewer outcome statements + +Use this wording when you need a concise review summary. Each statement maps to +the evidence path that supports it. + +| Review result | Safe statement | +| --- | --- | +| 30-second orientation completed | `sbom-diff-and-risk` is a local deterministic SBOM/dependency diff CLI with JSON, Markdown, SARIF, summary, and policy-sidecar outputs. | +| Artifact review completed | The checked-in examples show the default report shape, summary contract, policy sidecar, Markdown report, SARIF output, mocked enrichment snapshots, and CI workflow templates. | +| Local reproduction completed | The no-network checked-in examples are up to date with the current code according to `scripts/regenerate-example-artifacts.py --check`. | +| Route contract completed | The reviewer route still has required local links, anchors, path filters, boundary files, and non-claim phrases according to `scripts/validate-reviewer-routes.py`. | +| Release evidence reviewed | The tool's release artifacts have separate checksum, release-verification, workflow-attestation, and TestPyPI evidence paths. | + +Do not write: + +- `sbom-diff-and-risk` is a vulnerability scanner +- the checked-in examples prove a third-party dependency is safe +- mocked provenance or Scorecard snapshots represent current live package or + repository truth +- TestPyPI validation means production PyPI publishing is enabled +- GitHub Release checksums, workflow artifact attestations, and PyPI Trusted + Publishing prove the same thing + +Stop here if you need reviewer-safe wording for a PR description, review note, +or project summary. + ## Release evidence Use this section only when the review question is about the released