From 3fc106b1061620bfb18d76674c8857e6f7f973e1 Mon Sep 17 00:00:00 2001 From: Justin McLean Date: Mon, 29 Jun 2026 17:40:21 +1000 Subject: [PATCH 1/3] feat(meta): add adopter pilot-report template and validator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tools/pilot-report-validator — a stdlib-only Python tool that validates adopter pilot-report Markdown files against a required schema (frontmatter keys skill/date/target_repo/profile, valid profile value asf|non-asf|custom, and seven required body sections). Add docs/pilot-report-template.md — the template adopters copy and fill in to record a skill-family pilot run (blocked preflights, false positives, confirmation points, privacy/adapter notes, proposed spec changes). Wire the template reference into the experimental family Status sections for mentoring, pairing, repo-health, and issue-management so pilot evidence is captured consistently. Add the tool to docs/labels-and-capabilities.md and pyproject.toml workspace members; add a .gitignore exception so the committed framework tool is not caught by the evidence-package pilot-*/ pattern. Generated-by: Claude (Opus 4.7) --- .gitignore | 2 + docs/issue-management/README.md | 9 + docs/labels-and-capabilities.md | 1 + docs/mentoring/README.md | 9 + docs/pairing/README.md | 9 + docs/pilot-report-template.md | 112 +++++ docs/repo-health/README.md | 15 + pyproject.toml | 1 + tools/pilot-report-validator/README.md | 72 ++++ tools/pilot-report-validator/pyproject.toml | 58 +++ .../src/pilot_report_validator/__init__.py | 269 ++++++++++++ .../tests/test_pilot_report_validator.py | 400 ++++++++++++++++++ tools/pilot-report-validator/uv.lock | 112 +++++ uv.lock | 20 + 14 files changed, 1089 insertions(+) create mode 100644 docs/pilot-report-template.md create mode 100644 tools/pilot-report-validator/README.md create mode 100644 tools/pilot-report-validator/pyproject.toml create mode 100644 tools/pilot-report-validator/src/pilot_report_validator/__init__.py create mode 100644 tools/pilot-report-validator/tests/test_pilot_report_validator.py create mode 100644 tools/pilot-report-validator/uv.lock diff --git a/.gitignore b/.gitignore index 68453456..9a5a88d0 100644 --- a/.gitignore +++ b/.gitignore @@ -60,4 +60,6 @@ __pycache__/ campaign/ campaigns/ pilot-*/ +# tools/pilot-report-validator is a committed framework tool, not an evidence package +!/tools/pilot-report-validator/ reassess-*/ diff --git a/docs/issue-management/README.md b/docs/issue-management/README.md index 074b88ee..57e857c8 100644 --- a/docs/issue-management/README.md +++ b/docs/issue-management/README.md @@ -114,6 +114,15 @@ adopter's `/` directory: **Experimental.** No adopter pilot has run an evaluation against this family yet. Shape may change between framework versions. +To provide pilot feedback, copy +[`docs/pilot-report-template.md`](../pilot-report-template.md) into your +project notes, fill in each section, and optionally validate the filled-in +report with: + +```bash +uv run --project tools/pilot-report-validator pilot-report-validate +``` + ## Cross-references - [Top-level README — Adopting the framework](../../README.md#adopting-the-framework) — 3-step bootstrap. diff --git a/docs/labels-and-capabilities.md b/docs/labels-and-capabilities.md index 4b81cd30..5d2d148b 100644 --- a/docs/labels-and-capabilities.md +++ b/docs/labels-and-capabilities.md @@ -225,6 +225,7 @@ Tools under [`tools/`](../tools/). Tools with two values (separated by | [`tools/pr-management-stats`](../tools/pr-management-stats/) | `capability:stats` | PR-backlog analytics engine | | [`tools/preflight-audit`](../tools/preflight-audit/) | `capability:stats` | Dry-run the bulk-mode pre-flight classifier; measure skip-rate before / after any rule edit in the security-issue-sync skill | | [`tools/privacy-llm`](../tools/privacy-llm/) | `capability:setup` | Privacy-LLM PII-scrubbing gate | +| [`tools/pilot-report-validator`](../tools/pilot-report-validator/) | `capability:setup` | Validates adopter pilot-report files — required frontmatter keys, valid profile value, and required body sections | | [`tools/probe-templates`](../tools/probe-templates/) | `capability:setup` | Sandbox-doctor probe templates | | [`tools/sandbox-lint`](../tools/sandbox-lint/) | `capability:setup` | Sandbox settings linter | | [`tools/security-tracker-stats-dashboard`](../tools/security-tracker-stats-dashboard/) | `capability:stats` | Security-tracker analytics engine | diff --git a/docs/mentoring/README.md b/docs/mentoring/README.md index 1b635174..c475be34 100644 --- a/docs/mentoring/README.md +++ b/docs/mentoring/README.md @@ -91,6 +91,15 @@ required key documentation. contributor-to-committer interaction path under evaluation conditions yet; shape may change between framework versions. +To provide pilot feedback, copy +[`docs/pilot-report-template.md`](../pilot-report-template.md) into your +project notes, fill in each section, and optionally validate the filled-in +report with: + +```bash +uv run --project tools/pilot-report-validator pilot-report-validate +``` + ## Cross-references - [`MISSION.md` § Agentic Mentoring](../../MISSION.md#technical-scope) — diff --git a/docs/pairing/README.md b/docs/pairing/README.md index 2d59705f..20a8333b 100644 --- a/docs/pairing/README.md +++ b/docs/pairing/README.md @@ -94,6 +94,15 @@ project's tracker, label set, or any shared infrastructure. `skill-and-tool-validate`. No adopter-pilot evaluation has run yet; shape may change between framework versions. +To provide pilot feedback, copy +[`docs/pilot-report-template.md`](../pilot-report-template.md) into your +project notes, fill in each section, and optionally validate the filled-in +report with: + +```bash +uv run --project tools/pilot-report-validator pilot-report-validate +``` + --- ## Cross-references diff --git a/docs/pilot-report-template.md b/docs/pilot-report-template.md new file mode 100644 index 00000000..32ad4483 --- /dev/null +++ b/docs/pilot-report-template.md @@ -0,0 +1,112 @@ +--- +skill: +date: YYYY-MM-DD +target_repo: / +profile: asf +reporter: +--- + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Pilot report template](#pilot-report-template) + - [Instructions](#instructions) +- [Pilot report: \ on \](#pilot-report-skill-name-on-ownerrepo) + - [Skill or family](#skill-or-family) + - [Target repo and profile](#target-repo-and-profile) + - [Blocked preflights](#blocked-preflights) + - [False positives](#false-positives) + - [Confirmation points](#confirmation-points) + - [Privacy and adapter notes](#privacy-and-adapter-notes) + - [Proposed spec changes](#proposed-spec-changes) + + + + + +# Pilot report template + +Copy this file into your project notes, fill in each section, and share +it back with the framework maintainers (or keep it local). Pilot +reports are the primary feedback channel for moving a skill family from +`experimental` to `stable`. + +To validate a filled-in report file, run: + +```bash +uv run --project tools/pilot-report-validator pilot-report-validate +``` + +--- + +## Instructions + +1. Copy this file to a convenient location (e.g. + `/pilot-report--.md`). +2. Fill in the frontmatter block at the very top of the file (keep it + at the top, above the table of contents, so the validator detects it) + and each section below. +3. Replace placeholder text in *italics* with your findings. +4. For any section where nothing applies, write: **None observed.** +5. For proposed spec changes, reference the spec file path and section + where possible. + +--- + +# Pilot report: \ on \ + +## Skill or family + +*Which skill or skill family was piloted — e.g. `pairing-self-review`, +`mentoring`, `repo-health`. If you ran a full family sweep, list each +skill you exercised.* + +## Target repo and profile + +*The repository tested against (`owner/repo`) and the project profile +used (`asf`, `non-asf`, or `custom`). If your project-config directory +is public, link to it here.* + +## Blocked preflights + +*List any preflights the skill ran that blocked, were skipped, or +produced a confusing result. Include the preflight name or description +and why it triggered.* + +*If none: **None observed.*** + +## False positives + +*Findings the skill surfaced that were incorrect, irrelevant, or +misleading. Include the finding summary and why it was a false positive. +Distinguish clearly between "wrong finding" and "right finding but +unhelpful wording".* + +*If none: **None observed.*** + +## Confirmation points + +*Steps where the skill prompted for maintainer confirmation before +proceeding. Note any that felt misplaced — too early, too late, or +absent when one was expected.* + +*If no issues: **All confirmation points felt appropriate.*** + +## Privacy and adapter notes + +*Any Privacy-LLM gate activations, adapter mismatches, credential +preflight issues, or unexpected external-content handling. Note which +adapter path was exercised (e.g. GitHub Issues, GitHub PR, Gmail, +PonyMail) and whether the data-not-instructions boundary held.* + +*If none: **None observed.*** + +## Proposed spec changes + +*Specific changes to propose to the skill spec or adopter-contract docs. +For each proposal, note the spec file path and section, the current +wording, and the suggested change.* + +*If none: **No changes proposed at this time.*** diff --git a/docs/repo-health/README.md b/docs/repo-health/README.md index a38888f4..5412e600 100644 --- a/docs/repo-health/README.md +++ b/docs/repo-health/README.md @@ -9,6 +9,7 @@ - [`dependency-audit` (experimental)](#dependency-audit-experimental) - [`license-compliance-audit` (experimental)](#license-compliance-audit-experimental) - [`flaky-test-triage` (experimental)](#flaky-test-triage-experimental) + - [Status](#status) - [Adopter contract](#adopter-contract) - [Cross-references](#cross-references) @@ -132,6 +133,20 @@ to include or exclude. --- +## Status + +**Experimental.** All five skills shipped. No adopter-pilot evaluation +has run end-to-end yet; shape may change between framework versions. + +To provide pilot feedback, copy +[`docs/pilot-report-template.md`](../pilot-report-template.md) into your +project notes, fill in each section, and optionally validate the filled-in +report with: + +```bash +uv run --project tools/pilot-report-validator pilot-report-validate +``` + ## Adopter contract `projects/_template/repo-health-config.md` provides the per-project diff --git a/pyproject.toml b/pyproject.toml index b70c1897..77a94efb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,7 @@ members = [ "tools/security-tracker-stats-dashboard", "tools/skill-and-tool-validator", "tools/skill-evals", + "tools/pilot-report-validator", "tools/spec-status-index", "tools/spec-validator", "tools/vcs", diff --git a/tools/pilot-report-validator/README.md b/tools/pilot-report-validator/README.md new file mode 100644 index 00000000..be27ffbf --- /dev/null +++ b/tools/pilot-report-validator/README.md @@ -0,0 +1,72 @@ + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [pilot-report-validator](#pilot-report-validator) + - [Prerequisites](#prerequisites) + - [What it checks](#what-it-checks) + - [Usage](#usage) + - [Writing a pilot report](#writing-a-pilot-report) + + + + + +# pilot-report-validator + +**Capability:** capability:setup + +Validates adopter pilot-report files against the required schema defined +in [`docs/pilot-report-template.md`](../../docs/pilot-report-template.md). +Pilot reports are the feedback artefact that documents an adopter's +end-to-end run of an experimental skill family, and are the primary +evidence source for advancing a skill from `experimental` to `stable`. + +## Prerequisites + +- **Runtime:** Python 3.11+ run via `uv`; stdlib-only (no runtime + dependencies). The `dev` group pulls `pytest`, `ruff`. +- **CLIs:** None beyond the runtime. +- **Credentials / auth:** None. +- **Network:** Runs fully offline against local report files. + +## What it checks + +For every `.md` file that carries a YAML frontmatter block: + +1. **Required frontmatter keys** — `skill`, `date`, `target_repo`, + `profile`. +2. **Valid `profile` value** — `asf` | `non-asf` | `custom`. +3. **Required body sections** — `## Skill or family`, + `## Target repo and profile`, `## Blocked preflights`, + `## False positives`, `## Confirmation points`, + `## Privacy and adapter notes`, `## Proposed spec changes`. + +The frontmatter block must be at the very top of the file — YAML +frontmatter placed lower in the document is not detected. Files without +a top-of-file frontmatter block (e.g. README files) are silently +skipped. `docs/pilot-report-template.md` carries a top-of-file +frontmatter skeleton and so validates as a clean report. + +## Usage + +```bash +# Validate a single filled-in report +uv run --project tools/pilot-report-validator \ + pilot-report-validate path/to/my-pilot-report.md + +# Validate every report in a directory +uv run --project tools/pilot-report-validator \ + pilot-report-validate path/to/reports/ + +# Run the test suite (use --directory, not --project, to avoid the +# duplicate-`tests`-package collision when run from the repo root) +uv run --directory tools/pilot-report-validator --group dev pytest +``` + +## Writing a pilot report + +Copy `docs/pilot-report-template.md`, fill in the frontmatter and each +section, then validate with this tool before sharing. See the template +for detailed per-section instructions. diff --git a/tools/pilot-report-validator/pyproject.toml b/tools/pilot-report-validator/pyproject.toml new file mode 100644 index 00000000..123370af --- /dev/null +++ b/tools/pilot-report-validator/pyproject.toml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pilot-report-validator" +version = "0.1.0" +description = "Validate adopter pilot-report files — required frontmatter keys and body sections." +readme = "README.md" +requires-python = ">=3.11" +license = { text = "Apache-2.0" } +dependencies = [] + +[project.scripts] +pilot-report-validate = "pilot_report_validator:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/pilot_report_validator"] + +[tool.ruff] +line-length = 110 +target-version = "py311" +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "B", "UP", "SIM", "C4", "RUF"] +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["B", "SIM"] + +[dependency-groups] +dev = [ + "pytest>=8.0", + "ruff>=0.4", +] + +[tool.pytest.ini_options] +minversion = "8.0" +addopts = "-ra -q" +testpaths = ["tests"] diff --git a/tools/pilot-report-validator/src/pilot_report_validator/__init__.py b/tools/pilot-report-validator/src/pilot_report_validator/__init__.py new file mode 100644 index 00000000..0165b33a --- /dev/null +++ b/tools/pilot-report-validator/src/pilot_report_validator/__init__.py @@ -0,0 +1,269 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Validate adopter pilot-report Markdown files. + +Checks every .md file that carries a YAML frontmatter block for: + +1. Required frontmatter keys — skill, date, target_repo, profile. +2. Valid ``profile`` value — asf | non-asf | custom. +3. Required body sections — Skill or family, Target repo and profile, + Blocked preflights, False positives, Confirmation points, + Privacy and adapter notes, Proposed spec changes. + +The frontmatter block must be at the top of the file; YAML frontmatter +placed lower in the document is not detected. Files without a +top-of-file frontmatter block (e.g. README.md) are skipped silently. +``docs/pilot-report-template.md`` carries a top-of-file frontmatter +skeleton and so validates as a clean report. + +Run from repo root:: + + uv run --project tools/pilot-report-validator \ + pilot-report-validate docs/pilot-report-template.md +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +REQUIRED_FRONTMATTER_KEYS: frozenset[str] = frozenset({"skill", "date", "target_repo", "profile"}) + +ALLOWED_PROFILES: frozenset[str] = frozenset({"asf", "non-asf", "custom"}) + +REQUIRED_SECTIONS: tuple[str, ...] = ( + "Skill or family", + "Target repo and profile", + "Blocked preflights", + "False positives", + "Confirmation points", + "Privacy and adapter notes", + "Proposed spec changes", +) + +_HTML_COMMENT_RE = re.compile(r"") +_YAML_BLOCK_SCALAR_HEADERS: frozenset[str] = frozenset({"|", ">", "|-", "|+", ">-", ">+"}) + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +class Violation: + def __init__(self, path: Path, line: int | None, message: str) -> None: + self.path = path + self.line = line + self.message = message + + def __str__(self) -> str: + if self.line is not None: + return f"{self.path}:{self.line}: {self.message}" + return f"{self.path}: {self.message}" + + +# --------------------------------------------------------------------------- +# Frontmatter parsing +# --------------------------------------------------------------------------- + + +def _frontmatter_bounds(text: str) -> tuple[int, int] | None: + """Return (content_start, content_end) for the frontmatter block, or None. + + Handles files whose first non-whitespace content is an HTML comment + (e.g. an SPDX license header) before the opening ``---`` delimiter. + """ + idx = text.find("---\n") + if idx == -1: + return None + prefix = text[:idx] + clean = _HTML_COMMENT_RE.sub("", prefix).strip() + if clean: + return None + try: + end = text.index("\n---\n", idx + 4) + except ValueError: + return None + return (idx + 4, end) + + +def parse_frontmatter(text: str) -> dict[str, str] | None: + """Return a dict of top-level frontmatter key→value, or None if absent.""" + bounds = _frontmatter_bounds(text) + if bounds is None: + return None + block = text[bounds[0] : bounds[1]] + + result: dict[str, str] = {} + current_key: str | None = None + current_value_lines: list[str] = [] + + for raw_line in block.splitlines(): + line = raw_line.rstrip() + if line == "": + if current_key is not None: + current_value_lines.append("") + continue + if not line.startswith((" ", "\t")) and ":" in line: + if current_key is not None: + result[current_key] = "\n".join(current_value_lines).strip() + key, _, value = line.partition(":") + current_key = key.strip() + inline = value.strip() + current_value_lines = [inline] if inline and inline not in _YAML_BLOCK_SCALAR_HEADERS else [] + continue + if current_key is not None: + stripped = line[2:] if line.startswith(" ") else line + current_value_lines.append(stripped) + + if current_key is not None: + result[current_key] = "\n".join(current_value_lines).strip() + return result + + +# --------------------------------------------------------------------------- +# Body section helpers +# --------------------------------------------------------------------------- + + +def _spec_body(text: str) -> str: + """Return the document body — everything after the closing ``---`` delimiter.""" + bounds = _frontmatter_bounds(text) + if bounds is None: + return text + return text[bounds[1] + 5 :] + + +def extract_section_headings(text: str) -> set[str]: + """Return the text of every ## heading in the document body.""" + body = _spec_body(text) + headings: set[str] = set() + for line in body.splitlines(): + if line.startswith("## "): + headings.add(line[3:].strip()) + return headings + + +# --------------------------------------------------------------------------- +# Validators +# --------------------------------------------------------------------------- + + +def validate_frontmatter(path: Path, text: str) -> list[Violation]: + fm = parse_frontmatter(text) + if fm is None: + return [] # No frontmatter — not a report file; skip silently + + violations: list[Violation] = [] + + missing = REQUIRED_FRONTMATTER_KEYS - set(fm.keys()) + for key in sorted(missing): + violations.append(Violation(path, 1, f"missing required frontmatter key: '{key}'")) + + if "profile" in fm and fm["profile"] not in ALLOWED_PROFILES: + violations.append( + Violation( + path, + 1, + f"invalid profile '{fm['profile']}' — must be one of {sorted(ALLOWED_PROFILES)}", + ) + ) + + return violations + + +def validate_body(path: Path, text: str) -> list[Violation]: + if parse_frontmatter(text) is None: + return [] # Not a report file + + violations: list[Violation] = [] + headings = extract_section_headings(text) + + for section in REQUIRED_SECTIONS: + if section not in headings: + violations.append(Violation(path, None, f"missing required section: '## {section}'")) + + return violations + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + + +def validate_file(path: Path) -> list[Violation]: + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + return [Violation(path, None, f"cannot read file: {exc}")] + return validate_frontmatter(path, text) + validate_body(path, text) + + +def collect_report_files(target: Path) -> list[Path]: + """Return all .md files under *target* (or *target* itself if a file).""" + if target.is_file(): + return [target] + return sorted(target.rglob("*.md")) + + +def run_validation(target: Path) -> list[Violation]: + violations: list[Violation] = [] + for path in collect_report_files(target): + violations.extend(validate_file(path)) + return violations + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Validate adopter pilot-report files.", + epilog="Files without frontmatter (READMEs, templates) are silently skipped.", + ) + parser.add_argument( + "path", + help="Pilot-report file or directory to validate.", + ) + args = parser.parse_args(argv) + + target = Path(args.path) + if not target.exists(): + print(f"pilot-report-validate: path not found: {target}", file=sys.stderr) + return 1 + + violations = run_validation(target) + if not violations: + print("pilot-report-validate: OK (no violations)") + return 0 + + print(f"pilot-report-validate: {len(violations)} violation(s) found\n") + for v in violations: + print(v) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/pilot-report-validator/tests/test_pilot_report_validator.py b/tools/pilot-report-validator/tests/test_pilot_report_validator.py new file mode 100644 index 00000000..828e7518 --- /dev/null +++ b/tools/pilot-report-validator/tests/test_pilot_report_validator.py @@ -0,0 +1,400 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the pilot-report validator.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from pilot_report_validator import ( + ALLOWED_PROFILES, + REQUIRED_FRONTMATTER_KEYS, + REQUIRED_SECTIONS, + collect_report_files, + extract_section_headings, + main, + parse_frontmatter, + run_validation, + validate_body, + validate_frontmatter, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_VALID_REPORT = textwrap.dedent("""\ + + + --- + skill: pairing-self-review + date: 2024-06-01 + target_repo: example/myproject + profile: asf + reporter: jdoe + --- + + # Pilot report: pairing-self-review on example/myproject + + ## Skill or family + + pairing-self-review + + ## Target repo and profile + + example/myproject — ASF profile. + + ## Blocked preflights + + None observed. + + ## False positives + + None observed. + + ## Confirmation points + + All confirmation points felt appropriate. + + ## Privacy and adapter notes + + None observed. + + ## Proposed spec changes + + No changes proposed at this time. + """) + + +def _make_report( + *, + skill: str = "pairing-self-review", + date: str = "2024-06-01", + target_repo: str = "example/myproject", + profile: str = "asf", + extra_keys: str = "", + spdx: bool = True, +) -> str: + """Build a minimal valid pilot report.""" + spdx_header = ( + "\n\n" + if spdx + else "" + ) + fm = f"skill: {skill}\ndate: {date}\ntarget_repo: {target_repo}\nprofile: {profile}\n" + ( + f"{extra_keys}\n" if extra_keys else "" + ) + body_sections = "\n\n".join(f"## {s}\n\nContent." for s in REQUIRED_SECTIONS) + return f"{spdx_header}---\n{fm}---\n\n# Report\n\n{body_sections}\n" + + +# --------------------------------------------------------------------------- +# parse_frontmatter +# --------------------------------------------------------------------------- + + +class TestParseFrontmatter: + def test_valid_report(self) -> None: + fm = parse_frontmatter(_VALID_REPORT) + assert fm is not None + assert fm["skill"] == "pairing-self-review" + assert fm["profile"] == "asf" + + def test_no_frontmatter_returns_none(self) -> None: + assert parse_frontmatter("# Just a heading\n\nNo frontmatter.") is None + + def test_html_comment_prefix_allowed(self) -> None: + text = "\n---\nskill: foo\ndate: 2024-01-01\ntarget_repo: a/b\nprofile: asf\n---\n" + fm = parse_frontmatter(text) + assert fm is not None + assert fm["skill"] == "foo" + + def test_non_comment_prefix_returns_none(self) -> None: + text = "Some prose\n---\nskill: foo\n---\n" + assert parse_frontmatter(text) is None + + def test_extra_optional_key_included(self) -> None: + fm = parse_frontmatter(_VALID_REPORT) + assert fm is not None + assert fm.get("reporter") == "jdoe" + + +# --------------------------------------------------------------------------- +# extract_section_headings +# --------------------------------------------------------------------------- + + +class TestExtractSectionHeadings: + def test_extracts_required_sections(self) -> None: + headings = extract_section_headings(_VALID_REPORT) + for section in REQUIRED_SECTIONS: + assert section in headings, f"expected section '{section}' in headings" + + def test_ignores_h1(self) -> None: + headings = extract_section_headings(_VALID_REPORT) + assert "Pilot report: pairing-self-review on example/myproject" not in headings + + def test_no_frontmatter_still_works(self) -> None: + text = "# Title\n\n## Skill or family\n\ncontent\n" + headings = extract_section_headings(text) + assert "Skill or family" in headings + + +# --------------------------------------------------------------------------- +# validate_frontmatter +# --------------------------------------------------------------------------- + + +class TestValidateFrontmatter: + def test_valid_report_no_violations(self, tmp_path: Path) -> None: + p = tmp_path / "report.md" + p.write_text(_VALID_REPORT) + assert validate_frontmatter(p, _VALID_REPORT) == [] + + def test_no_frontmatter_skipped(self, tmp_path: Path) -> None: + text = "# No frontmatter\n\ncontent\n" + p = tmp_path / "readme.md" + assert validate_frontmatter(p, text) == [] + + def test_missing_required_keys(self, tmp_path: Path) -> None: + text = "---\nskill: foo\n---\n# t\n" + p = tmp_path / "report.md" + violations = validate_frontmatter(p, text) + messages = [v.message for v in violations] + assert any("date" in m for m in messages) + assert any("target_repo" in m for m in messages) + assert any("profile" in m for m in messages) + + def test_all_required_keys_present(self, tmp_path: Path) -> None: + for key in sorted(REQUIRED_FRONTMATTER_KEYS): + assert key in {"skill", "date", "target_repo", "profile"} + + @pytest.mark.parametrize("profile", sorted(ALLOWED_PROFILES)) + def test_all_valid_profiles_pass(self, tmp_path: Path, profile: str) -> None: + text = _make_report(profile=profile) + p = tmp_path / "report.md" + violations = [v for v in validate_frontmatter(p, text) if "profile" in v.message] + assert violations == [] + + def test_invalid_profile(self, tmp_path: Path) -> None: + text = _make_report(profile="unknown") + p = tmp_path / "report.md" + violations = validate_frontmatter(p, text) + assert any("invalid profile" in v.message for v in violations) + + def test_violation_line_number_is_1(self, tmp_path: Path) -> None: + text = _make_report(profile="bad") + p = tmp_path / "report.md" + violations = [v for v in validate_frontmatter(p, text) if "profile" in v.message] + assert violations[0].line == 1 + + def test_missing_key_violation_line_number_is_1(self, tmp_path: Path) -> None: + text = "---\nskill: foo\n---\n# t\n" + p = tmp_path / "report.md" + violations = validate_frontmatter(p, text) + assert all(v.line == 1 for v in violations) + + +# --------------------------------------------------------------------------- +# validate_body +# --------------------------------------------------------------------------- + + +class TestValidateBody: + def test_valid_report_no_violations(self, tmp_path: Path) -> None: + p = tmp_path / "report.md" + p.write_text(_VALID_REPORT) + assert validate_body(p, _VALID_REPORT) == [] + + def test_no_frontmatter_skipped(self, tmp_path: Path) -> None: + text = "# No frontmatter\n\n## Skill or family\n\ncontent\n" + p = tmp_path / "readme.md" + assert validate_body(p, text) == [] + + @pytest.mark.parametrize("section", REQUIRED_SECTIONS) + def test_missing_section_flagged(self, tmp_path: Path, section: str) -> None: + text = _make_report() + text_no_section = text.replace(f"## {section}\n", "## REPLACED\n") + p = tmp_path / "report.md" + violations = validate_body(p, text_no_section) + assert any(section in v.message for v in violations) + + def test_all_sections_present_no_violations(self, tmp_path: Path) -> None: + text = _make_report() + p = tmp_path / "report.md" + assert validate_body(p, text) == [] + + +# --------------------------------------------------------------------------- +# run_validation (integration) +# --------------------------------------------------------------------------- + + +class TestRunValidation: + def test_valid_directory_no_violations(self, tmp_path: Path) -> None: + (tmp_path / "report_a.md").write_text(_VALID_REPORT) + (tmp_path / "report_b.md").write_text(_make_report(profile="non-asf")) + assert run_validation(tmp_path) == [] + + def test_readme_without_frontmatter_skipped(self, tmp_path: Path) -> None: + (tmp_path / "README.md").write_text("# README\n\nNo frontmatter.\n") + assert run_validation(tmp_path) == [] + + def test_invalid_report_produces_violations(self, tmp_path: Path) -> None: + text = "---\nskill: only-skill\n---\n# broken\n" + (tmp_path / "broken.md").write_text(text) + violations = run_validation(tmp_path) + assert len(violations) > 0 + + def test_single_file_target(self, tmp_path: Path) -> None: + p = tmp_path / "report.md" + p.write_text(_VALID_REPORT) + assert run_validation(p) == [] + + def test_nested_directory_scanned(self, tmp_path: Path) -> None: + subdir = tmp_path / "reports" + subdir.mkdir() + (subdir / "report.md").write_text(_VALID_REPORT) + assert run_validation(tmp_path) == [] + + +# --------------------------------------------------------------------------- +# collect_report_files +# --------------------------------------------------------------------------- + + +class TestCollectReportFiles: + def test_file_target_returns_self(self, tmp_path: Path) -> None: + p = tmp_path / "report.md" + p.write_text("# Test") + assert collect_report_files(p) == [p] + + def test_directory_scans_recursively(self, tmp_path: Path) -> None: + sub = tmp_path / "sub" + sub.mkdir() + (tmp_path / "a.md").write_text("a") + (sub / "b.md").write_text("b") + files = collect_report_files(tmp_path) + assert len(files) == 2 + + def test_non_md_files_excluded(self, tmp_path: Path) -> None: + (tmp_path / "report.md").write_text("x") + (tmp_path / "notes.txt").write_text("y") + files = collect_report_files(tmp_path) + assert all(f.suffix == ".md" for f in files) + + +# --------------------------------------------------------------------------- +# CLI (main) +# --------------------------------------------------------------------------- + + +class TestMain: + def test_nonexistent_path_returns_1(self, capsys: pytest.CaptureFixture[str]) -> None: + rc = main(["/nonexistent/path"]) + assert rc == 1 + captured = capsys.readouterr() + assert "not found" in captured.err + + def test_valid_report_returns_0(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + (tmp_path / "report.md").write_text(_VALID_REPORT) + rc = main([str(tmp_path)]) + assert rc == 0 + captured = capsys.readouterr() + assert "OK" in captured.out + + def test_invalid_report_returns_1(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + (tmp_path / "bad.md").write_text("---\nskill: only\n---\n# bad\n") + rc = main([str(tmp_path)]) + assert rc == 1 + captured = capsys.readouterr() + assert "violation" in captured.out + + def test_readme_only_returns_0(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + (tmp_path / "README.md").write_text("# README\n\nNo frontmatter, no validation.\n") + rc = main([str(tmp_path)]) + assert rc == 0 + + def test_output_lists_violations(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + text = _make_report(profile="bad-profile") + (tmp_path / "report.md").write_text(text) + rc = main([str(tmp_path)]) + assert rc == 1 + captured = capsys.readouterr() + assert "invalid profile" in captured.out + + +# --------------------------------------------------------------------------- +# Template file smoke test +# --------------------------------------------------------------------------- + + +class TestTemplate: + """The shipped template must be a structurally valid report. + + Regression guard for the frontmatter-placement bug: the template's + frontmatter must sit at the very top of the file so that reports + copied from it are actually detected and validated. If the frontmatter + is moved lower (e.g. below the table of contents), the validator + silently skips it — and every real report along with it. + """ + + @staticmethod + def _find_template() -> Path: + start = Path(__file__).resolve() + for candidate in (start, *start.parents): + template = candidate / "docs" / "pilot-report-template.md" + if template.is_file(): + return template + pytest.skip("docs/pilot-report-template.md not found — skipping template smoke test") + + def test_template_frontmatter_is_detected_at_top(self) -> None: + text = self._find_template().read_text(encoding="utf-8") + fm = parse_frontmatter(text) + assert fm is not None, ( + "template frontmatter is not detected — it must be at the very top of the file, " + "above the table of contents, or reports copied from it will be silently skipped" + ) + assert REQUIRED_FRONTMATTER_KEYS <= set(fm), ( + f"template frontmatter is missing required keys: {sorted(REQUIRED_FRONTMATTER_KEYS - set(fm))}" + ) + + def test_template_validates_clean(self) -> None: + template = self._find_template() + assert run_validation(template) == [], "the shipped template should validate with no violations" + + def test_filled_report_from_template_validates(self) -> None: + """A user copying the template and filling in real values gets a clean run.""" + text = self._find_template().read_text(encoding="utf-8") + filled = ( + text.replace("", "pairing-self-review") + .replace("YYYY-MM-DD", "2026-06-29") + .replace("/", "example/myproject") + .replace("", "jdoe") + ) + fm = parse_frontmatter(filled) + assert fm is not None + assert fm["skill"] == "pairing-self-review" + assert fm["profile"] == "asf" + headings = extract_section_headings(filled) + for section in REQUIRED_SECTIONS: + assert section in headings, f"filled report missing required section '{section}'" diff --git a/tools/pilot-report-validator/uv.lock b/tools/pilot-report-validator/uv.lock new file mode 100644 index 00000000..37458af5 --- /dev/null +++ b/tools/pilot-report-validator/uv.lock @@ -0,0 +1,112 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P7D" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pilot-report-validator" +version = "0.1.0" +source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0" }, + { name = "ruff", specifier = ">=0.4" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/98/1295ad5a5aa9bc85bdcdfa5d82fe7b49c61af5657df4f227637ff9de0da6/ruff-0.15.18.tar.gz", hash = "sha256:2698a964c70e8bf402dcb99c8810472d270d141e7aa8c4e13599fd52033a2f33", size = 4761437, upload-time = "2026-06-18T18:25:39.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/d0/686e984941269621e2be72612d5c1e461f8f7b38415a2a7d7a81c8ae6715/ruff-0.15.18-py3-none-linux_armv6l.whl", hash = "sha256:8b6850172348c8381b8b3084c5915a4393c2373b9b54cd5b5e1ea15812bc10df", size = 10887308, upload-time = "2026-06-18T18:25:03.062Z" }, + { url = "https://files.pythonhosted.org/packages/ed/21/bc4123e3f5515ee99f8ce1eb93a14a0628fe4d1678663cd08f933ac16931/ruff-0.15.18-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3fccc153a85417dcd976883160cacce486997b0a0058dd18f54b8aaaac7d1ce2", size = 11281305, upload-time = "2026-06-18T18:25:30.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/93/4769464c25cf7ab2acb3c7dda9cad3d867eb41c59565b3e2a9d17249c90c/ruff-0.15.18-py3-none-macosx_11_0_arm64.whl", hash = "sha256:08d4c86a68f2c3ec2c9d56380a71fb4a4f65373055cbb8caabd645e9102f38d4", size = 10641215, upload-time = "2026-06-18T18:25:15.802Z" }, + { url = "https://files.pythonhosted.org/packages/6c/42/56926d17120db2c208d76bf60a1a019644dd9e91dc27f0f95c9caddb1366/ruff-0.15.18-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37e5108745c2c0705da916d7d4de533ddf547051ef45f62888c31bae73f66318", size = 10957224, upload-time = "2026-06-18T18:25:36.955Z" }, + { url = "https://files.pythonhosted.org/packages/22/4f/d43fab8d8189afde803103022d000a8ef9f230616d436d52a8b2b8d63b50/ruff-0.15.18-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56949a6ce8b3abde54c0bcb22cebfe57e8771cadc84b407ae8b8eaf67ebdcd43", size = 10699024, upload-time = "2026-06-18T18:25:05.707Z" }, + { url = "https://files.pythonhosted.org/packages/63/42/1e3e4c68bd408b9768cf3e439acbe2c78245225faef253f7028a0cdb63e0/ruff-0.15.18-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01a754cd6a1b630d3f97e33eb452cf7a98040482318e870f8bc52a5a30e62657", size = 11491458, upload-time = "2026-06-18T18:25:20.275Z" }, + { url = "https://files.pythonhosted.org/packages/20/77/47a3484bea8521e14a203d98c389c5c97846675e4f02734672da4a69b52a/ruff-0.15.18-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6ba7a07e03a44dbf10bb086ee06705b173625014ec99f73a7e6836a5e5590a0c", size = 12383752, upload-time = "2026-06-18T18:25:22.535Z" }, + { url = "https://files.pythonhosted.org/packages/0a/ca/054159590787023d83b658a1a1819c4c8910114e7015069340b71c0961cb/ruff-0.15.18-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a2c40a41a4cadbcf5897b548ab29dfe248b20c540961c0247d98a3973c70403", size = 11577923, upload-time = "2026-06-18T18:25:10.702Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ff/d353d6b7bbd73cc0ec37f4463d7540e45e894338abdd9964eee0de332708/ruff-0.15.18-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f0480ce690cbb6c4db6e5d08f19fce98e10ba131a8b60c1bcdac42771e3ae2d", size = 11583925, upload-time = "2026-06-18T18:25:32.391Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4a/891f89b9c296ed3e5f3ece1a5629badc989d9a8fdaa30431aaf4774bc1c2/ruff-0.15.18-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2330215f1f393fa8733f55edce04fcf94c36a2c460fcde31f78cc84e4951e9b1", size = 11582834, upload-time = "2026-06-18T18:25:27.309Z" }, + { url = "https://files.pythonhosted.org/packages/32/a3/ed9e370154bf85de360b93c03026157f02d4943b2d01ff4945f4429f8e8a/ruff-0.15.18-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a6aa6a3d979e48ae617578183674bf264fbe7d0114a796a26bd678d67963c7ff", size = 10927328, upload-time = "2026-06-18T18:25:34.676Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d1/5cf5909329fedb5d39d555ee818ba5cf4638e1a301b89785d34f2905bfcb/ruff-0.15.18-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a81beadbbff2c9c245561ae3f77b16709d87f35eec650d0501679239d3449b22", size = 10693187, upload-time = "2026-06-18T18:25:08.245Z" }, + { url = "https://files.pythonhosted.org/packages/fd/44/ff6c635cf2c4f4e7b618b6640da057376baa36014695487d88aed4794268/ruff-0.15.18-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2186d9e940ae332ab293623a75b5f4fe49565f449954d50a72a046683aa6b809", size = 11208721, upload-time = "2026-06-18T18:25:41.327Z" }, + { url = "https://files.pythonhosted.org/packages/88/d9/5baa2a30861adfb7022cf33c1e35b2fc18085b08c16f83eff4c7b99a5f48/ruff-0.15.18-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5c2abf140438032bc77b2284a6c9944ecd8a19e5f1c7b52b1b8e4a0a80d19a7a", size = 11678599, upload-time = "2026-06-18T18:25:13.607Z" }, + { url = "https://files.pythonhosted.org/packages/c3/1a/0725a7cfdc32ff769efb96ee782bec882e16448c5d9e3be947ec4c04ce27/ruff-0.15.18-py3-none-win32.whl", hash = "sha256:02299e6e9fa5b297a3f6d5d10d7bcd655c925b028bb8b9d4588214549c6b9ec4", size = 10901903, upload-time = "2026-06-18T18:25:24.755Z" }, + { url = "https://files.pythonhosted.org/packages/f3/51/805d9f6fb7970505c3504794a5ec350f605361b807fef4dcf214ebd35e72/ruff-0.15.18-py3-none-win_amd64.whl", hash = "sha256:dac80dc8d26b2257dbefabed62f5d255c3937b4ccb122da1fc634794fa3578b3", size = 12041189, upload-time = "2026-06-18T18:25:17.915Z" }, + { url = "https://files.pythonhosted.org/packages/29/4c/67bb45e41609eb4726f1bfeb59e083cf91d14c696d4bd14c234a980be93d/ruff-0.15.18-py3-none-win_arm64.whl", hash = "sha256:b2c9257fcbd4a3e5b977a1904e6facca016bafe2edc17df24db67cfaee03b4e4", size = 11329958, upload-time = "2026-06-18T18:25:43.686Z" }, +] diff --git a/uv.lock b/uv.lock index 32260832..d18e00a8 100644 --- a/uv.lock +++ b/uv.lock @@ -24,6 +24,7 @@ members = [ "magpie-vcs", "oauth-draft", "permission-audit", + "pilot-report-validator", "pr-management-stats", "preflight-audit", "redactor", @@ -620,6 +621,25 @@ name = "permission-audit" version = "0.1.0" source = { editable = "tools/permission-audit" } +[[package]] +name = "pilot-report-validator" +version = "0.1.0" +source = { editable = "tools/pilot-report-validator" } + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0" }, + { name = "ruff", specifier = ">=0.4" }, +] + [[package]] name = "pluggy" version = "1.6.0" From 2e33fac23384e41caf410946323a989837970f39 Mon Sep 17 00:00:00 2001 From: Justin McLean Date: Mon, 29 Jun 2026 20:07:31 +1000 Subject: [PATCH 2/3] checks that required keys are present --- tools/pilot-report-validator/README.md | 10 ++-- .../src/pilot_report_validator/__init__.py | 32 ++++++++++-- .../tests/test_pilot_report_validator.py | 51 +++++++++++++++---- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/tools/pilot-report-validator/README.md b/tools/pilot-report-validator/README.md index be27ffbf..a8482a1f 100644 --- a/tools/pilot-report-validator/README.md +++ b/tools/pilot-report-validator/README.md @@ -38,7 +38,10 @@ For every `.md` file that carries a YAML frontmatter block: 1. **Required frontmatter keys** — `skill`, `date`, `target_repo`, `profile`. 2. **Valid `profile` value** — `asf` | `non-asf` | `custom`. -3. **Required body sections** — `## Skill or family`, +3. **No unfilled placeholders** — frontmatter values must not contain + un-substituted `<...>` tokens, and `date` must be a real ISO 8601 + date (`YYYY-MM-DD`). +4. **Required body sections** — `## Skill or family`, `## Target repo and profile`, `## Blocked preflights`, `## False positives`, `## Confirmation points`, `## Privacy and adapter notes`, `## Proposed spec changes`. @@ -46,8 +49,9 @@ For every `.md` file that carries a YAML frontmatter block: The frontmatter block must be at the very top of the file — YAML frontmatter placed lower in the document is not detected. Files without a top-of-file frontmatter block (e.g. README files) are silently -skipped. `docs/pilot-report-template.md` carries a top-of-file -frontmatter skeleton and so validates as a clean report. +skipped. `docs/pilot-report-template.md` ships with placeholder +frontmatter values, so running the validator on the unedited template +reports those placeholders until you fill them in. ## Usage diff --git a/tools/pilot-report-validator/src/pilot_report_validator/__init__.py b/tools/pilot-report-validator/src/pilot_report_validator/__init__.py index 0165b33a..107fad63 100644 --- a/tools/pilot-report-validator/src/pilot_report_validator/__init__.py +++ b/tools/pilot-report-validator/src/pilot_report_validator/__init__.py @@ -21,15 +21,19 @@ 1. Required frontmatter keys — skill, date, target_repo, profile. 2. Valid ``profile`` value — asf | non-asf | custom. -3. Required body sections — Skill or family, Target repo and profile, +3. No unfilled placeholders — frontmatter values must not contain + un-substituted ``<...>`` tokens, and ``date`` must be a real ISO 8601 + date (YYYY-MM-DD). +4. Required body sections — Skill or family, Target repo and profile, Blocked preflights, False positives, Confirmation points, Privacy and adapter notes, Proposed spec changes. The frontmatter block must be at the top of the file; YAML frontmatter placed lower in the document is not detected. Files without a top-of-file frontmatter block (e.g. README.md) are skipped silently. -``docs/pilot-report-template.md`` carries a top-of-file frontmatter -skeleton and so validates as a clean report. +``docs/pilot-report-template.md`` ships with placeholder frontmatter +values, so running the validator on the unedited template reports those +placeholders until they are filled in. Run from repo root:: @@ -65,6 +69,11 @@ _HTML_COMMENT_RE = re.compile(r"") _YAML_BLOCK_SCALAR_HEADERS: frozenset[str] = frozenset({"|", ">", "|-", "|+", ">-", ">+"}) +# An un-substituted template placeholder, e.g. "" or "/". +_ANGLE_PLACEHOLDER_RE = re.compile(r"<[^<>\n]+>") +# ISO 8601 calendar date (YYYY-MM-DD); also rejects the "YYYY-MM-DD" placeholder. +_ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + # --------------------------------------------------------------------------- # Data structures # --------------------------------------------------------------------------- @@ -189,6 +198,23 @@ def validate_frontmatter(path: Path, text: str) -> list[Violation]: ) ) + # Un-substituted template placeholders left in any frontmatter value. + for key in sorted(fm): + if _ANGLE_PLACEHOLDER_RE.search(fm[key]): + violations.append( + Violation( + path, + 1, + f"frontmatter key '{key}' still contains an un-substituted placeholder: {fm[key]!r}", + ) + ) + + # Date value must be a real ISO 8601 date (also catches the 'YYYY-MM-DD' placeholder). + if fm.get("date") and not _ISO_DATE_RE.match(fm["date"]): + violations.append( + Violation(path, 1, f"invalid date '{fm['date']}' — must be ISO 8601 format YYYY-MM-DD") + ) + return violations diff --git a/tools/pilot-report-validator/tests/test_pilot_report_validator.py b/tools/pilot-report-validator/tests/test_pilot_report_validator.py index 828e7518..2e76d2e4 100644 --- a/tools/pilot-report-validator/tests/test_pilot_report_validator.py +++ b/tools/pilot-report-validator/tests/test_pilot_report_validator.py @@ -213,6 +213,30 @@ def test_missing_key_violation_line_number_is_1(self, tmp_path: Path) -> None: violations = validate_frontmatter(p, text) assert all(v.line == 1 for v in violations) + def test_angle_bracket_placeholder_flagged(self, tmp_path: Path) -> None: + text = _make_report(skill="") + p = tmp_path / "report.md" + violations = [v.message for v in validate_frontmatter(p, text)] + assert any("placeholder" in m and "skill" in m for m in violations), violations + + @pytest.mark.parametrize("bad_date", ["YYYY-MM-DD", "2026/06/29", "29-06-2026", "not-a-date"]) + def test_non_iso_date_flagged(self, tmp_path: Path, bad_date: str) -> None: + text = _make_report(date=bad_date) + p = tmp_path / "report.md" + violations = [v.message for v in validate_frontmatter(p, text)] + assert any("must be ISO 8601" in m for m in violations), violations + + def test_iso_date_passes(self, tmp_path: Path) -> None: + text = _make_report(date="2026-06-29") + p = tmp_path / "report.md" + date_violations = [v.message for v in validate_frontmatter(p, text) if "date" in v.message] + assert date_violations == [] + + def test_filled_values_no_violations(self, tmp_path: Path) -> None: + text = _make_report(skill="pairing-self-review", date="2026-06-29", target_repo="example/myproject") + p = tmp_path / "report.md" + assert validate_frontmatter(p, text) == [] + # --------------------------------------------------------------------------- # validate_body @@ -378,11 +402,21 @@ def test_template_frontmatter_is_detected_at_top(self) -> None: f"template frontmatter is missing required keys: {sorted(REQUIRED_FRONTMATTER_KEYS - set(fm))}" ) - def test_template_validates_clean(self) -> None: + def test_template_flags_unfilled_placeholders(self) -> None: + """The unfilled template reports its placeholder values, and nothing else. + + Every violation must be a placeholder / date-format finding — not a + missing key, bad profile, or missing section — which proves the + frontmatter and body are structurally complete and only the values + remain to be filled in. + """ template = self._find_template() - assert run_validation(template) == [], "the shipped template should validate with no violations" + messages = [v.message for v in run_validation(template)] + assert messages, "unfilled template should report its placeholder values as violations" + for m in messages: + assert "placeholder" in m or "must be ISO 8601" in m, f"unexpected violation: {m}" - def test_filled_report_from_template_validates(self) -> None: + def test_filled_report_from_template_validates(self, tmp_path: Path) -> None: """A user copying the template and filling in real values gets a clean run.""" text = self._find_template().read_text(encoding="utf-8") filled = ( @@ -391,10 +425,7 @@ def test_filled_report_from_template_validates(self) -> None: .replace("/", "example/myproject") .replace("", "jdoe") ) - fm = parse_frontmatter(filled) - assert fm is not None - assert fm["skill"] == "pairing-self-review" - assert fm["profile"] == "asf" - headings = extract_section_headings(filled) - for section in REQUIRED_SECTIONS: - assert section in headings, f"filled report missing required section '{section}'" + report = tmp_path / "filled-report.md" + report.write_text(filled, encoding="utf-8") + violations = [str(v) for v in run_validation(report)] + assert violations == [], f"filled report should validate clean, got: {violations}" From 6144a46a4e8144e212b3bbf390fb52b851007f22 Mon Sep 17 00:00:00 2001 From: Justin McLean Date: Mon, 29 Jun 2026 20:55:49 +1000 Subject: [PATCH 3/3] test: make _find_template return paths explicit (CodeQL no implicit None) --- .../tests/test_pilot_report_validator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/pilot-report-validator/tests/test_pilot_report_validator.py b/tools/pilot-report-validator/tests/test_pilot_report_validator.py index 2e76d2e4..b057717a 100644 --- a/tools/pilot-report-validator/tests/test_pilot_report_validator.py +++ b/tools/pilot-report-validator/tests/test_pilot_report_validator.py @@ -389,7 +389,10 @@ def _find_template() -> Path: template = candidate / "docs" / "pilot-report-template.md" if template.is_file(): return template + # pytest.skip() raises Skipped, so control never falls through; the explicit + # raise keeps every path either returning a Path or raising (no implicit None). pytest.skip("docs/pilot-report-template.md not found — skipping template smoke test") + raise AssertionError("unreachable") # pragma: no cover def test_template_frontmatter_is_detected_at_top(self) -> None: text = self._find_template().read_text(encoding="utf-8")