diff --git a/.doc-path-baseline.yml b/.doc-path-baseline.yml index 73b2b30..63d2f3f 100644 --- a/.doc-path-baseline.yml +++ b/.doc-path-baseline.yml @@ -35,10 +35,3 @@ entries: expiry: 2027-09-04 # --- unimplemented -------------------------------------------------------- - - path: docs/audit/baseline-review.md - reason: unimplemented — .security-baselines/README.md promises a quarterly baseline review document that was never created - expiry: 2026-12-01 - - - path: docs/decisions/README.md - reason: unimplemented — docs/COMPLIANCE_MAPPING.md cites an ADR index as EU AI Act Art. 11 evidence; this repo has 45 ADRs and no index (the generated service does get one, which is what MIGRATION.md describes) - expiry: 2026-12-01 diff --git a/.github/workflows/validate-templates.yml b/.github/workflows/validate-templates.yml index db8ac8b..61962fd 100644 --- a/.github/workflows/validate-templates.yml +++ b/.github/workflows/validate-templates.yml @@ -753,6 +753,10 @@ jobs: python-version: "3.11" - name: Check documentation path references resolve run: python3 scripts/check_doc_path_refs.py + - name: Check the ADR index is current + # docs/COMPLIANCE_MAPPING.md cites this index as EU AI Act Art. 11 + # evidence. A stale index is a falsifiable compliance claim. + run: python3 scripts/generate_adr_index.py --check vendored-runtime-drift: name: Vendored Runtime Drift Gate diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e81269d..d16074b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -285,6 +285,17 @@ repos: language: system pass_filenames: false files: \.(md|txt)$|^scripts/check_doc_path_refs\.py$|^\.doc-path-baseline\.yml$ + - id: adr-index + name: ADR index is current + description: | + docs/decisions/README.md is generated from the ADR files and + cited by docs/COMPLIANCE_MAPPING.md as EU AI Act Art. 11 + evidence. Adding an ADR without regenerating leaves a + compliance claim an auditor can falsify. + entry: python3 scripts/generate_adr_index.py --check + language: system + pass_filenames: false + files: ^(docs/decisions/|scripts/generate_adr_index\.py) - id: service-adr-references name: generated service ADR references are resolvable description: | diff --git a/.security-baselines/tfsec.yml b/.security-baselines/tfsec.yml index 343e43e..fc7a89e 100644 --- a/.security-baselines/tfsec.yml +++ b/.security-baselines/tfsec.yml @@ -16,7 +16,8 @@ exclude: # 1.25. GKE clusters on REGULAR channel run 1.27+; enabling PSP would # cause a Terraform error. Pod Security Standards (PSS) are enforced via # namespace labels in every overlay (D-29, templates/k8s/overlays/*/ - # namespace.yaml). Review-by: 2027-01 (when tfsec migrates to trivy/PSS). + # namespace.yaml). Re-evaluate when tfsec migrates to trivy/PSS. + # expiry: 2027-01-01 reason: ADR-024 §Review — PSS via namespace labels (D-29) - google-gke-enforce-pod-security-policy # master_authorized_networks_config IS present in compute.tf (lines 38–49) @@ -25,7 +26,8 @@ exclude: # block as absent even when the HCL is correct. Environment overlays # MUST supply non-empty `master_authorized_networks` for staging/prod # (enforced by the variable validation rule in variables.tf). - # Review-by: 2027-01 (re-evaluate after tfsec→trivy migration). + # Re-evaluate after the tfsec->trivy migration. + # expiry: 2027-01-01 reason: ADR-024 §Review — dynamic block not evaluated by tfsec v1.28 - google-gke-enable-master-networks # tfsec evaluates `node_config.metadata["disable-legacy-endpoints"]` only on @@ -45,8 +47,9 @@ exclude: # — additionally blocks the v0.1 endpoint that does not require # the Metadata-Flavor header. Redundant but kept (belt + suspenders). # - # Review-by: 2027-01 (re-evaluate after tfsec→trivy migration; trivy may - # correlate node_pool resources to the cluster correctly). + # Re-evaluate after the tfsec->trivy migration; trivy may correlate + # node_pool resources to the cluster correctly. + # expiry: 2027-01-01 reason: ADR-024 §Review — metadata hardening lives on the node pools - google-gke-metadata-endpoints-disabled minimum_severity: HIGH diff --git a/CHANGELOG.md b/CHANGELOG.md index 012b0a5..ab9f80b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,56 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Sem - The secret patterns are themselves pinned by a test, because a scanner that silently stops matching passes everything. - Baseline down from 5 entries to 4. +### Fixed — the baseline expiry gate could not see the entries it exists to watch + +- `scripts/check_baselines_expiry.py` reported `OK — no expired or + unannotated entries` while **three HIGH-severity GKE checks sat + suppressed** in `.security-baselines/tfsec.yml`. It was not lying about + the entries it saw; it saw none. +- Its `yaml_entry` pattern required a suppression id to start with an + uppercase letter. That matches checkov (`CKV_AWS_18`) and misses every + tfsec id (`google-gke-enable-master-networks`). Zero matches, zero + expired, pass. +- Widening it to lowercase alone would have been wrong: `framework: + [terraform, kubernetes, dockerfile]` in `checkov.yml` is a sequence too. + The scanner is now **block-aware** and treats only items under + `exclude:` / `skip-check:` as entries — which is what the code's own + comment had claimed since it was written. +- The three entries also carried `Review-by: 2027-01`, a format the gate + does not parse. Normalised to `# expiry: 2027-01-01` with the original + justification prose kept intact. +- Verified: all three are now seen and in-date, `--as-of 2027-06-01` fails + all three as expired, and `framework:` still produces no false positive. + +### Added — `docs/audit/baseline-review.md`, the review record that was promised + +- `.security-baselines/README.md` step 4 has instructed reviewers to + update this file since the baselines were introduced. It did not exist, + so three HIGH suppressions had a justification in a YAML comment and no + review record anywhere. +- The first dated review records all three, each with its compensating + control and expiry. All three are **tool limitations, not accepted + risks**: PSP was removed from Kubernetes 1.25 and PSS covers it via + namespace labels; `master_authorized_networks_config` exists as a + `dynamic` block tfsec cannot evaluate; the metadata attribute lives on + the node pools rather than the cluster tfsec inspects. + +### Added — a generated ADR index, and the Art. 11 claim it backs + +- `docs/COMPLIANCE_MAPPING.md` cited an ADR index as EU AI Act Art. 11 + evidence. The index did not exist, and the same row claimed "ADRs (37)" + against 45 on disk. A compliance mapping that points at a missing + artefact and miscounts the one it has is worse than none: it is an + assertion an auditor falsifies in one command. +- `scripts/generate_adr_index.py` generates `docs/decisions/README.md` + from the files themselves and `--check` fails CI when it goes stale, so + the next ADR cannot reintroduce the drift. Wired as a CI step and a + pre-commit hook. It parses both heading conventions in use + (`# ADR-001: Title` and `# ADR-045 — Title`). +- The Art. 11 row now links the index instead of restating a count, which + removes the drift surface rather than correcting one instance of it. +- Baseline down to 2 entries once the three PRs in this series land, both + `runtime-artifact` — no `unimplemented` entries remain. ### Fixed — the Copier migration relocated the template tree and the prose never followed diff --git a/Makefile b/Makefile index 609c38d..dd7d1e3 100644 --- a/Makefile +++ b/Makefile @@ -202,6 +202,12 @@ verify: ## Run every fast CI gate locally (the pre-push contract). Slow E2E live printf ' %-34s FAIL\n' "adapter-sync"; \ failed="$$failed sync_agentic_adapters"; \ fi; \ + if python3 scripts/generate_adr_index.py --check >/tmp/adr_index.out 2>&1; then \ + printf ' %-34s PASS\n' "adr-index"; \ + else \ + printf ' %-34s FAIL\n' "adr-index"; \ + failed="$$failed generate_adr_index"; \ + fi; \ if [ -n "$$failed" ]; then \ echo "$(RED)✗ failing gates:$$failed$(NC)"; \ for g in $$failed; do echo "--- $$g ---"; tail -20 /tmp/$$g.out; done; \ diff --git a/docs/COMPLIANCE_MAPPING.md b/docs/COMPLIANCE_MAPPING.md index 14b2f36..9924cd0 100644 --- a/docs/COMPLIANCE_MAPPING.md +++ b/docs/COMPLIANCE_MAPPING.md @@ -74,7 +74,7 @@ good engineering independent of the regulatory calendar. |---|---|---|---| | **Art. 9** — Risk management system | Continuous risk identification/mitigation across the lifecycle | Quality gates + fairness gate + drift monitoring + dynamic risk escalation (ADR-010) | A closed loop: measure → gate → escalate, not a one-time check | | **Art. 10** — Data governance | Training/validation/test data subject to quality criteria; bias examination | 6-phase EDA with leakage gate; Pandera schema validation; fairness DIR gate | `eda_pipeline.py` phase 4 (leakage) blocks training on a leaking feature | -| **Art. 11** — Technical documentation | Documentation sufficient to assess compliance | ADRs (37) + model card + `RUNBOOK.md` + `docs/decisions/README.md` index | Documentation is versioned alongside code, not a separate artifact that drifts (rule 16 / ADR-031 doc-coherence gate) | +| **Art. 11** — Technical documentation | Documentation sufficient to assess compliance | The [ADR index](decisions/README.md) (generated from the files, verified in CI) + model card + `RUNBOOK.md` | Documentation is versioned alongside code, not a separate artifact that drifts (rule 16 / ADR-031 doc-coherence gate). The count is deliberately not restated here: it was wrong (37 against 45 on disk) for as long as the index it cited did not exist. | | **Art. 12** — Record-keeping (logging) | Automatic logging of events over the system's lifetime | `ops/audit.jsonl` (append-only), `prediction_logger.py`, MLflow tracking | Every prediction + every agent action is logged with a trace id | | **Art. 13** — Transparency | Sufficient information for deployers to interpret output | SHAP explainability (`?explain=true`), model card "intended use" section | Per-prediction feature attribution, in the original feature space | | **Art. 14** — Human oversight | Natural persons can oversee, and if needed intervene/stop | AUTO/CONSULT/STOP protocol; STOP-class rollback (human approves every step) | The agent cannot promote, deploy to prod, or rollback without a human in the loop | diff --git a/docs/audit/baseline-review.md b/docs/audit/baseline-review.md new file mode 100644 index 0000000..5345351 --- /dev/null +++ b/docs/audit/baseline-review.md @@ -0,0 +1,90 @@ +# Security Baseline Review + +**Cadence**: quarterly +**Owner**: `security_owner` (see `templates/config/company_context.example.yaml`) +**Enforcement**: `scripts/check_baselines_expiry.py` + the `security-baseline-expiry` job in `.github/workflows/validate-templates.yml` +**Procedure**: `.security-baselines/README.md` step 4 +**Related**: ADR-024 §Review (the decision that made tfsec/checkov/trivy hard-fail with explicit baselines) + +## Why this document exists + +`.security-baselines/README.md` has instructed reviewers to "update +`docs/audit/baseline-review.md` (next quarterly review)" since the +baselines were introduced. The document did not exist, so three +HIGH-severity suppressions had a written justification in a YAML comment +and no review record anywhere. + +A suppression without a review record is indistinguishable from a +suppression nobody remembers making. This is that record. + +## Review — 2026-09-04 + +### What the review found first: the gate was not watching + +`scripts/check_baselines_expiry.py` reported `OK — no expired or +unannotated entries` while three HIGH GKE checks sat suppressed. It was +not lying about the entries it saw; it saw none. + +Its `yaml_entry` pattern required a suppression id to begin with an +uppercase letter, which matches checkov ids (`CKV_AWS_18`) and misses +every tfsec id (`google-gke-enable-master-networks`). The file was +scanned, zero entries matched, and zero expired entries is a pass. + +Two changes were needed, because the obvious one is wrong on its own: + +1. The pattern is now case-insensitive. +2. Matching lowercase alone would have swept in `framework: [terraform, + kubernetes, dockerfile]` from `checkov.yml`, which is a sequence but + not a suppression. The scanner is now **block-aware** and only treats + items under `exclude:` / `skip-check:` as entries — which is what the + code's own comment had claimed since it was written. + +The three entries also carried `Review-by: 2027-01`, a format the gate +does not parse; the gate requires `# expiry: YYYY-MM-DD` on the entry line +or the line directly above it. They were normalised, keeping the original +prose justification intact above the annotation. + +Verified after the change: the gate reports all three today (in-date), and +`--as-of 2027-06-01` fails all three as expired. + +### Suppressions in force + +All three are tfsec exclusions. `checkov.yml` carries `skip-check: []` — +no suppressions — and no trivy ignore file is present. + +| Check | Severity | Why suppressed | Compensating control | Expiry | +|---|---|---|---|---| +| `google-gke-enforce-pod-security-policy` | HIGH | PodSecurityPolicy was deprecated in Kubernetes 1.21 and **removed** in 1.25. GKE REGULAR channel runs 1.27+, so enabling it would fail the apply outright. | Pod Security Standards enforced via namespace labels in every overlay (D-29, `templates/service/k8s/overlays/*/namespace.yaml`) | 2027-01-01 | +| `google-gke-enable-master-networks` | HIGH | `master_authorized_networks_config` **is** present, as a `dynamic` block. tfsec v1.28 does not evaluate dynamic blocks and reports it absent even when the HCL is correct. | The block is real; staging and prod must supply a non-empty `master_authorized_networks`, enforced by a variable validation rule in `variables.tf` | 2027-01-01 | +| `google-gke-metadata-endpoints-disabled` | HIGH | tfsec reads `node_config.metadata` on `google_container_cluster` only. This module uses `remove_default_node_pool = true` plus two `google_container_node_pool` resources, so the cluster has no `node_config` and the attribute lives on the pools. | Workload Identity on both pools (alone sufficient against metadata SSRF) **plus** `disable-legacy-endpoints = "true"` on each pool | 2027-01-01 | + +Every one is a **tool limitation, not an accepted risk**: in all three +cases the control exists and tfsec cannot see it. That distinction is the +reason the expiries are tied to the tfsec → trivy migration rather than to +a risk-acceptance window. + +### Verdict + +All three suppressions remain justified. No new suppressions were added. +No entry was extended. + +## Next review + +**Due 2027-01-01**, when all three expiries land together and CI will fail +until each is re-justified or removed. The trigger to close them early is +the tfsec → trivy migration: trivy evaluates dynamic blocks and correlates +node pools to their cluster, so at least the second and third suppressions +are expected to become unnecessary rather than merely re-approved. + +## How to run a review + +1. `python3 scripts/check_baselines_expiry.py` — confirms nothing is + expired or unannotated today. +2. `python3 scripts/check_baselines_expiry.py --as-of ` — + shows what is about to come due. +3. For each entry still in force: confirm the compensating control still + exists, in the file the justification names. A justification that + points at a control that has since moved is a suppression with no + backing. +4. Record the outcome as a new dated section in this file. Do not edit a + previous review; the point of the record is that it accumulates. diff --git a/docs/decisions/README.md b/docs/decisions/README.md new file mode 100644 index 0000000..34712ec --- /dev/null +++ b/docs/decisions/README.md @@ -0,0 +1,68 @@ + + +# Architecture Decision Records + +Every non-trivial decision in this template is recorded here with its +measured trade-offs. This index is generated from the files themselves, so +it cannot drift from what the directory actually contains. + +`docs/COMPLIANCE_MAPPING.md` cites this index as EU AI Act Art. 11 +evidence (technical documentation sufficient to assess compliance). It is +therefore load-bearing: an auditor reading the mapping lands here. + +Numbering is dense and gaps are deliberate — a withdrawn ADR keeps its +number and says so, rather than being deleted and leaving a hole +(`scripts/check_doc_coherence.py` C5 enforces this). + + +**45 decisions recorded.** + +| ADR | Decision | +|---|---| +| 001 | [Template Scope Boundaries](ADR-001-template-scope-boundaries.md) | +| 002 | [Model Promotion Governance as Opt-in Module](ADR-002-model-promotion-governance.md) | +| 003 | [Feast Integration Pattern (External Feature Repo)](ADR-003-feast-integration-pattern.md) | +| 004 | [EDA Phase Integration into the Agentic Pipeline](ADR-004-eda-phase-integration.md) | +| 005 | [Agent Behavior Protocol + Supply Chain Security](ADR-005-agent-behavior-and-security.md) | +| 006 | [Closed-Loop Monitoring with Delayed Ground-Truth Labels](ADR-006-closed-loop-monitoring.md) | +| 007 | [Sliced Performance Analysis as First-Class Monitoring](ADR-007-sliced-performance-analysis.md) | +| 008 | [Champion/Challenger Statistical Gate Before Promotion](ADR-008-champion-challenger-statistical-gate.md) | +| 009 | [Retraining Orchestration — When (and When Not) to Migrate Beyond GitHub Actions](ADR-009-retraining-orchestration-triggers.md) | +| 010 | [Dynamic Behavior Protocol via MCP-Prometheus](ADR-010-dynamic-behavior-protocol.md) | +| 011 | [Environment Promotion Gates (dev → staging → prod)](ADR-011-environment-promotion-gates.md) | +| 012 | [API Evolution Policy (reserved number, withdrawn)](ADR-012-api-evolution-policy.md) | +| 013 | [GitOps Strategy — `kubectl apply` now, ArgoCD when it pays off](ADR-013-gitops-strategy.md) | +| 014 | [Gap Remediation Plan — v1.9.0 → v2.0.0 Public Release](ADR-014-gap-remediation-plan.md) | +| 015 | [Productization Roadmap (post-audit)](ADR-015-productization-roadmap.md) | +| 016 | [External Audit R2 — Remediation Plan](ADR-016-external-audit-r2-remediation.md) | +| 017 | [Network Mode + Per-Environment IAM Split (PR-A1)](ADR-017-network-iam-split.md) | +| 018 | [Operational Memory Plane](ADR-018-operational-memory-plane.md) | +| 019 | [Agentic CI Self-Healing](ADR-019-agentic-ci-self-healing.md) | +| 020 | [External Audit R4 — Remediation Plan (master)](ADR-020-r4-audit-remediation.md) | +| 021 | [Fairness Thresholds — Disparate Impact Ratio Floor](ADR-021-fairness-thresholds.md) | +| 022 | [PSI Drift Thresholds — Warn / Alert Cutoffs](ADR-022-psi-thresholds.md) | +| 023 | [Agentic Portability Layer and Contextualization](ADR-023-agentic-portability-and-context.md) | +| 024 | [May 2026 Audit Remediation & Posture Correction](ADR-024-audit-may-2026-remediation.md) | +| 025 | [`common_utils/` distribution model](ADR-025-common-utils-distribution.md) | +| 026 | [Branch Protection & Tag Immutability via GitHub Rulesets](ADR-026-branch-protection.md) | +| 027 | [Vendor-Neutral Canonical Agentic Surface](ADR-027-vendor-neutral-canonical-surface.md) | +| 028 | [LLM-Assist Integration for Template Maintenance and Day-2 Operations](ADR-028-llm-assist-integration.md) | +| 029 | [Agentic Adoption Contract & Interoperability Strategy](ADR-029-agentic-adoption-contract.md) | +| 030 | [Copier-based Scaffolding Migration](ADR-030-copier-scaffolding-migration.md) | +| 031 | [Documentation Coherence System](ADR-031-documentation-coherence-system.md) | +| 032 | [BentoML as an Optional Alternative Serving Backend](ADR-032-bentoml-alternative-serving-backend.md) | +| 033 | [Local-first Stack Profiles](ADR-033-local-first-stack-profiles.md) | +| 034 | [CCDS-aligned generated layout view](ADR-034-ccds-aligned-generated-layout.md) | +| 035 | [uv adoption + Copier index publication](ADR-035-uv-adoption-copier-index.md) | +| 036 | [Batch-Only Deployment Topology](ADR-036-batch-only-deployment-topology.md) | +| 037 | [Dual-Namespace Retrieval Separation (Operational Memory vs. Pedagogical RAG)](ADR-037-dual-namespace-retrieval-separation.md) | +| 038 | [Compliance Mapping (NIST AI RMF / ISO 42001 / EU AI Act)](ADR-038-compliance-mapping.md) | +| 039 | [CI-Green Verification as a Separated-Verb Agentic Gate](ADR-039-ci-green-verification-gate.md) | +| 040 | [Documentation Language and Private-Reference Guard](ADR-040-doc-language-and-privacy-guard.md) | +| 041 | [Agentic Skill Expansion and Domain Taxonomy (External Landscape Review)](ADR-041-agentic-skill-and-domain-expansion.md) | +| 042 | [Native-Cloud-First Edge Protection (Cloud Armor / AWS WAF+Shield), Cloudflare Optional](ADR-042-native-cloud-edge-protection.md) | +| 043 | [Audit-Grade Quality Guardian: a Maintenance Agent for Enterprise Audit Standards](ADR-043-audit-quality-guardian.md) | +| 044 | [Consolidate black + isort + flake8 into ruff](ADR-044-ruff-toolchain-consolidation.md) | +| 045 | [Separate the release-channel tag namespace from frozen audit snapshots](ADR-045-tag-namespace-separation.md) | diff --git a/scripts/check_baselines_expiry.py b/scripts/check_baselines_expiry.py index 27693a0..435f8f5 100755 --- a/scripts/check_baselines_expiry.py +++ b/scripts/check_baselines_expiry.py @@ -61,13 +61,26 @@ # list / empty file has no entries to expire, which is the valid v0.15.0 # default state. ENTRY_PATTERNS = { - # YAML files: any line that starts with " - " inside a sequence we care - # about. We look at exclude: / skip-check: blocks. - "yaml_entry": re.compile(r"^\s*-\s+[\"']?[A-Z]+[A-Z0-9_-]*[\"']?\s*(#.*)?$"), + # YAML files: a sequence item inside a suppression block. Case-insensitive + # because checkov ids are upper (`CKV_AWS_18`) and tfsec ids are lower + # (`google-gke-enable-master-networks`). + # + # The block-awareness is not decoration. This pattern used to require an + # uppercase first character, so the three lowercase tfsec exclusions were + # invisible: the gate that exists to stop suppressed HIGH findings from + # ageing reported "no entries" while three HIGH GKE checks sat suppressed. + # Matching lowercase alone is not enough either — `framework: [terraform, + # kubernetes, dockerfile]` in checkov.yml is a sequence too, and is not a + # suppression. Only items under `exclude:` / `skip-check:` are entries. + "yaml_entry": re.compile(r"^\s*-\s+[\"']?[A-Za-z][A-Za-z0-9_.-]*[\"']?\s*(#.*)?$"), # Trivy ignore: any non-blank, non-comment line. "trivy_entry": re.compile(r"^\s*([A-Z]+-\d{4}-\d+)\s*(#.*)?$"), } +# Top-level keys whose sequence items are suppressions subject to expiry. +SUPPRESSION_KEYS = ("exclude", "skip-check", "skip_check") +_BLOCK_KEY = re.compile(r"^(?P\s*)(?P[A-Za-z_][A-Za-z0-9_.-]*)\s*:") + EXPIRY_PATTERN = re.compile(r"#\s*expiry:\s*(\d{4}-\d{2}-\d{2})", re.IGNORECASE) @@ -95,7 +108,14 @@ def _scan_yaml(path: Path, today: dt.date) -> list[BaselineFinding]: return findings text = path.read_text(encoding="utf-8") lines = text.splitlines() + in_suppression_block = False for i, line in enumerate(lines, start=1): + key_match = _BLOCK_KEY.match(line) + if key_match: + in_suppression_block = key_match.group("key") in SUPPRESSION_KEYS + continue + if not in_suppression_block: + continue if not ENTRY_PATTERNS["yaml_entry"].match(line): continue # Look for an expiry annotation on the entry line OR the line above diff --git a/scripts/generate_adr_index.py b/scripts/generate_adr_index.py new file mode 100644 index 0000000..b4d6fd1 --- /dev/null +++ b/scripts/generate_adr_index.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Generate (and verify) the ADR index at ``docs/decisions/README.md``. + +Why this script exists +---------------------- +``docs/COMPLIANCE_MAPPING.md`` cites an ADR index as EU AI Act Art. 11 +evidence — "Documentation sufficient to assess compliance". The index did +not exist, and the same row claimed "ADRs (37)" when the directory held +45. A compliance mapping that points at a missing artefact and miscounts +the one it does have is worse than no mapping: it is an assertion an +auditor can falsify in one command. + +Writing the index by hand would have reproduced the defect on the next +ADR. It is generated from the files themselves, and ``--check`` fails CI +when the committed index no longer matches, so the count in the mapping +can be replaced by a link to something that cannot drift. + +Title parsing +------------- +Two heading conventions are in use and both are honoured:: + + # ADR-001: Template Scope Boundaries (older, colon) + # ADR-045 — Separate the release-channel … (newer, em dash) + +Exit codes +---------- +- 0: index written, or (with ``--check``) the committed index is current. +- 1: ``--check`` and the index is stale or missing. +- 2: an ADR filename or heading could not be parsed. + +Usage +----- +:: + + python3 scripts/generate_adr_index.py # rewrite the index + python3 scripts/generate_adr_index.py --check # CI / pre-commit +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +ADR_DIR = REPO_ROOT / "docs" / "decisions" +INDEX_FILE = ADR_DIR / "README.md" + +_FILENAME = re.compile(r"^ADR-(?P\d{3})-(?P[a-z0-9-]+)\.md$") +# `# ADR-001: Title` or `# ADR-045 — Title`; the separator drifted over time. +_HEADING = re.compile(r"^#\s*ADR-(?P\d+)\s*[:—–-]\s*(?P.+?)\s*$") + +_PREAMBLE = """<!-- GENERATED FILE — do not edit by hand. + Regenerate with: python3 scripts/generate_adr_index.py + Verified in CI by: python3 scripts/generate_adr_index.py --check --> + +# Architecture Decision Records + +Every non-trivial decision in this template is recorded here with its +measured trade-offs. This index is generated from the files themselves, so +it cannot drift from what the directory actually contains. + +`docs/COMPLIANCE_MAPPING.md` cites this index as EU AI Act Art. 11 +evidence (technical documentation sufficient to assess compliance). It is +therefore load-bearing: an auditor reading the mapping lands here. + +Numbering is dense and gaps are deliberate — a withdrawn ADR keeps its +number and says so, rather than being deleted and leaving a hole +(`scripts/check_doc_coherence.py` C5 enforces this). + +""" + + +def _collect() -> list[tuple[int, str, str]]: + """Return ``[(number, title, filename)]`` sorted by number.""" + rows: list[tuple[int, str, str]] = [] + errors: list[str] = [] + for path in sorted(ADR_DIR.glob("ADR-*.md")): + name_match = _FILENAME.match(path.name) + if not name_match: + errors.append(f"{path.name}: filename does not match ADR-NNN-slug.md") + continue + first_line = path.read_text(encoding="utf-8").splitlines()[0] if path.stat().st_size else "" + heading = _HEADING.match(first_line) + if not heading: + errors.append(f"{path.name}: first line is not '# ADR-NNN<sep> Title' (got {first_line!r})") + continue + if int(heading.group("number")) != int(name_match.group("number")): + errors.append(f"{path.name}: heading number {heading.group('number')} != filename number") + continue + rows.append((int(name_match.group("number")), heading.group("title"), path.name)) + + if errors: + for error in errors: + sys.stderr.write(f"::error::{error}\n") + raise SystemExit(2) + return rows + + +def _render(rows: list[tuple[int, str, str]]) -> str: + lines = [_PREAMBLE, f"**{len(rows)} decisions recorded.**\n", "| ADR | Decision |", "|---|---|"] + lines += [f"| {number:03d} | [{title}]({filename}) |" for number, title, filename in rows] + return "\n".join(lines) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail if the committed index is stale") + args = parser.parse_args() + + rendered = _render(_collect()) + + if not args.check: + INDEX_FILE.write_text(rendered, encoding="utf-8") + print(f"[adr-index] wrote {INDEX_FILE.relative_to(REPO_ROOT)}") + return 0 + + if not INDEX_FILE.exists(): + sys.stderr.write("::error::docs/decisions/README.md is missing — run scripts/generate_adr_index.py\n") + return 1 + if INDEX_FILE.read_text(encoding="utf-8") != rendered: + sys.stderr.write( + "::error::docs/decisions/README.md is stale. An ADR was added, renamed or " + "retitled without regenerating the index.\n" + "::error::Fix: python3 scripts/generate_adr_index.py\n" + ) + return 1 + + print(f"[adr-index] OK — index lists all {len(_collect())} ADRs and is current.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())