|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Assert that what a document or a build file names still exists. |
| 3 | +
|
| 4 | +Prose and build files are full of references into the repository, and almost |
| 5 | +none of them are checked. `mkdocs build --strict` resolves a Markdown *link* |
| 6 | +between two pages, and the docs-links workflow resolves an external URL, but a |
| 7 | +path written in code font in a sentence is just text, and a path in a CMake |
| 8 | +list, a Compose mount or a workflow's path filter is text nothing reads back |
| 9 | +out. A rename leaves the reference behind, green, and pointing nowhere. |
| 10 | +
|
| 11 | +This is one pass over those files, pulling out every reference of a known kind |
| 12 | +and asserting each resolves. Today there is one kind, the repo-relative path. |
| 13 | +The second, deferred to #740, is the SolidSyslog symbol a page names: the same |
| 14 | +files, the same extraction pass, and the same exception problem, which is why it |
| 15 | +belongs here as another KINDS row rather than in a second script. |
| 16 | +
|
| 17 | +**The extraction is deliberately narrow**, because a heuristic loose enough to |
| 18 | +need a long exception list is one that will be switched off. Three rules do the |
| 19 | +work: |
| 20 | +
|
| 21 | +* A token is read as a path only when its first segment is something git tracks |
| 22 | + at the top of the repository, or `.` / `..`. That excludes an include of a |
| 23 | + third-party header, a URL, and every `and/or` in a sentence without naming any |
| 24 | + of them. What it costs is a top-level directory renamed wholesale. |
| 25 | +* Anything git ignores is a build artefact rather than a reference — it does not |
| 26 | + exist in a fresh checkout and asserting it would be asserting the build ran. |
| 27 | +* A YAML block scalar is a shell script, not repository text. Its paths are |
| 28 | + relative to a working directory this cannot know, and most of them name files |
| 29 | + a job creates. Nothing is lost by leaving them: a shell command naming a path |
| 30 | + that does not exist fails the job it is in, which is exactly what a nav entry |
| 31 | + or a path filter does not do. |
| 32 | +
|
| 33 | +**The exception list is the part to watch.** Each entry says why the reference |
| 34 | +is meant not to resolve. All of them so far are one thing: a document quoting a |
| 35 | +path as some *other* file would write it, to state a rule about how paths are |
| 36 | +written. Past a handful of entries the extraction is wrong and should be |
| 37 | +tightened rather than the list grown. |
| 38 | +
|
| 39 | +Run: python3 scripts/check_references.py |
| 40 | +""" |
| 41 | + |
| 42 | +import os |
| 43 | +import re |
| 44 | +import subprocess |
| 45 | +import sys |
| 46 | + |
| 47 | +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 48 | + |
| 49 | +# Deliberate exceptions, each with the reason the reference is meant not to |
| 50 | +# resolve. Matched on (file, token). Printed on every run: an exception nobody |
| 51 | +# sees is an exception nobody revisits. |
| 52 | +ALLOWED = [ |
| 53 | + ( |
| 54 | + "CLAUDE.md", |
| 55 | + "../LICENSE.md", |
| 56 | + "states the rule for how a page under docs/ links a root document, so the " |
| 57 | + "path is counted from that page rather than from here", |
| 58 | + ), |
| 59 | + ( |
| 60 | + "CLAUDE.md", |
| 61 | + "../../LICENSE.md", |
| 62 | + "the same rule, counted from a page one level deeper", |
| 63 | + ), |
| 64 | + ( |
| 65 | + "mkdocs.yml", |
| 66 | + "../SECURITY.md", |
| 67 | + "names a link as a docs page writes it, to say what the source-link hook rewrites", |
| 68 | + ), |
| 69 | + ( |
| 70 | + ".github/workflows/ci.yml", |
| 71 | + "../SECURITY.md", |
| 72 | + "the same link, in the comment on the step that would fail if the hook " |
| 73 | + "stopped rewriting it", |
| 74 | + ), |
| 75 | + ( |
| 76 | + "Tests/Lwip/CMakeLists.txt", |
| 77 | + "Tests/X/", |
| 78 | + "the shape the next platform's test directory would take, not one that exists", |
| 79 | + ), |
| 80 | +] |
| 81 | + |
| 82 | +# Where references are looked for. Build files are here because nothing else |
| 83 | +# reads them back: a path in a CMake list or a path filter is checked by the job |
| 84 | +# it belongs to failing, months later, on a branch that did not touch it. |
| 85 | +SCANNED_NAMES = ("CMakeLists.txt",) |
| 86 | +SCANNED_SUFFIXES = (".md", ".yaml", ".yml", ".cmake") |
| 87 | + |
| 88 | +# A fence opens and closes a block whose whole content is candidate text rather |
| 89 | +# than prose. |
| 90 | +FENCE = re.compile(r"^\s*(?:```|~~~)") |
| 91 | + |
| 92 | +# Markdown carries references in two shapes: a code span, and a link target. |
| 93 | +CODE_SPAN = re.compile(r"`([^`\n]+)`") |
| 94 | +LINK_TARGET = re.compile(r"\]\(([^)\s]+)\)") |
| 95 | + |
| 96 | +# A YAML key introducing a block scalar, and the indent its body sits under. |
| 97 | +BLOCK_SCALAR = re.compile(r"^(\s*)(?:-\s+)?[^:#]+:\s*[|>][-+0-9]*\s*(?:#.*)?$") |
| 98 | + |
| 99 | +# A step's script written inline rather than as a block. Same reasoning: it is |
| 100 | +# shell, so its paths are the job's, not the repository's. |
| 101 | +SHELL = re.compile(r"^\s*(?:-\s+)?(?:run|command|entrypoint):\s*\S") |
| 102 | + |
| 103 | +# Punctuation a token collects from the sentence or the syntax around it. A |
| 104 | +# trailing full stop goes; a trailing `.md` does not, since `d` is not stripped. |
| 105 | +LEADING = "([{<\"'`,;:" |
| 106 | +TRAILING = ".,;:!?)]}>\"'`" |
| 107 | + |
| 108 | +# What disqualifies a word before its shape is considered: a URL or an address, |
| 109 | +# a glob or a placeholder, a shell or CMake variable, or a character no path in |
| 110 | +# this repository uses. Each names something other than one file here. |
| 111 | +NOT_A_PATH = re.compile(r"://|[^A-Za-z0-9._/+:-]") |
| 112 | + |
| 113 | + |
| 114 | +def read(relative): |
| 115 | + with open(os.path.join(ROOT, relative), encoding="utf-8") as handle: |
| 116 | + return handle.read() |
| 117 | + |
| 118 | + |
| 119 | +def git(*arguments, stdin=None): |
| 120 | + """Git is the authority on what this repository holds and what it ignores; |
| 121 | + a hand-kept copy of either would be one more thing to keep in step.""" |
| 122 | + return subprocess.run( |
| 123 | + ["git", "-C", ROOT, *arguments], |
| 124 | + input=stdin, |
| 125 | + capture_output=True, |
| 126 | + text=True, |
| 127 | + check=False, |
| 128 | + ).stdout.splitlines() |
| 129 | + |
| 130 | + |
| 131 | +def tracked_roots(): |
| 132 | + """What git tracks at the top of the repository — the first segment a path |
| 133 | + must have to be read as one. Untracked output (`build/`, `site/`) is not |
| 134 | + here, so nothing under it is ever mined.""" |
| 135 | + return {name.split("/")[0] for name in git("ls-files")} |
| 136 | + |
| 137 | + |
| 138 | +def ignored(paths): |
| 139 | + """The subset git ignores: build artefacts, which a fresh checkout lacks.""" |
| 140 | + if not paths: |
| 141 | + return set() |
| 142 | + return set(git("check-ignore", "--stdin", stdin="\n".join(sorted(paths)) + "\n")) |
| 143 | + |
| 144 | + |
| 145 | +def scanned(): |
| 146 | + """Every file references are looked for in, repo-relative and sorted. Taken |
| 147 | + from git rather than from a walk, so generated trees never appear.""" |
| 148 | + return sorted( |
| 149 | + name |
| 150 | + for name in git("ls-files") |
| 151 | + if os.path.basename(name) in SCANNED_NAMES or name.endswith(SCANNED_SUFFIXES) |
| 152 | + ) |
| 153 | + |
| 154 | + |
| 155 | +def candidates(relative, line, verbatim): |
| 156 | + """The text on one line that may hold a reference. |
| 157 | +
|
| 158 | + Prose is mined only inside a code span or a link target — a path in a |
| 159 | + sentence is written in code font by convention, and one that is not is prose |
| 160 | + about a path rather than a reference to it. Verbatim text, a fenced block or |
| 161 | + a build file, is candidate text whole. |
| 162 | + """ |
| 163 | + if relative.endswith(".md") and not verbatim: |
| 164 | + return [m.group(1) for m in CODE_SPAN.finditer(line)] + [ |
| 165 | + m.group(1) for m in LINK_TARGET.finditer(line) |
| 166 | + ] |
| 167 | + return [line] |
| 168 | + |
| 169 | + |
| 170 | +def words(text): |
| 171 | + """The path-shaped words of some candidate text, unpunctuated. |
| 172 | +
|
| 173 | + A word is split on `:` after it has been judged, so a Compose mount |
| 174 | + (`../Bdd/output:/var/log`) yields both sides and a URL yields neither. |
| 175 | + """ |
| 176 | + for word in text.split(): |
| 177 | + word = word.lstrip(LEADING).rstrip(TRAILING) |
| 178 | + if word and not NOT_A_PATH.search(word): |
| 179 | + for part in word.split(":"): |
| 180 | + yield part |
| 181 | + |
| 182 | + |
| 183 | +def names_a_file(token): |
| 184 | + """A path is written as one: it names a file, or it carries the trailing |
| 185 | + slash that says it is a directory. |
| 186 | +
|
| 187 | + The repository is full of slash-joined names that are not paths — a branch |
| 188 | + (`ci/pin-action-shas`), a component (`Bdd/Targets/Common/BddTargetInteractive`), |
| 189 | + a test (`Tests/Lwip/SolidSyslogLwipRawDnsResolverTest`), a pair of |
| 190 | + directories written as one (`Core/Platform`). Requiring the shape separates |
| 191 | + them without naming any of them, at the cost of a directory referred to |
| 192 | + without its slash. |
| 193 | + """ |
| 194 | + tail = token.rsplit("/", 1)[-1] |
| 195 | + return token.endswith("/") or ("." in tail and tail not in (".", "..")) |
| 196 | + |
| 197 | + |
| 198 | +def paths_in(relative, line, verbatim, roots): |
| 199 | + """Every token on this line that is a reference to a path in this repository. |
| 200 | +
|
| 201 | + An anchor or a query names a place within the target rather than a different |
| 202 | + target, so both are cut before the path is resolved. |
| 203 | + """ |
| 204 | + for text in candidates(relative, line, verbatim): |
| 205 | + for word in words(text): |
| 206 | + token = word.split("#")[0].split("?")[0] |
| 207 | + if "/" not in token or not names_a_file(token): |
| 208 | + continue |
| 209 | + if token.split("/")[0] in roots or token.startswith((".", "..")): |
| 210 | + yield token |
| 211 | + |
| 212 | + |
| 213 | +def path_targets(relative, token): |
| 214 | + """What a path token could mean, as repo-relative paths. |
| 215 | +
|
| 216 | + Both spellings are used and both are correct: from the repository root, and |
| 217 | + from the directory of the file that names it. Anything that normalises to |
| 218 | + outside the repository is not a reference into it and drops out here. |
| 219 | + """ |
| 220 | + directory = os.path.dirname(relative) |
| 221 | + spellings = {os.path.normpath(token), os.path.normpath(os.path.join(directory, token))} |
| 222 | + return {p for p in spellings if not p.startswith("..") and not os.path.isabs(p)} |
| 223 | + |
| 224 | + |
| 225 | +def path_resolves(relative, token): |
| 226 | + return any(os.path.exists(os.path.join(ROOT, p)) for p in path_targets(relative, token)) |
| 227 | + |
| 228 | + |
| 229 | +def path_artefacts(found): |
| 230 | + """The references naming something the build produces rather than something |
| 231 | + the repository holds. Asked of git in one call, since a reference costs |
| 232 | + nothing to extract and a process costs a great deal to start.""" |
| 233 | + artefacts = ignored({target for reference in found for target in path_targets(*reference)}) |
| 234 | + return { |
| 235 | + reference |
| 236 | + for reference in found |
| 237 | + if any(target in artefacts for target in path_targets(*reference)) |
| 238 | + } |
| 239 | + |
| 240 | + |
| 241 | +class Kind: |
| 242 | + """One class of reference: how to find it, how to resolve it, what to say. |
| 243 | +
|
| 244 | + `unassertable` is given every reference of this kind at once and returns |
| 245 | + those that cannot be asserted at all, as against those that fail — a batch |
| 246 | + so that a check needing to ask git something asks it once. |
| 247 | + """ |
| 248 | + |
| 249 | + def __init__(self, extract, resolves, complaint, unassertable): |
| 250 | + self.extract = extract |
| 251 | + self.resolves = resolves |
| 252 | + self.complaint = complaint |
| 253 | + self.unassertable = unassertable |
| 254 | + |
| 255 | + |
| 256 | +# The kinds of reference asserted. #740 adds the SolidSyslog symbol a page names |
| 257 | +# as a second row here, reusing the pass above and the exception list below. |
| 258 | +KINDS = ( |
| 259 | + Kind( |
| 260 | + extract=paths_in, |
| 261 | + resolves=path_resolves, |
| 262 | + complaint="names a path that does not exist", |
| 263 | + unassertable=path_artefacts, |
| 264 | + ), |
| 265 | +) |
| 266 | + |
| 267 | + |
| 268 | +def references(kind): |
| 269 | + """Every (file, token) of one kind, with the line each was found on.""" |
| 270 | + roots = tracked_roots() |
| 271 | + found = {} |
| 272 | + for relative in scanned(): |
| 273 | + verbatim = not relative.endswith(".md") |
| 274 | + block = None |
| 275 | + for number, line in enumerate(read(relative).splitlines(), 1): |
| 276 | + if relative.endswith(".md"): |
| 277 | + if FENCE.match(line): |
| 278 | + verbatim = not verbatim |
| 279 | + continue |
| 280 | + else: |
| 281 | + block = inside_block(line, block) |
| 282 | + if block is not None or SHELL.match(line): |
| 283 | + continue |
| 284 | + for token in kind.extract(relative, line, verbatim, roots): |
| 285 | + found.setdefault((relative, token), number) |
| 286 | + return found |
| 287 | + |
| 288 | + |
| 289 | +def inside_block(line, block): |
| 290 | + """The indent of the block scalar this line sits in, or None. A block ends |
| 291 | + at the first non-blank line indented no further than the key that opened |
| 292 | + it.""" |
| 293 | + if block is not None: |
| 294 | + if not line.strip() or len(line) - len(line.lstrip()) > block: |
| 295 | + return block |
| 296 | + opened = BLOCK_SCALAR.match(line) |
| 297 | + return len(opened.group(1)) if opened else None |
| 298 | + |
| 299 | + |
| 300 | +def check(): |
| 301 | + # Everything below is asked of git, so a run outside a working tree would |
| 302 | + # find no files, assert nothing, and pass. Fail loudly instead. |
| 303 | + if not scanned(): |
| 304 | + sys.exit(f"no documents or build files found under {ROOT} — is this a git checkout?") |
| 305 | + |
| 306 | + exempt = {(path, token) for path, token, _ in ALLOWED} |
| 307 | + faults = [] |
| 308 | + for kind in KINDS: |
| 309 | + found = {r: n for r, n in references(kind).items() if r not in exempt} |
| 310 | + assertable = set(found) - kind.unassertable(set(found)) |
| 311 | + faults += [ |
| 312 | + f"{relative}:{found[(relative, token)]} {kind.complaint}: {token}" |
| 313 | + for relative, token in sorted(assertable) |
| 314 | + if not kind.resolves(relative, token) |
| 315 | + ] |
| 316 | + return sorted(faults) |
| 317 | + |
| 318 | + |
| 319 | +if __name__ == "__main__": |
| 320 | + problems = check() |
| 321 | + for problem in problems: |
| 322 | + print(f"error: {problem}", file=sys.stderr) |
| 323 | + if problems: |
| 324 | + print( |
| 325 | + f"\n{len(problems)} problem(s). Fix the reference, or — if it is " |
| 326 | + "deliberately unresolvable — add it to ALLOWED with the reason.", |
| 327 | + file=sys.stderr, |
| 328 | + ) |
| 329 | + sys.exit(1) |
| 330 | + for path, token, reason in ALLOWED: |
| 331 | + print(f"allowed: {path} may name {token} — {reason}") |
| 332 | + print(f"every path named by {len(scanned())} documents and build files exists") |
0 commit comments