|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Agent index contracts: the opt-in LLMSFULL full-text bundle. |
| 3 | +
|
| 4 | +The bundle concatenates the same semantic Markdown the per-page output |
| 5 | +publishes, in the sidebar reading order, one file per enabled top-level |
| 6 | +section per language. This checker owns: |
| 7 | +
|
| 8 | +- language isolation: each bundle carries only its own language's pages; |
| 9 | +- source integrity: every `Source:` pointer resolves to a built artifact, |
| 10 | + appears exactly once, and the section index leads the sequence; |
| 11 | +- reading order: pages follow their declared front-matter weights; |
| 12 | +- discovery: llms.txt lists the enabled bundle for its own language; |
| 13 | +- determinism: two builds of the same sources produce identical bundles; |
| 14 | +- the negative contract: enabling LLMSFULL on a nested section warns, emits |
| 15 | + nothing, keeps a plain build usable, and fails --panicOnWarning. |
| 16 | +
|
| 17 | +Bundle sizes are reported as evidence, never enforced: a model-context |
| 18 | +ceiling is a consumer's judgement, not a build gate. |
| 19 | +
|
| 20 | + bin/check-agent-indexes.py # build the fixture and check |
| 21 | + bin/check-agent-indexes.py --hugo PATH # build with another Hugo binary |
| 22 | + bin/check-agent-indexes.py --public DIR # reuse a build (skips determinism) |
| 23 | +""" |
| 24 | + |
| 25 | +from __future__ import annotations |
| 26 | + |
| 27 | +import argparse |
| 28 | +import re |
| 29 | +import subprocess |
| 30 | +import sys |
| 31 | +import tempfile |
| 32 | +from pathlib import Path |
| 33 | + |
| 34 | +from test_site import ROOT, TEST_SITE, build_fixture_public, fixture_config |
| 35 | + |
| 36 | +BASE_URL = "https://example.org/" |
| 37 | +BUILD_TIMEOUT = 120 |
| 38 | + |
| 39 | +BUNDLES = { |
| 40 | + "docs/llms-full.txt": {"lang": "en", "index": "docs/index.md"}, |
| 41 | + "zh/docs/llms-full.txt": {"lang": "zh", "index": "zh/docs/index.md"}, |
| 42 | +} |
| 43 | + |
| 44 | + |
| 45 | +def require(condition: bool, message: str, errors: list[str]) -> None: |
| 46 | + if not condition: |
| 47 | + errors.append(message) |
| 48 | + |
| 49 | + |
| 50 | +def bundle_sources(text: str) -> list[str]: |
| 51 | + return re.findall(r"^Source: (\S+)$", text, flags=re.MULTILINE) |
| 52 | + |
| 53 | + |
| 54 | +def weight_order(language: str) -> list[str]: |
| 55 | + """Expected relative URL order of the docs pages, from front-matter weights. |
| 56 | +
|
| 57 | + An independent oracle: the bundle must present pages in ascending declared |
| 58 | + weight, the same order the sidebar and pager derive from the content tree. |
| 59 | + Only pages with an explicit unique weight participate; the fixture keeps |
| 60 | + them unique on purpose. |
| 61 | + """ |
| 62 | + |
| 63 | + weighted: list[tuple[int, str]] = [] |
| 64 | + for path in sorted((TEST_SITE / "content/docs").glob("*.md")): |
| 65 | + if path.name.startswith("_index"): |
| 66 | + continue |
| 67 | + is_zh = path.name.endswith(".zh.md") |
| 68 | + if (language == "zh") != is_zh: |
| 69 | + continue |
| 70 | + front = path.read_text(encoding="utf-8").split("---\n") |
| 71 | + if len(front) < 2: |
| 72 | + continue |
| 73 | + match = re.search(r"^weight:\s*(\d+)\s*$", front[1], flags=re.MULTILINE) |
| 74 | + if not match: |
| 75 | + continue |
| 76 | + slug = path.name.removesuffix(".zh.md").removesuffix(".md") |
| 77 | + prefix = "zh/" if language == "zh" else "" |
| 78 | + weighted.append((int(match.group(1)), f"{prefix}docs/{slug}/index.md")) |
| 79 | + weighted.sort() |
| 80 | + return [url for _, url in weighted] |
| 81 | + |
| 82 | + |
| 83 | +def check_bundle(public: Path, rel_path: str, spec: dict, errors: list[str]) -> None: |
| 84 | + bundle_path = public / rel_path |
| 85 | + require(bundle_path.exists(), f"{rel_path} was not built", errors) |
| 86 | + if not bundle_path.exists(): |
| 87 | + return |
| 88 | + text = bundle_path.read_text(encoding="utf-8") |
| 89 | + sources = bundle_sources(text) |
| 90 | + require(len(sources) >= 2, f"{rel_path} holds fewer than two pages", errors) |
| 91 | + require(len(sources) == len(set(sources)), |
| 92 | + f"{rel_path} repeats a Source pointer", errors) |
| 93 | + separators = text.count("================\n") |
| 94 | + require(separators == 2 * len(sources), |
| 95 | + f"{rel_path} separator count {separators} does not frame " |
| 96 | + f"{len(sources)} Source pointers", errors) |
| 97 | + |
| 98 | + paths: list[str] = [] |
| 99 | + for url in sources: |
| 100 | + require(url.startswith(BASE_URL), |
| 101 | + f"{rel_path} lists an off-site source: {url}", errors) |
| 102 | + if not url.startswith(BASE_URL): |
| 103 | + continue |
| 104 | + rel = url.removeprefix(BASE_URL) |
| 105 | + paths.append(rel) |
| 106 | + require((public / rel).exists(), |
| 107 | + f"{rel_path} points at an unbuilt source: {url}", errors) |
| 108 | + in_zh = rel.startswith("zh/") |
| 109 | + require(in_zh == (spec["lang"] == "zh"), |
| 110 | + f"{rel_path} crosses languages with {url}", errors) |
| 111 | + |
| 112 | + require(bool(paths) and paths[0] == spec["index"], |
| 113 | + f"{rel_path} does not lead with its section index", errors) |
| 114 | + |
| 115 | + expected = weight_order(spec["lang"]) |
| 116 | + listed = [path for path in paths if path in set(expected)] |
| 117 | + require(listed == expected, |
| 118 | + f"{rel_path} order diverges from front-matter weights:\n" |
| 119 | + f" bundle: {listed}\n expected: {expected}", errors) |
| 120 | + |
| 121 | + # One renderer, provably: each bundle segment must equal the page's own |
| 122 | + # published .md byte-for-byte (modulo surrounding blank lines). Author |
| 123 | + # markup quoted in source flows through both on purpose; output purity |
| 124 | + # itself is owned by the per-page markdown gates. |
| 125 | + segments = re.split( |
| 126 | + r"^================\nSource: (\S+)\n================\n\n", |
| 127 | + text, flags=re.MULTILINE) |
| 128 | + for url, body in zip(segments[1::2], segments[2::2]): |
| 129 | + rel = url.removeprefix(BASE_URL) |
| 130 | + if not rel.endswith(".md"): |
| 131 | + continue |
| 132 | + page_file = public / rel |
| 133 | + if not page_file.exists(): |
| 134 | + continue # already reported by the source-integrity check |
| 135 | + require(body.strip() == page_file.read_text(encoding="utf-8").strip(), |
| 136 | + f"{rel_path} segment for {rel} diverges from the per-page output", |
| 137 | + errors) |
| 138 | + |
| 139 | + print(f" {rel_path}: {len(sources)} pages, {len(text.encode('utf-8'))} bytes") |
| 140 | + |
| 141 | + |
| 142 | +def check_discovery(public: Path, errors: list[str]) -> None: |
| 143 | + for llms, bundle in (("llms.txt", "docs/llms-full.txt"), |
| 144 | + ("zh/llms.txt", "zh/docs/llms-full.txt")): |
| 145 | + index = public / llms |
| 146 | + require(index.exists(), f"{llms} was not built", errors) |
| 147 | + if not index.exists(): |
| 148 | + continue |
| 149 | + text = index.read_text(encoding="utf-8") |
| 150 | + require(f"{BASE_URL}{bundle}" in text, |
| 151 | + f"{llms} does not list the enabled bundle {bundle}", errors) |
| 152 | + |
| 153 | + |
| 154 | +def check_determinism(hugo: str, first: Path, errors: list[str]) -> None: |
| 155 | + second, result = build_fixture_public(hugo, "--panicOnWarning") |
| 156 | + require(result.returncode == 0, |
| 157 | + "second fixture build failed; determinism unverified", errors) |
| 158 | + if result.returncode != 0: |
| 159 | + return |
| 160 | + for rel_path in BUNDLES: |
| 161 | + a = (first / rel_path).read_bytes() if (first / rel_path).exists() else b"" |
| 162 | + b = (second / rel_path).read_bytes() if (second / rel_path).exists() else b"" |
| 163 | + require(a == b, f"{rel_path} differs between two identical builds", errors) |
| 164 | + |
| 165 | + |
| 166 | +def check_nested_section(hugo: str, errors: list[str]) -> None: |
| 167 | + """Enabling LLMSFULL below the top level warns and blocks publication.""" |
| 168 | + |
| 169 | + with tempfile.TemporaryDirectory(prefix="oink-agent-indexes-nested-") as temp: |
| 170 | + temp_path = Path(temp) |
| 171 | + override = temp_path / "override.yaml" |
| 172 | + override.write_text( |
| 173 | + "outputs:\n section: [HTML, print, RSS, markdown, LLMSFULL]\n", |
| 174 | + encoding="utf-8", |
| 175 | + ) |
| 176 | + command = [ |
| 177 | + hugo, |
| 178 | + "--source", str(TEST_SITE), |
| 179 | + "--themesDir", str(ROOT.parent), |
| 180 | + "--destination", str(temp_path / "public"), |
| 181 | + "--config", fixture_config(override), |
| 182 | + "--logLevel", "warn", |
| 183 | + ] |
| 184 | + result = subprocess.run(command, capture_output=True, text=True, |
| 185 | + check=False, timeout=BUILD_TIMEOUT) |
| 186 | + output = result.stdout + result.stderr |
| 187 | + expected = "LLMSFULL output requires a top-level section" |
| 188 | + require(expected in output, |
| 189 | + f"nested-section LLMSFULL did not report {expected!r}", errors) |
| 190 | + require(result.returncode == 0, |
| 191 | + "nested-section LLMSFULL stopped a plain build instead of warning", |
| 192 | + errors) |
| 193 | + nested = temp_path / "public/fixtures/guides/llms-full.txt" |
| 194 | + require((not nested.exists()) or nested.read_text(encoding="utf-8") == "", |
| 195 | + "nested-section LLMSFULL emitted content instead of nothing", errors) |
| 196 | + # A wedge under --panicOnWarning is the panic path seizing; the build |
| 197 | + # certainly did not publish, which is the assertion. |
| 198 | + try: |
| 199 | + strict = subprocess.run( |
| 200 | + command + ["--panicOnWarning"], |
| 201 | + capture_output=True, text=True, check=False, |
| 202 | + timeout=BUILD_TIMEOUT, |
| 203 | + ) |
| 204 | + require(strict.returncode != 0, |
| 205 | + "nested-section LLMSFULL survived --panicOnWarning", errors) |
| 206 | + except subprocess.TimeoutExpired: |
| 207 | + print(f"hugo wedged after {BUILD_TIMEOUT}s under --panicOnWarning; " |
| 208 | + "counting the wedge as the expected failure", file=sys.stderr) |
| 209 | + |
| 210 | + |
| 211 | +def main() -> int: |
| 212 | + parser = argparse.ArgumentParser( |
| 213 | + description=__doc__, |
| 214 | + formatter_class=argparse.RawDescriptionHelpFormatter, |
| 215 | + ) |
| 216 | + parser.add_argument("--hugo", default="hugo") |
| 217 | + parser.add_argument("--public", type=Path) |
| 218 | + args = parser.parse_args() |
| 219 | + |
| 220 | + errors: list[str] = [] |
| 221 | + if args.public: |
| 222 | + public = args.public |
| 223 | + else: |
| 224 | + public, result = build_fixture_public( |
| 225 | + args.hugo, "--printPathWarnings", "--panicOnWarning") |
| 226 | + if result.returncode != 0: |
| 227 | + print(result.stdout + result.stderr, file=sys.stderr) |
| 228 | + raise SystemExit("regression fixture build failed") |
| 229 | + |
| 230 | + for rel_path, spec in BUNDLES.items(): |
| 231 | + check_bundle(public, rel_path, spec, errors) |
| 232 | + check_discovery(public, errors) |
| 233 | + if args.public is None: |
| 234 | + check_determinism(args.hugo, public, errors) |
| 235 | + check_nested_section(args.hugo, errors) |
| 236 | + else: |
| 237 | + print(" (reused build: determinism and nested-section cases skipped)") |
| 238 | + |
| 239 | + if errors: |
| 240 | + print("Agent index checks failed:") |
| 241 | + for error in errors: |
| 242 | + print(f" {error}") |
| 243 | + return 1 |
| 244 | + print("Agent index checks passed") |
| 245 | + return 0 |
| 246 | + |
| 247 | + |
| 248 | +if __name__ == "__main__": |
| 249 | + sys.exit(main()) |
0 commit comments