From 62ed32746396a387331632254dd9ef780d8ab22e Mon Sep 17 00:00:00 2001 From: jmservera Date: Sat, 1 Aug 2026 19:39:51 +0200 Subject: [PATCH 1/3] feat(ci): validate embed source_page references before build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add scripts/check_embed_sources.py, which verifies every content/embeds/* page's source_page resolves to an existing content/data page. A dangling reference makes the Hugo build abort via errorf (layouts/embeds/single.html, layouts/shortcodes/observatory-chart.html) — the failure mode behind issue #627. - Runs as a pytest (tests/test_embed_sources.py) in the required Python job, so a PR that adds an embed without its data page fails CI, not the deploy. - Runs as an explicit fast-fail step in the production-site job before the Hugo build, giving deploy-parity coverage. Implements Phase 4 (CI deploy-parity guard) of the deploy-hydration remediation plan. --- .github/workflows/ci.yml | 3 ++ scripts/check_embed_sources.py | 71 ++++++++++++++++++++++++++++++++++ tests/test_embed_sources.py | 47 ++++++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 scripts/check_embed_sources.py create mode 100644 tests/test_embed_sources.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aec64b0..3c91df1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,9 @@ jobs: "lighthouse@${LIGHTHOUSE_VERSION}" npx --no-install playwright install --with-deps chromium + - name: Validate embed source pages + run: python3 scripts/check_embed_sources.py + - name: Build site and capture Hugo duration id: hugo-build run: | diff --git a/scripts/check_embed_sources.py b/scripts/check_embed_sources.py new file mode 100644 index 0000000..73f8667 --- /dev/null +++ b/scripts/check_embed_sources.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Validate that every embed page's source_page resolves to an existing data page. + +A `content/embeds/*` page renders a chart from another page referenced by its +`source_page` front matter. If that target page is missing, the Hugo build aborts +(`layouts/embeds/single.html` and `layouts/shortcodes/observatory-chart.html` +call `errorf`). This check catches a dangling reference before the build, so the +failure surfaces in CI rather than in the production deploy (see issue #627). +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + +SOURCE_PAGE_PATTERN = re.compile( + r"""^\s*source_page\s*[:=]\s*["']?(?P[^"'\n]+?)["']?\s*$""", + re.MULTILINE, +) + + +def _source_page(document: str) -> str | None: + match = SOURCE_PAGE_PATTERN.search(document) + return match.group("value").strip() if match else None + + +def _resolves(content_root: Path, source_page: str) -> bool: + relative = source_page.strip("/") + if not relative: + return False + target = content_root / relative + return (target / "index.md").is_file() or (target / "_index.md").is_file() + + +def check_embed_sources(root: Path = ROOT) -> list[str]: + """Return a list of human-readable problems, empty when all embeds resolve.""" + content_root = root / "content" + problems: list[str] = [] + + for embed in sorted(content_root.glob("embeds/*/index.md")): + source_page = _source_page(embed.read_text(encoding="utf-8")) + relative_embed = embed.relative_to(root).as_posix() + if not source_page: + problems.append(f"{relative_embed}: missing source_page") + continue + if not _resolves(content_root, source_page): + problems.append(f"{relative_embed}: source_page does not resolve: {source_page}") + + return problems + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", default=str(ROOT), type=Path, help="Repository root") + args = parser.parse_args(argv) + + problems = check_embed_sources(args.root) + for problem in problems: + print(f"::error::{problem}", file=sys.stderr) + if problems: + return 1 + print("All embed source_page references resolve.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_embed_sources.py b/tests/test_embed_sources.py new file mode 100644 index 0000000..b5bd180 --- /dev/null +++ b/tests/test_embed_sources.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import tempfile +from pathlib import Path + +from scripts.check_embed_sources import check_embed_sources + +ROOT = Path(__file__).resolve().parent.parent + + +def test_repository_embed_sources_all_resolve() -> None: + assert check_embed_sources(ROOT) == [] + + +def _write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def test_missing_source_page_is_reported() -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write(root / "content/embeds/chart/index.md", '+++\ntitle = "Chart"\n+++\n') + problems = check_embed_sources(root) + assert any("missing source_page" in problem for problem in problems) + + +def test_dangling_source_page_is_reported() -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write( + root / "content/embeds/chart/index.md", + '+++\nsource_page = "/data/does-not-exist/"\n+++\n', + ) + problems = check_embed_sources(root) + assert any("does not resolve" in problem for problem in problems) + + +def test_resolving_source_page_passes() -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write( + root / "content/embeds/chart/index.md", + '+++\nsource_page = "/data/live/"\n+++\n', + ) + _write(root / "content/data/live/index.md", '+++\ntitle = "Live"\n+++\n') + assert check_embed_sources(root) == [] From e64ae1e13816a21b8c15b8a0ba72d3c3d460cdf4 Mon Sep 17 00:00:00 2001 From: jmservera Date: Sat, 1 Aug 2026 19:52:34 +0200 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20address=20#641=20review=20=E2=80=94?= =?UTF-8?q?=20require=20data=20page=20target,=20use=20python?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- scripts/check_embed_sources.py | 8 ++++++-- tests/test_embed_sources.py | 13 +++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c91df1..fae8954 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,7 +122,7 @@ jobs: npx --no-install playwright install --with-deps chromium - name: Validate embed source pages - run: python3 scripts/check_embed_sources.py + run: python scripts/check_embed_sources.py - name: Build site and capture Hugo duration id: hugo-build diff --git a/scripts/check_embed_sources.py b/scripts/check_embed_sources.py index 73f8667..de1545c 100644 --- a/scripts/check_embed_sources.py +++ b/scripts/check_embed_sources.py @@ -30,7 +30,8 @@ def _source_page(document: str) -> str | None: def _resolves(content_root: Path, source_page: str) -> bool: relative = source_page.strip("/") - if not relative: + # NFR-012: an embed's source_page must be a data page under content/data/. + if relative != "data" and not relative.startswith("data/"): return False target = content_root / relative return (target / "index.md").is_file() or (target / "_index.md").is_file() @@ -48,7 +49,10 @@ def check_embed_sources(root: Path = ROOT) -> list[str]: problems.append(f"{relative_embed}: missing source_page") continue if not _resolves(content_root, source_page): - problems.append(f"{relative_embed}: source_page does not resolve: {source_page}") + problems.append( + f"{relative_embed}: source_page does not resolve to a content/data page: " + f"{source_page}" + ) return problems diff --git a/tests/test_embed_sources.py b/tests/test_embed_sources.py index b5bd180..95fa0ea 100644 --- a/tests/test_embed_sources.py +++ b/tests/test_embed_sources.py @@ -45,3 +45,16 @@ def test_resolving_source_page_passes() -> None: ) _write(root / "content/data/live/index.md", '+++\ntitle = "Live"\n+++\n') assert check_embed_sources(root) == [] + + +def test_non_data_source_page_is_reported() -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write( + root / "content/embeds/chart/index.md", + '+++\nsource_page = "/weekly/2026/w22/"\n+++\n', + ) + # The referenced page exists, but it is not a content/data page. + _write(root / "content/weekly/2026/w22/index.md", '+++\ntitle = "W22"\n+++\n') + problems = check_embed_sources(root) + assert any("does not resolve to a content/data page" in problem for problem in problems) From a40cc751dadc79342169b3bddb981955ec079469 Mon Sep 17 00:00:00 2001 From: jmservera Date: Sat, 1 Aug 2026 20:10:57 +0200 Subject: [PATCH 3/3] fix: use Path default for --root in embed source check (#641 review) --- scripts/check_embed_sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/check_embed_sources.py b/scripts/check_embed_sources.py index de1545c..6343d3a 100644 --- a/scripts/check_embed_sources.py +++ b/scripts/check_embed_sources.py @@ -59,10 +59,10 @@ def check_embed_sources(root: Path = ROOT) -> list[str]: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", default=str(ROOT), type=Path, help="Repository root") + parser.add_argument("--root", default=ROOT, type=Path, help="Repository root") args = parser.parse_args(argv) - problems = check_embed_sources(args.root) + problems = check_embed_sources(Path(args.root)) for problem in problems: print(f"::error::{problem}", file=sys.stderr) if problems: