Merge pull request #10 from IONIS-AI/feat/recaps-into-serving-layer

feat: contest recaps come from PostgreSQL — retire the SQLite render dependency

Author: KI7MT

import_recaps.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""One-time import of contest recap datasets into the PostgreSQL serving layer.
+
+    import_recaps.py --contest-dir /var/tmp/contests          # import every recap in recaps.yaml
+    import_recaps.py --contest-dir /var/tmp/contests --dry-run
+
+WHY THIS IS A ONE-TIME IMPORT AND NOT A REFRESH JOB.
+
+Contest recaps are static. ARRL DX CW and SSB ran in March 2026; the contests are over, the
+signatures are frozen, and no amount of re-reading will change a number. There is nothing to
+keep fresh — so this is not part of refresh.py's cadence groups. It runs once per contest,
+when that contest's dataset is built, and never again.
+
+WHY IT READS SQLITE RATHER THAN REBUILDING FROM CLICKHOUSE.
+
+The datasets are built from ClickHouse by ionis-devel/contests/export_contest_sqlite.py, so
+rebuilding looks tempting. It is not safe: that script's current PSKR query hardcodes
+avg_distance = 0, while the published recap pages show real distances (160m: 3,476 km). The
+files on disk do not match what the current script would produce, and reproducing them means
+resolving that provenance question first. Importing the artifact that actually rendered the
+published pages has no such risk.
+
+WHAT THIS ENDS.
+
+publish.py read 1.3 GB of SQLite at render time, off a filesystem that existed only on the
+9975. That coupling is what broke the contest pages when publishing moved to publish-1 — the
+loader warned, returned None, and the run still reported "2 recap(s) loaded" and exited 0.
+After this import nothing reads SQLite at render time, on any host.
+"""
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import sqlite3
+import sys
+from pathlib import Path
+
+import psycopg
+import yaml
+
+ROOT = Path(os.environ.get("HAMSTATS_ROOT") or Path(__file__).parent)
+DB_FILE = os.environ.get("HAMSTATS_DB_FILE", "/etc/hamstats/db-rw.dsn")
+
+sys.path.insert(0, str(ROOT))
+from publish import load_recap_data_sqlite  # noqa: E402
+
+# Static data does not expire. The serving layer's staleness check compares refreshed_at
+# against max_age, and a recap that is a year old is exactly as correct as one imported today.
+STATIC = "100 years"
+
+DATASETS = ("band_summary", "hourly_activity", "solar_timeline", "distance_stats")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--contest-dir", required=True,
+                    help="directory holding the recap .sqlite files")
+    ap.add_argument("--dry-run", action="store_true", help="read and report, write nothing")
+    args = ap.parse_args()
+
+    os.environ["CONTEST_DATA_DIR"] = args.contest_dir
+    import publish
+    publish.CONTEST_DATA_DIR = Path(args.contest_dir)
+
+    recaps = yaml.safe_load((ROOT / "data" / "recaps.yaml").read_text()) or []
+    print(f"Importing {len(recaps)} recap(s) from {args.contest_dir}")
+
+    staged, missing = [], []
+    for recap in recaps:
+        slug = recap.get("slug")
+        # load_recap_data is publish.py's OWN loader, unchanged. Whatever it produced when
+        # rendering from SQLite is exactly what lands in PostgreSQL — the import cannot drift
+        # from the thing it is replacing, because it is the same code.
+        data = load_recap_data_sqlite(recap)
+        if data is None:
+            print(f"  MISSING  {slug}: no dataset at {args.contest_dir}/{recap.get('dataset')}")
+            missing.append(slug)
+            continue
+        for key in DATASETS:
+            rows = data.get(key) or []
+            staged.append((f"recap:{slug}:{key}", rows))
+            print(f"  {slug:22} {key:18} {len(rows):>5} rows")
+
+    if missing:
+        # Importing a partial set would leave some recaps rendering empty with no indication
+        # that anything is wrong -- the exact failure this import exists to end.
+        print(f"ERROR: {len(missing)} recap(s) had no dataset: {', '.join(missing)}. "
+              f"Nothing written.", file=sys.stderr)
+        return 1
+
+    if args.dry_run:
+        print("Dry run — nothing written.")
+        return 0
+
+    dsn = Path(DB_FILE).read_text().strip()
+    with psycopg.connect(dsn, autocommit=False) as conn:
+        conn.execute((ROOT / "sql" / "serving_schema.sql").read_text())
+        for name, rows in staged:
+            conn.execute(
+                """
+                INSERT INTO serving.query_results
+                    (name, payload, row_count, refreshed_at, max_age,
+                     source_rows_read, source_elapsed_ms)
+                VALUES (%s, %s::jsonb, %s, now(), %s::interval, NULL, NULL)
+                ON CONFLICT (name) DO UPDATE SET
+                    payload      = EXCLUDED.payload,
+                    row_count    = EXCLUDED.row_count,
+                    refreshed_at = EXCLUDED.refreshed_at,
+                    max_age      = EXCLUDED.max_age
+                """,
+                (name, json.dumps(rows, default=_json_default), len(rows), STATIC),
+            )
+        conn.commit()
+    print(f"Wrote {len(staged)} recap dataset(s) to the serving layer.")
+    return 0
+
+
+def _json_default(o):
+    if isinstance(o, (dt.datetime, dt.date)):
+        return o.isoformat()
+    raise TypeError(f"{type(o).__name__} is not JSON-serialisable: {o!r}")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ionis-hamstats.spec

@@ -1,7 +1,7 @@
 %global debug_package %{nil}
 
 Name:           ionis-hamstats
-Version:        1.1.2
+Version:        1.2.0
 Release:        1%{?dist}
 Summary:        Ham Stats publishing pipeline — ClickHouse aggregates to a static site
 
@@ -108,6 +108,18 @@ fi
 %dir %{_sysconfdir}/hamstats
 
 %changelog
+* Wed Sep 09 2026 Greg Beam <ki7mt@yahoo.com> - 1.2.0-1
+- Contest recaps come from the PostgreSQL serving layer. publish.py aggregated 1.3 GB of
+  SQLite at render time off a filesystem that existed only on the 9975; moving the publisher
+  to publish-1 emptied both contest pages, and the run still reported "2 recap(s) loaded" and
+  exited 0 because that count counted recap DEFINITIONS, not loaded data.
+- import_recaps.py imports them once. Recaps are static -- the contests are over -- so this is
+  not a refresh cadence, it is a one-time import per contest, reusing publish.py's own SQLite
+  loader so the imported data cannot drift from what it replaces.
+- A recap with no data in the serving layer now fails the run instead of rendering a page
+  without its band tables.
+- Retires the last filesystem dependency: nothing reads SQLite at render time on any host.
+
 * Wed Sep 09 2026 Greg Beam <ki7mt@yahoo.com> - 1.1.2-1
 - Non-finite floats become null. The weekly refresh read 9.4 billion rows and then failed to
   write them: storm_snr_comparison.after_snr is null in ClickHouse for a storm too recent to

publish.py

@@ -225,6 +225,30 @@ def run_all_queries(client) -> dict:
     return data
 
 
+DATASETS = ("band_summary", "hourly_activity", "solar_timeline", "distance_stats")
+
+
+def load_recap_from_serving(recap: dict, data: dict) -> dict | None:
+    """Read a recap's pre-computed datasets out of the serving layer.
+
+    Recaps used to be aggregated from 1.3 GB of SQLite at render time, off a filesystem that
+    existed only on the 9975. That coupling broke every contest page the moment publishing
+    moved hosts: the loader warned, returned None, and the run still reported "2 recap(s)
+    loaded" and exited 0, so empty pages reached the site unnoticed.
+
+    They are static -- the contests are over -- so they are imported once and served forever.
+    Returning None here now means genuinely absent, and the caller treats it as a failure
+    rather than as "static findings only".
+    """
+    got = {}
+    for key in DATASETS:
+        rows = data.get(f"recap:{recap['slug']}:{key}")
+        if rows is None:
+            return None
+        got[key] = rows
+    return got
+
+
 # ---------------------------------------------------------------------------
 # IONIS V22-gamma + PhysicsOverrideLayer Predictions
 # ---------------------------------------------------------------------------
@@ -387,7 +411,13 @@ def sqlite_ro_uri(db_path: Path) -> str:
     return f"file:{path}?mode=ro&immutable=1"
 
 
-def load_recap_data(recap: dict) -> dict | None:
+def load_recap_data_sqlite(recap: dict) -> dict | None:
+    """Aggregate a recap from its SQLite dataset. USED ONLY BY import_recaps.py.
+
+    This is no longer on the render path. publish.py reads recaps from the serving layer;
+    this function exists so the one-time import produces exactly what rendering from SQLite
+    produced, using the same code rather than a reimplementation of it.
+    """
     """Load aggregated stats from a contest SQLite file.
 
     Returns dict with band_summary, hourly_activity, solar_timeline,
@@ -1002,18 +1032,24 @@ def main():
     if dx_predictions:
         print(f"  Generated predictions for {len(dx_predictions)} DXpeditions")
 
-    # 5. Contest recaps (from SQLite datasets)
+    # 5. Contest recaps — pre-imported into the serving layer, not read from SQLite
     print("Loading contest recaps...")
     recap_defs = load_recaps()
     recaps = []
     for recap in recap_defs:
-        recap_data = load_recap_data(recap)
+        recap_data = load_recap_from_serving(recap, data)
         if recap_data:
             print(f"  {recap['slug']}: {len(recap_data.get('band_summary', []))} bands")
         else:
-            print(f"  {recap['slug']}: no dataset (static findings only)")
+            # NOT "static findings only". That wording made an absent dataset look like a
+            # deliberate mode, and the count below counted DEFINITIONS regardless — so two
+            # empty contest pages published cleanly and nothing said otherwise.
+            print(f"  {recap['slug']}: NOT IMPORTED — page will render without band data",
+                  file=sys.stderr)
+            stale.append(f"recap {recap['slug']} has no data in the serving layer "
+                         f"(run import_recaps.py for it)")
         recaps.append((recap, recap_data))
-    print(f"  {len(recaps)} recap(s) loaded")
+    print(f"  {sum(1 for _, d in recaps if d)} of {len(recaps)} recap(s) have data")
 
     context = build_context(
         data, predictions, now,

test_serving.py

@@ -124,6 +124,34 @@ def __repr__(self): return "<weird>"
 check("the schema forbids a row_count that disagrees with the payload",
       "row_count = jsonb_array_length(payload)" in Path("sql/serving_schema.sql").read_text(), True)
 
+print("== contest recaps come from the serving layer, not the filesystem ==")
+# publish.py aggregated 1.3 GB of SQLite at render time, off a filesystem that existed only on
+# the 9975. Moving the publisher to publish-1 emptied both contest pages -- and the run still
+# printed "2 recap(s) loaded" and exited 0, because that count counted DEFINITIONS. The pages
+# were live and wrong for 40 minutes.
+pcode = Path("publish.py").read_text()
+check("the render path no longer opens SQLite",
+      "load_recap_data_sqlite(recap)" in code, False)
+check("the SQLite loader survives for the one-time import",
+      "def load_recap_data_sqlite" in pcode, True)
+check("rendering reads the serving layer",
+      "load_recap_from_serving(recap, data)" in code, True)
+# `code` is comment-stripped: the comment explaining the removal names the old wording on
+# purpose, and a check that forbids describing the bug punishes the fix.
+# The exact former print, not the phrase: the docstring and the comment both name the old
+# wording deliberately, and `code` strips comments but not docstrings.
+check("a missing recap is reported as missing, not as a mode",
+      "no dataset (static findings only)" in code, False)
+check("and it fails the run", "stale.append(f\"recap {recap['slug']}" in pcode, True)
+check("the count reports recaps WITH DATA, not definitions",
+      "sum(1 for _, d in recaps if d)" in pcode, True)
+
+rcode = Path("import_recaps.py").read_text()
+check("the import reuses publish's own loader rather than reimplementing it",
+      "from publish import load_recap_data_sqlite" in rcode, True)
+check("a partial import writes nothing",
+      "Nothing written." in rcode, True)
+
 print()
 if failures:
     print(f"  {len(failures)} FAILED")