feat(security): an origin gate for the app container, shipped off

.github/workflows/bot-serving-check.yml

@@ -15,6 +15,13 @@
 # that broke in the incident above; Cloudflare-edge issues are out of this
 # monitor's reach by design.
 #
+# Skipping the edge is exactly what the app's origin gate refuses
+# (app/origin-gate.conf.template), so every probe below carries the header the
+# edge would have stamped, out of the ORIGIN_SECRET repository secret. This
+# monitor is the reason the gate could not be a Host rule: it cannot spoof the
+# Host either, and anything else it might present as an exception would be
+# public with this repository.
+#
 # What the bot pages are is DERIVED from the repo, never written out here. The
 # routes come from the `@router.get("/seo-proxy/…")` decorators in
 # api/routers/seo.py and the spec title from plots/<spec>/specification.yaml.
@@ -56,14 +63,15 @@ jobs:
     # 36 check() calls (10 derived bot routes + spec page + impl page +
     # ClaudeBot + 15 crawler UAs + 404 + robots + sitemap + 3 llms + 2 human
     # controls) x (--retry 2 -> up to 3 attempts x --max-time 30) can reach
-    # ~54 min worst-case, plus five non-retried probes (llms.txt charset,
+    # ~54 min worst-case, plus the retried /_health gate probe (90s) and five
+    # non-retried probes (llms.txt charset,
     # trailing slash, og-image, .well-known redirect, the /{spec}/{language}
-    # 301 — 30s each); 62 leaves
+    # 301 — 30s each); 64 leaves
     # room to report a clean failure rather than dying to the job timeout,
     # which reports nothing useful. Recompute this when adding checks: the
     # ceiling is check() calls x 90s, plus margin. The route sweep grows with
     # api/routers/seo.py, so a new bot page adds 90s to that ceiling.
-    timeout-minutes: 62
+    timeout-minutes: 64
     steps:
       # The routes and the expected title are read out of the repo, so it has
       # to be here before the first request goes out.
@@ -73,6 +81,19 @@ jobs:
           fetch-depth: 1
 
       - name: Crawler UAs must get 200 + per-route pages
+        env:
+          # The header the Cloudflare Transform Rule stamps on everything it
+          # proxies for anyplot.ai. These checks target the Cloud Run ORIGIN on
+          # purpose (see the file header: Cloudflare 403s runner IPs even for a
+          # UA-spoofed Googlebot), so they skip the edge and have to stamp their
+          # own — the same repository secret sync-postgres.yml sends to the API
+          # gate, and the same value both services are given. It reaches curl
+          # through the environment and this step runs without `set -x`, so it
+          # is never echoed. An exception keyed on anything else — a header this
+          # workflow invents, a user agent — would be public with this
+          # repository, which is why the shared secret is the only exception the
+          # gate can afford.
+          ORIGIN_SECRET: ${{ secrets.ORIGIN_SECRET }}
         run: |
           set -uo pipefail
           # Cloud Run origin of the anyplot-app service (see header comment
@@ -85,12 +106,46 @@ jobs:
           HUMAN="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
           fail=0
 
+          # Absent secret => no header at all. Sending an empty one instead
+          # would read as a mismatch, i.e. as a half-applied rotation rather
+          # than as the missing secret it is.
+          HDR=()
+          if [ -n "${ORIGIN_SECRET}" ]; then HDR=(-H "X-Origin-Secret: ${ORIGIN_SECRET}"); fi
+
+          # Ask the gate before anything else. /_health is its one exempt path,
+          # so it answers whatever the gate is doing, and it reports the verdict
+          # for the request it was asked with — never the value. Without this,
+          # an armed gate plus a missing repository secret would red all ~36
+          # checks below with 403s and open an incident that reads "every
+          # crawler page is broken" — the wrong thing to be woken up for, and
+          # the wrong thing to go looking for.
+          gate=$(curl -sS --retry 2 --max-time 30 -o /dev/null -D - "${HDR[@]}" "$ORIGIN/_health" \
+            | tr -d '\r' | sed -n 's/^[Xx]-[Oo]rigin-[Gg]ate: //p') || gate=""
+          case "$gate" in
+            ok|off-seen)
+              echo "origin gate: $gate — the probes below carry the header" ;;
+            off)
+              echo "::warning::the origin gate is not armed AND this run sent no X-Origin-Secret. Set the ORIGIN_SECRET repository secret to the value the anyplot-app service will be given, before it is armed." ;;
+            missing)
+              echo "::error::the origin gate is armed and this run has no ORIGIN_SECRET repository secret — every check below would be a 403. Set it to the same value as the ORIGIN_SECRET on the anyplot-app Cloud Run service."
+              exit 1 ;;
+            mismatch)
+              echo "::error::the origin gate is armed and the ORIGIN_SECRET repository secret is not the value the anyplot-app service was given — a half-applied rotation. Roll the gate back, finish the rotation, arm again."
+              exit 1 ;;
+            *)
+              # No header at all: the running revision predates the gate. That
+              # is deploy lag, not a serving fault, and every check below still
+              # means exactly what it always meant — so it is said out loud and
+              # the run continues.
+              echo "::warning::$ORIGIN/_health carries no X-Origin-Gate header, so the deployed revision predates the origin gate. Check for a pending deploy." ;;
+          esac
+
           check() {
             local ua="$1" url="$2" expect="$3" want="${4:-200}"
             local code
             # On curl failure REPLACE the code — a failing curl can still have
             # printed a partial -w code; appending would yield e.g. "200000".
-            code=$(curl -sS --retry 2 --max-time 30 -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000"
+            code=$(curl -sS --retry 2 --max-time 30 "${HDR[@]}" -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000"
             if [ "$code" != "$want" ]; then
               echo "::error::$url with UA '$ua' returned HTTP $code (expected $want)"
               fail=1
@@ -250,7 +305,7 @@ jobs:
           # to the same hub would consolidate nothing, and the endpoint documents
           # a permanent redirect.
           read -r lang_code lang_target <<< "$(curl -sS --max-time 30 -o /dev/null \
-            -A "$GOOGLEBOT" -w '%{http_code} %{redirect_url}' "$ORIGIN/$SPEC/python")"
+            "${HDR[@]}" -A "$GOOGLEBOT" -w '%{http_code} %{redirect_url}' "$ORIGIN/$SPEC/python")"
           if [ "$lang_code" != "301" ]; then
             echo "::error::/$SPEC/python answered HTTP $lang_code (expected a permanent 301 onto the hub)"
             fail=1
@@ -312,15 +367,15 @@ jobs:
 
           # The site card must be the FILE for a preview bot, not the proxy —
           # a preview bot that lands on /seo-proxy/og-image.png shows nothing.
-          code=$(curl -sS --max-time 30 -A "$TWITTERBOT" -o /dev/null -w '%{http_code} %{content_type}' "$ORIGIN/og-image.png") || code="000"
+          code=$(curl -sS --max-time 30 "${HDR[@]}" -A "$TWITTERBOT" -o /dev/null -w '%{http_code} %{content_type}' "$ORIGIN/og-image.png") || code="000"
           case "$code" in
             "200 image/png"*) echo "OK: og-image.png served as image to a preview bot" ;;
             *) echo "::error::og-image.png for a preview bot: $code (expected 200 image/png)"; fail=1 ;;
           esac
 
           # A guessed /.well-known/llms.txt must land on the file, not on the
           # SPA shell (which soft-404'd it with 200 until 2026-08-28).
-          wk_target=$(curl -sS --max-time 30 -o /dev/null -A "$CHATGPTUSER" \
+          wk_target=$(curl -sS --max-time 30 -o /dev/null "${HDR[@]}" -A "$CHATGPTUSER" \
             -w '%{redirect_url}' "$ORIGIN/.well-known/llms.txt")
           case "$wk_target" in
             */llms.txt) echo "OK: .well-known/llms.txt -> $wk_target" ;;
@@ -340,7 +395,7 @@ jobs:
 
           # llms.txt carries UTF-8 punctuation (em dashes, arrows); without an
           # explicit charset a strict client decodes it as Latin-1 mojibake.
-          ct=$(curl -sS --max-time 30 -A "$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$ORIGIN/llms.txt")
+          ct=$(curl -sS --max-time 30 "${HDR[@]}" -A "$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$ORIGIN/llms.txt")
           case "$ct" in
             *charset=utf-8*) echo "OK: llms.txt content-type: $ct" ;;
             *)
@@ -351,7 +406,7 @@ jobs:
           # A trailing slash must normalise to the canonical URL on THIS host.
           # It used to 307 to http://api.anyplot.ai/seo-proxy/... — internal
           # path, wrong host, plain http, and that host disallows all crawling.
-          slash_target=$(curl -sS --max-time 30 -o /dev/null -A "$GOOGLEBOT" \
+          slash_target=$(curl -sS --max-time 30 -o /dev/null "${HDR[@]}" -A "$GOOGLEBOT" \
             -w '%{redirect_url}' "$ORIGIN/$SPEC/")
           case "$slash_target" in
             "")