Bot Serving Check #54
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Synthetic monitor for the nginx bot -> seo-proxy path. | |
| # | |
| # Humans get the SPA shell and never notice when the bot hop breaks: the site | |
| # looks healthy, Plausible shows normal traffic, CI is green — and every | |
| # crawler sees an error page. Exactly that happened between 2026-06-12 and | |
| # 2026-07-09: the @seo_proxy upstream TLS verification failed (default | |
| # proxy_ssl_verify_depth 1 vs a 4-deep Let's Encrypt chain) and every bot UA | |
| # received HTTP 502 on every page, undetected for ~4 weeks. This check is the | |
| # alarm that was missing. | |
| # | |
| # The checks target the Cloud Run ORIGIN, not https://anyplot.ai: Cloudflare's | |
| # bot management 403s GitHub-runner (datacenter) IPs — including UA-spoofed | |
| # "Googlebot", which only passes verified-bot checks from real Google IPs — | |
| # verified on the first dispatched run. The origin is also exactly the layer | |
| # that broke in the incident above; Cloudflare-edge issues are out of this | |
| # monitor's reach by design. | |
| name: Bot Serving Check | |
| on: | |
| schedule: | |
| - cron: "23 6 * * *" # daily 06:23 UTC | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| bot-serving: | |
| runs-on: ubuntu-latest | |
| # 22 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can | |
| # reach ~33 min worst-case, plus two non-retried probes (llms.txt charset, | |
| # trailing slash — 30s each); 38 leaves room to report a clean failure | |
| # rather than dying to the job timeout, which reports nothing useful. | |
| # Recompute this when adding checks: the ceiling is check() calls x 90s, | |
| # plus margin. | |
| timeout-minutes: 38 | |
| steps: | |
| - name: Crawler UAs must get 200 + per-route titles | |
| run: | | |
| set -uo pipefail | |
| # Cloud Run origin of the anyplot-app service (see header comment | |
| # for why not https://anyplot.ai). | |
| ORIGIN="https://anyplot-app-r3tvmejsmq-ez.a.run.app" | |
| GOOGLEBOT="Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" | |
| TWITTERBOT="Twitterbot/1.0" | |
| CLAUDEBOT="Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)" | |
| CHATGPTUSER="Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)" | |
| HUMAN="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" | |
| fail=0 | |
| check() { | |
| local ua="$1" url="$2" expect="$3" | |
| local code | |
| # On curl failure REPLACE the code — a failing curl can still have | |
| # printed a partial -w code; appending would yield e.g. "200000". | |
| code=$(curl -sS --retry 2 --max-time 30 -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000" | |
| if [ "$code" != "200" ]; then | |
| echo "::error::$url with UA '$ua' returned HTTP $code (expected 200)" | |
| fail=1 | |
| elif ! grep -qF "$expect" body.html; then | |
| echo "::error::$url with UA '$ua' returned 200 but the body is missing: $expect" | |
| fail=1 | |
| else | |
| echo "OK: $url ($ua)" | |
| fi | |
| } | |
| # Bot path: prerendered per-route HTML from the seo-proxy | |
| # Prefix, not the full title: the copy changed to "anyplot.ai — | |
| # AI-generated plot catalog for 15 libraries" and this check went red | |
| # for ten consecutive days without anyone noticing. The prefix still | |
| # separates the prerendered page from the SPA shell, whose title is | |
| # "any.plot() — any library.", which is the property under test. | |
| check "$GOOGLEBOT" "$ORIGIN/" "<title>anyplot.ai" | |
| check "$GOOGLEBOT" "$ORIGIN/scatter-basic" "<title>Basic Scatter Plot | anyplot.ai</title>" | |
| check "$TWITTERBOT" "$ORIGIN/scatter-basic/python/matplotlib" "<title>Basic Scatter Plot - Matplotlib | anyplot.ai</title>" | |
| # AI assistants take the same prerendered path (nginx $is_bot). These | |
| # checks hit the ORIGIN, so they verify the nginx map independently of | |
| # whether Cloudflare's AI Crawl Control currently 403s these UAs at | |
| # the edge — an edge-level policy change needs no change here. | |
| check "$CLAUDEBOT" "$ORIGIN/scatter-basic" "<title>Basic Scatter Plot | anyplot.ai</title>" | |
| # User-directed fetchers: a human asked their assistant to open the | |
| # page. All of these were verified receiving the empty SPA shell on | |
| # 2026-08-18 — an assistant asked about a plot could describe nothing. | |
| # Google documents its own as generally ignoring robots.txt, so the | |
| # nginx map is the only control point and the only thing this guards. | |
| for ua in \ | |
| "Mozilla/5.0 (compatible; Google-GeminiNotebook)" \ | |
| "Mozilla/5.0 (compatible; Google-NotebookLM)" \ | |
| "Mozilla/5.0 (compatible; Gemini-Deep-Research)" \ | |
| "Mozilla/5.0 (compatible; GoogleAgent-Mariner)" \ | |
| "meta-externalfetcher/1.1" \ | |
| "Mozilla/5.0 (compatible; Meta-WebIndexer/1.0)" \ | |
| "Mozilla/5.0 (compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)" \ | |
| "Mozilla/5.0 (compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)" \ | |
| "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)" \ | |
| "Mozilla/5.0 (compatible; Amzn-SearchBot/1.0)" \ | |
| "Mozilla/5.0 (compatible; Amzn-User/1.0)" \ | |
| "Mozilla/5.0 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" \ | |
| "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)" \ | |
| "Grok/1.0" | |
| do | |
| check "$ua" "$ORIGIN/scatter-basic" "<title>Basic Scatter Plot | anyplot.ai</title>" | |
| done | |
| # llms.txt must be served directly, never proxied to the seo backend — | |
| # including for a mapped crawler UA, which is the whole point of the | |
| # `location = /llms.txt` bypass. | |
| check "$GOOGLEBOT" "$ORIGIN/llms.txt" "# anyplot" | |
| check "$CHATGPTUSER" "$ORIGIN/llms.txt" "# anyplot" | |
| # llms-full.txt is proxied to the API for EVERY client (mapped or | |
| # not) — before, the SPA catch-all soft-404'd it with the homepage | |
| # shell. The catalogue-index line proves the API generated it. | |
| check "$HUMAN" "$ORIGIN/llms-full.txt" "# anyplot — full catalogue index" | |
| # llms.txt carries UTF-8 punctuation (em dashes, arrows); without an | |
| # explicit charset a strict client decodes it as Latin-1 mojibake. | |
| ct=$(curl -sS --max-time 30 -A "$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$ORIGIN/llms.txt") | |
| case "$ct" in | |
| *charset=utf-8*) echo "OK: llms.txt content-type: $ct" ;; | |
| *) | |
| echo "::error::llms.txt served without utf-8 charset: $ct" | |
| fail=1 ;; | |
| esac | |
| # A trailing slash must normalise to the canonical URL on THIS host. | |
| # It used to 307 to http://api.anyplot.ai/seo-proxy/... — internal | |
| # path, wrong host, plain http, and that host disallows all crawling. | |
| slash_target=$(curl -sS --max-time 30 -o /dev/null -A "$GOOGLEBOT" \ | |
| -w '%{redirect_url}' "$ORIGIN/scatter-basic/") | |
| case "$slash_target" in | |
| "") | |
| # No redirect at all: %{redirect_url} is empty, which the previous | |
| # form printed as "OK: trailing slash -> " and passed. Copilot | |
| # raised this twice; it means the rewrite has disappeared. | |
| echo "::error::trailing slash produced no redirect — the rewrite is gone" | |
| fail=1 ;; | |
| *"/seo-proxy"*|http://*|*:8080/*) | |
| echo "::error::trailing-slash redirect leaks or downgrades: $slash_target" | |
| fail=1 ;; | |
| *) echo "OK: trailing slash -> $slash_target" ;; | |
| esac | |
| # Control: humans must still get the SPA shell | |
| check "$HUMAN" "$ORIGIN/" '<div id="root">' | |
| exit $fail |