Bot Serving Check #60
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Synthetic monitor for the nginx bot -> seo-proxy path. | |
| # | |
| # Humans get the SPA shell and never notice when the bot hop breaks: the site | |
| # looks healthy, Plausible shows normal traffic, CI is green — and every | |
| # crawler sees an error page. Exactly that happened between 2026-06-12 and | |
| # 2026-07-09: the @seo_proxy upstream TLS verification failed (default | |
| # proxy_ssl_verify_depth 1 vs a 4-deep Let's Encrypt chain) and every bot UA | |
| # received HTTP 502 on every page, undetected for ~4 weeks. This check is the | |
| # alarm that was missing. | |
| # | |
| # The checks target the Cloud Run ORIGIN, not https://anyplot.ai: Cloudflare's | |
| # bot management 403s GitHub-runner (datacenter) IPs — including UA-spoofed | |
| # "Googlebot", which only passes verified-bot checks from real Google IPs — | |
| # verified on the first dispatched run. The origin is also exactly the layer | |
| # that broke in the incident above; Cloudflare-edge issues are out of this | |
| # monitor's reach by design. | |
| name: Bot Serving Check | |
| on: | |
| schedule: | |
| - cron: "23 6 * * *" # daily 06:23 UTC | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| bot-serving: | |
| runs-on: ubuntu-latest | |
| # 27 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can | |
| # reach ~41 min worst-case, plus four non-retried probes (llms.txt charset, | |
| # trailing slash, og-image, .well-known redirect — 30s each); 46 leaves | |
| # room to report a clean failure rather than dying to the job timeout, | |
| # which reports nothing useful. Recompute this when adding checks: the | |
| # ceiling is check() calls x 90s, plus margin. | |
| timeout-minutes: 46 | |
| steps: | |
| - name: Crawler UAs must get 200 + per-route titles | |
| run: | | |
| set -uo pipefail | |
| # Cloud Run origin of the anyplot-app service (see header comment | |
| # for why not https://anyplot.ai). | |
| ORIGIN="https://anyplot-app-r3tvmejsmq-ez.a.run.app" | |
| GOOGLEBOT="Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" | |
| TWITTERBOT="Twitterbot/1.0" | |
| CLAUDEBOT="Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)" | |
| CHATGPTUSER="Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)" | |
| HUMAN="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" | |
| fail=0 | |
| check() { | |
| local ua="$1" url="$2" expect="$3" want="${4:-200}" | |
| local code | |
| # On curl failure REPLACE the code — a failing curl can still have | |
| # printed a partial -w code; appending would yield e.g. "200000". | |
| code=$(curl -sS --retry 2 --max-time 30 -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000" | |
| if [ "$code" != "$want" ]; then | |
| echo "::error::$url with UA '$ua' returned HTTP $code (expected $want)" | |
| fail=1 | |
| elif ! grep -qF "$expect" body.html; then | |
| echo "::error::$url with UA '$ua' returned $want but the body is missing: $expect" | |
| fail=1 | |
| else | |
| echo "OK: $url ($ua)" | |
| fi | |
| } | |
| # Bot path: prerendered per-route HTML from the seo-proxy | |
| # Prefix, not the full title: the copy changed to "anyplot.ai — | |
| # AI-generated plot catalog for 15 libraries" and this check went red | |
| # for ten consecutive days without anyone noticing. The prefix still | |
| # separates the prerendered page from the SPA shell, whose title is | |
| # "any.plot() — any library.", which is the property under test. | |
| check "$GOOGLEBOT" "$ORIGIN/" "<title>anyplot.ai" | |
| check "$GOOGLEBOT" "$ORIGIN/scatter-basic" "<title>Basic Scatter Plot | anyplot.ai</title>" | |
| check "$TWITTERBOT" "$ORIGIN/scatter-basic/python/matplotlib" "<title>Basic Scatter Plot - Matplotlib | anyplot.ai</title>" | |
| # AI assistants take the same prerendered path (nginx $is_bot). These | |
| # checks hit the ORIGIN, so they verify the nginx map independently of | |
| # whether Cloudflare's AI Crawl Control currently 403s these UAs at | |
| # the edge — an edge-level policy change needs no change here. | |
| check "$CLAUDEBOT" "$ORIGIN/scatter-basic" "<title>Basic Scatter Plot | anyplot.ai</title>" | |
| # User-directed fetchers: a human asked their assistant to open the | |
| # page. All of these were verified receiving the empty SPA shell on | |
| # 2026-08-18 — an assistant asked about a plot could describe nothing. | |
| # Google documents its own as generally ignoring robots.txt, so the | |
| # nginx map is the only control point and the only thing this guards. | |
| for ua in \ | |
| "Mozilla/5.0 (compatible; Google-GeminiNotebook)" \ | |
| "Mozilla/5.0 (compatible; Google-NotebookLM)" \ | |
| "Mozilla/5.0 (compatible; Gemini-Deep-Research)" \ | |
| "Mozilla/5.0 (compatible; GoogleAgent-Mariner)" \ | |
| "meta-externalfetcher/1.1" \ | |
| "Mozilla/5.0 (compatible; Meta-WebIndexer/1.0)" \ | |
| "Mozilla/5.0 (compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)" \ | |
| "Mozilla/5.0 (compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)" \ | |
| "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)" \ | |
| "Mozilla/5.0 (compatible; Amzn-SearchBot/1.0)" \ | |
| "Mozilla/5.0 (compatible; Amzn-User/1.0)" \ | |
| "Mozilla/5.0 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" \ | |
| "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)" \ | |
| "Grok/1.0" \ | |
| "Mozilla/5.0 (compatible; xAI-Bot/1.0)" | |
| do | |
| check "$ua" "$ORIGIN/scatter-basic" "<title>Basic Scatter Plot | anyplot.ai</title>" | |
| done | |
| # A crawler asking for a URL that is no page gets a real 404 from the | |
| # seo-proxy — the SPA shell would answer 200 (soft-404), and did for | |
| # 161 stale migration URLs before the proxy learned to say no. | |
| check "$GOOGLEBOT" "$ORIGIN/this-spec-does-not-exist" '"status":404' 404 | |
| # The machine files must be served directly, never proxied to the | |
| # seo backend — including for a mapped crawler UA, which is the whole | |
| # point of the `location =` bypasses. robots.txt and sitemap.xml | |
| # join llms.txt below: the sitemap is proxied to the API for every | |
| # client, and a broken proxy would hand a crawler the SPA shell. | |
| check "$GOOGLEBOT" "$ORIGIN/robots.txt" "User-agent: Bytespider" | |
| check "$GOOGLEBOT" "$ORIGIN/sitemap.xml" "<urlset" | |
| # The site card must be the FILE for a preview bot, not the proxy — | |
| # a preview bot that lands on /seo-proxy/og-image.png shows nothing. | |
| code=$(curl -sS --max-time 30 -A "$TWITTERBOT" -o /dev/null -w '%{http_code} %{content_type}' "$ORIGIN/og-image.png") || code="000" | |
| case "$code" in | |
| "200 image/png"*) echo "OK: og-image.png served as image to a preview bot" ;; | |
| *) echo "::error::og-image.png for a preview bot: $code (expected 200 image/png)"; fail=1 ;; | |
| esac | |
| # A guessed /.well-known/llms.txt must land on the file, not on the | |
| # SPA shell (which soft-404'd it with 200 until 2026-08-28). | |
| wk_target=$(curl -sS --max-time 30 -o /dev/null -A "$CHATGPTUSER" \ | |
| -w '%{redirect_url}' "$ORIGIN/.well-known/llms.txt") | |
| case "$wk_target" in | |
| */llms.txt) echo "OK: .well-known/llms.txt -> $wk_target" ;; | |
| *) echo "::error::.well-known/llms.txt did not redirect to the guide: '$wk_target'"; fail=1 ;; | |
| esac | |
| # llms.txt must be served directly, never proxied to the seo backend — | |
| # including for a mapped crawler UA, which is the whole point of the | |
| # `location = /llms.txt` bypass. | |
| check "$GOOGLEBOT" "$ORIGIN/llms.txt" "# anyplot" | |
| check "$CHATGPTUSER" "$ORIGIN/llms.txt" "# anyplot" | |
| # llms-full.txt is proxied to the API for EVERY client (mapped or | |
| # not) — before, the SPA catch-all soft-404'd it with the homepage | |
| # shell. The catalogue-index line proves the API generated it. | |
| check "$HUMAN" "$ORIGIN/llms-full.txt" "# anyplot — full catalogue index" | |
| # llms.txt carries UTF-8 punctuation (em dashes, arrows); without an | |
| # explicit charset a strict client decodes it as Latin-1 mojibake. | |
| ct=$(curl -sS --max-time 30 -A "$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$ORIGIN/llms.txt") | |
| case "$ct" in | |
| *charset=utf-8*) echo "OK: llms.txt content-type: $ct" ;; | |
| *) | |
| echo "::error::llms.txt served without utf-8 charset: $ct" | |
| fail=1 ;; | |
| esac | |
| # A trailing slash must normalise to the canonical URL on THIS host. | |
| # It used to 307 to http://api.anyplot.ai/seo-proxy/... — internal | |
| # path, wrong host, plain http, and that host disallows all crawling. | |
| slash_target=$(curl -sS --max-time 30 -o /dev/null -A "$GOOGLEBOT" \ | |
| -w '%{redirect_url}' "$ORIGIN/scatter-basic/") | |
| case "$slash_target" in | |
| "") | |
| # No redirect at all: %{redirect_url} is empty, which the previous | |
| # form printed as "OK: trailing slash -> " and passed. Copilot | |
| # raised this twice; it means the rewrite has disappeared. | |
| echo "::error::trailing slash produced no redirect — the rewrite is gone" | |
| fail=1 ;; | |
| *"/seo-proxy"*|http://*|*:8080/*) | |
| echo "::error::trailing-slash redirect leaks or downgrades: $slash_target" | |
| fail=1 ;; | |
| *) echo "OK: trailing slash -> $slash_target" ;; | |
| esac | |
| # Control: humans must still get the SPA shell — on the home page | |
| # and on a deep route. | |
| check "$HUMAN" "$ORIGIN/" '<div id="root">' | |
| check "$HUMAN" "$ORIGIN/scatter-basic" '<div id="root">' | |
| exit $fail |