IndexNow: changed URLs #336
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "SEO: IndexNow submit" | |
| run-name: "IndexNow: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'changed' }} URLs" | |
| # Tells the IndexNow engines (Bing, Yandex, Seznam, Naver, Yep) which pages | |
| # changed, instead of waiting for their next crawl. Google does not take part; | |
| # it keeps reading the sitemap. | |
| # | |
| # On every push to main that touches plots/, the diff is mapped to page URLs: | |
| # plots/<spec>/specification.* → https://anyplot.ai/<spec> | |
| # plots/<spec>/metadata/<lang>/<lib>.yaml → https://anyplot.ai/<spec>/<lang>/<lib> | |
| # plots/<spec>/implementations/<lang>/<lib>.<ext> → same page | |
| # A deleted implementation is submitted too — IndexNow is "this URL changed", | |
| # which covers removals. | |
| # | |
| # The key is public by design: it only proves the submitter controls the host, | |
| # and the engines verify it by fetching https://anyplot.ai/<key>.txt. The same | |
| # key appears in three places — app/public/<key>.txt (the file itself), the | |
| # exact-match `location` in app/nginx.conf (so a crawler UA is not proxied to | |
| # /seo-proxy and a 404), and INDEXNOW_KEY below. Keep all three in sync when | |
| # rotating it. | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'plots/**' | |
| # A change to this workflow exercises itself with the full list; a | |
| # fix that never runs until the next plot merge is a fix nobody saw. | |
| - '.github/workflows/indexnow-submit.yml' | |
| workflow_dispatch: | |
| inputs: | |
| scope: | |
| description: "'changed' submits the URLs touched by the latest commit on main; 'sitemap' submits every URL in the live sitemap (initial load, or after a long outage)" | |
| required: false | |
| default: changed | |
| type: choice | |
| options: [changed, sitemap] | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: indexnow-submit | |
| cancel-in-progress: false | |
| env: | |
| HOST: anyplot.ai | |
| INDEXNOW_KEY: anyplot-indexnow-ab738f04ea92446a | |
| SCOPE: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'changed' }} | |
| jobs: | |
| submit: | |
| runs-on: ubuntu-latest | |
| # Worst case: ~12 min waiting for the key file (16 probes, 15 s timeout, | |
| # 30 s apart) + the 10 min verification deadline + one last request with | |
| # its retries (up to ~4 min), with room to spare. | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| # HEAD~1 is the previous main for a squash merge, which is how every | |
| # pipeline PR lands; a rare multi-commit push submits its last commit. | |
| fetch-depth: 2 | |
| - name: Collect URLs | |
| id: urls | |
| env: | |
| # Number of commits in the push; a squash merge is exactly one. | |
| PUSH_COMMITS: ${{ github.event_name == 'push' && toJSON(github.event.commits) || '[]' }} | |
| run: | | |
| set -euo pipefail | |
| # The full list comes from the checkout, not from the live sitemap: | |
| # every spec directory is a hub page and every metadata file an | |
| # implementation page (the same rule the sitemap follows), plus the | |
| # static pages. No fetch means no Cloudflare edge or bot management | |
| # between a GitHub runner and the list, and the shallow checkout | |
| # still carries the complete tree. | |
| full_list() { | |
| for p in / /plots /specs /libraries /map /palette /about /mcp /legal /stats; do | |
| echo "https://${HOST}${p}" | |
| done | |
| git ls-tree -d --name-only HEAD plots/ | awk -F/ '{ print "https://'"$HOST"'/" $2 }' | |
| git ls-tree -r --name-only HEAD plots/ | awk -F/ ' | |
| $3 == "metadata" && NF == 5 { lib = $5; sub(/\.[^.]+$/, "", lib); print "https://'"$HOST"'/" $2 "/" $4 "/" lib }' | |
| } | |
| commits=$(jq 'length' <<<"$PUSH_COMMITS") | |
| if [ "$SCOPE" = "sitemap" ]; then | |
| full_list | sort -u > urls.txt | |
| elif [ "$commits" -eq 1 ] && git diff --name-only HEAD~1 HEAD -- .github/workflows/indexnow-submit.yml | grep -q .; then | |
| # The workflow itself changed (the push.paths entry above): run | |
| # the real thing end to end rather than an empty diff. | |
| echo "::notice::workflow file changed; submitting the full list" | |
| full_list | sort -u > urls.txt | |
| elif [ "$commits" -gt 1 ]; then | |
| # A multi-commit push (rare on main) is not what fetch-depth 2 can | |
| # diff; submitting everything is cheap and always correct. | |
| echo "::notice::push carries ${commits} commits; submitting the full list instead of a diff" | |
| full_list | sort -u > urls.txt | |
| else | |
| # A changed specification touches every page of that spec: the | |
| # implementation pages render the spec's title and description | |
| # too. A changed implementation or metadata file touches its own | |
| # page and the hub that lists it. | |
| impl_pages() { | |
| git ls-tree -r --name-only HEAD "plots/$1/metadata/" | awk -F/ ' | |
| NF == 5 { lib = $5; sub(/\.[^.]+$/, "", lib); print "https://'"$HOST"'/" $2 "/" $4 "/" lib }' | |
| } | |
| git diff --name-only HEAD~1 HEAD -- plots/ | while IFS=/ read -r top spec third fourth fifth rest; do | |
| [ "$top" = plots ] && [ -n "$spec" ] && [ -n "$third" ] || continue | |
| case "$third" in | |
| specification.*) | |
| echo "https://${HOST}/${spec}" | |
| impl_pages "$spec" ;; | |
| metadata|implementations) | |
| [ -n "$fifth" ] && [ -z "$rest" ] || continue | |
| echo "https://${HOST}/${spec}" | |
| echo "https://${HOST}/${spec}/${fourth}/${fifth%.*}" ;; | |
| esac | |
| done | sort -u > urls.txt | |
| fi | |
| n=$(wc -l < urls.txt) | |
| echo "count=$n" >> "$GITHUB_OUTPUT" | |
| echo "::notice::${n} URL(s) to submit (scope: ${SCOPE})" | |
| head -20 urls.txt | |
| - name: Submit to IndexNow | |
| if: steps.urls.outputs.count != '0' | |
| run: | | |
| set -euo pipefail | |
| # The engines validate a submission by fetching the key file. It is | |
| # served by the app deploy, which a rollout or a key rotation may | |
| # still have in flight — wait for it (up to ~8 min), but submit | |
| # either way: this probe is a courtesy, IndexNow's own key fetch | |
| # below is the authoritative check. | |
| for i in $(seq 1 16); do | |
| # On a transport failure curl still prints 000 for %{http_code}; | |
| # the fallback is assigned afterwards so nothing is appended. | |
| status=$(curl -sS --max-time 15 -o /dev/null -w '%{http_code}' \ | |
| "https://${HOST}/${INDEXNOW_KEY}.txt") || status=000 | |
| case "$status" in | |
| 200) echo "::notice::key file reachable"; break ;; | |
| # Cloudflare's bot management answers 403 before the origin is | |
| # asked, so this says nothing about the deploy either way, and | |
| # waiting cannot change it. Bing's fetch is not subject to it. | |
| 403) echo "::notice::key file answers 403 to this runner (edge bot management), which cannot confirm the deploy; skipping the wait, IndexNow verifies the key itself"; break ;; | |
| esac | |
| if [ "$i" -eq 16 ]; then | |
| echo "::warning::key file not confirmed reachable after 8 min (last status ${status}); submitting anyway" | |
| else | |
| sleep 30 | |
| fi | |
| done | |
| # 10,000 URLs per request is the protocol limit; the full sitemap is | |
| # under that today (4.6k) but the split keeps this future-proof. | |
| split -l 10000 -d urls.txt batch_ | |
| for f in batch_*; do | |
| # The body goes through a file: a full-sitemap batch is ~250 KB, | |
| # and a single command-line argument is capped at 128 KB on Linux | |
| # — the first sitemap-scope run (33665718870) failed to exec curl | |
| # at all and reported HTTP 000 for 4,593 URLs. | |
| jq -n --arg host "$HOST" --arg key "$INDEXNOW_KEY" \ | |
| --arg loc "https://${HOST}/${INDEXNOW_KEY}.txt" \ | |
| --rawfile list "$f" \ | |
| '{host: $host, key: $key, keyLocation: $loc, | |
| urlList: ($list | split("\n") | map(select(length > 0)))}' > body.json | |
| n=$(grep -c . "$f") | |
| # First use of a key (rollout or rotation): IndexNow verifies the | |
| # key file asynchronously and answers 403 SiteVerificationNotCompleted | |
| # until it has — seen minutes after the file went live. Retry that | |
| # case once a minute until a ten-minute deadline has passed (an | |
| # elapsed deadline, so slow responses cannot stretch it); if it | |
| # persists, the run FAILS, so a `changed` run keeps its URL set and | |
| # can be re-run — a green warning would have dropped those URLs for | |
| # good, because later pushes submit only their own diffs. | |
| deadline=$(( $(date +%s) + 600 )) | |
| while :; do | |
| # Bounded: a slow or flaky api.indexnow.org must not burn the job | |
| # timeout; three retries cover a transient error, and a transport | |
| # error after them yields code 000 for the branch below instead of | |
| # aborting under `set -e`. | |
| code=$(curl -sS -o response.txt -w '%{http_code}' \ | |
| --max-time 60 --retry 3 --retry-delay 5 --retry-all-errors \ | |
| -X POST "https://api.indexnow.org/indexnow" \ | |
| -H "Content-Type: application/json; charset=utf-8" \ | |
| --data @body.json) || code="000" | |
| if [ "$code" = 403 ] && grep -q SiteVerificationNotCompleted response.txt 2>/dev/null \ | |
| && [ "$(date +%s)" -lt "$deadline" ]; then | |
| echo "::notice::IndexNow has not finished verifying the key file yet; retrying in 60 s" | |
| sleep 60 | |
| continue | |
| fi | |
| break | |
| done | |
| case "$code" in | |
| 200|202) echo "::notice::IndexNow accepted ${n} URL(s) (HTTP ${code})" ;; | |
| # 4xx is a protocol or key problem on our side (or the key still | |
| # unverified after ten minutes); make it visible and keep the run | |
| # re-runnable. | |
| 4*) echo "::error::IndexNow rejected ${n} URL(s) (HTTP ${code}): $(head -c 300 response.txt 2>/dev/null)"; exit 1 ;; | |
| # 5xx / no response: their side. Every later push resubmits its | |
| # own URLs and `scope=sitemap` covers a longer gap, so warn. | |
| *) echo "::warning::IndexNow unavailable (HTTP ${code}) for ${n} URL(s); resubmit with scope=sitemap if it persists" ;; | |
| esac | |
| done |