From de665ec49b7939c8a7c8f53e28b094b3c5a00850 Mon Sep 17 00:00:00 2001 From: Andre Levesque <0sniffs_scaled@icloud.com> Date: Fri, 17 Jul 2026 14:56:04 -0400 Subject: [PATCH 1/3] docs: add AGENTS.md, ground-truth.json, and Makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AGENTS.md (agents.md standard) gives every coding agent — Claude, OpenCode, Codex, Cursor — one predictable operating manual: commands, prod-breaking invariants, owner doctrines (lean/clean-links/no-AI- bios/highest-confidence), and PR rules including the all-CI-green merge rule. ground-truth.json is the machine-readable slow-moving canon (venue/event ids and dates, constants, infra identifiers) so agents stop re-deriving or guessing facts. The Makefile is the canonical task runner — real exit codes end the pipe-masking bug class, and make e2e codifies the full local Playwright recipe that previously lived in session notes. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 101 ++++++++++++++++++++++++++++++++++++++++++++++ Makefile | 81 +++++++++++++++++++++++++++++++++++++ ground-truth.json | 87 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 269 insertions(+) create mode 100644 AGENTS.md create mode 100644 Makefile create mode 100644 ground-truth.json diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..2e981769 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,101 @@ +# AGENTS.md — settimes.ca + +Instructions for AI coding agents (Claude Code, OpenCode, Codex, Cursor, …) working in this repo. +Format: [agents.md](https://agents.md). Humans: start with `README.md`; deep Claude-specific +workflow and the full invariant catalogue live in `CLAUDE.md` — read it before non-trivial work. + +## What this is + +settimes.ca — a **lean, highly performant** multi-venue live-music event platform for Waterloo +Region, ON. Fan-facing (schedule builder, band profiles, archives) and admin tooling are equal +priority. Flagship event: **Long Weekend Band Crawl Vol. 17 — August 2, 2026**. + +Machine-readable canon (venues, events, constants, doctrines): **`ground-truth.json`**. +Read it before asserting facts about events, venues, or dates — do not re-derive or guess. + +## Stack + +React 19 + Vite + Tailwind 4 (`frontend/`) · Cloudflare Pages Functions (`functions/`) · +Cloudflare D1 / SQLite (`migrations/`) · R2 for images · Vitest + Playwright · GitHub Actions. + +## Setup & commands + +Use the Makefile — it exists so every agent runs the same gates with real exit codes: + +- `make install` — install root + frontend deps +- `make gate` — **the full pre-commit gate** (format → lint → tests → build, both stacks). Run before every commit; do not commit if it fails. +- `make test` / `make test-backend` / `make test-frontend` +- `make build` — frontend production build +- `make e2e` — full local E2E: builds, seeds an isolated D1, serves wrangler on :8788, runs Playwright, cleans up +- `make schema-check` — after touching `migrations/`: regenerates `setup-complete.sql` and checks drift (its schema section is generated — never hand-edit) +- `make validate-openapi` — if `docs/api-spec.yaml` changed + +Never pipe a gate command through `tail`/`grep` inside a `&&` chain — the pipe's exit code +masks failures (this shipped a red lint to CI once). `make` propagates failures natively. + +## Top invariants (full catalogue in CLAUDE.md — these are the prod-breakers) + +- **After-midnight sets** (start before 06:00) belong to the previous evening. Threshold is + `AFTER_MIDNIGHT_THRESHOLD_HOUR = 6` in `frontend/src/utils/festivalDays.js`. Never remove, + lower, or re-encode it; any time-based sort/filter must apply the offset or use `prepareBands()`. +- **Server-side "today" is Toronto-local**, never UTC: `eventLocalToday()` / + `eventLocalClock()` from `functions/utils/eventDay.js`. `toISOString().slice(0,10)` flips + to tomorrow at 8 PM Eastern — a recurring bug class. +- **SQLite datetimes use a space separator** (`YYYY-MM-DD HH:MM:SS`). A `T` breaks string + comparison against `datetime('now')` silently. Exception: `lucia_sessions.expires_at` is + INTEGER Unix seconds — compare with `unixepoch()`. +- **D1 has no BEGIN/COMMIT** — use `DB.batch([...])` (atomic) or compensating deletes. +- **PBKDF2, never bcrypt** (no native binaries on Workers). MFA TOTP is hand-rolled on + Web Crypto — do not reintroduce otplib or pure-JS crypto. +- **Theme tokens on public pages** — `text-text-*`, `bg-surface*`, `border-border`; never + hardcoded `text-white`/`bg-white` (invisible on light themes). Admin (`frontend/src/admin/`) + is dark-pinned; hardcoded white there is intentional. +- **Two CSPs**: `frontend/public/_headers` governs the document (Turnstile, inline scripts — + the theme-flash script is hash-allowlisted; editing it requires regenerating the hash); + `functions/_middleware.js` governs API responses only. +- **Any list query with a LIMIT must bound the entity it lists** — GROUP BY the entity id or + paginate with a hasMore signal. A LIMIT on join-multiplied rows silently drops whole + entities (this once hid 47 of 218 bands from the admin roster). + +## Data doctrines (owner-set, non-negotiable) + +- **Lean and highly performant** is a design principle, not a preference. No media-heavy + features, no referral-junk URLs, weigh every feature against page weight first. +- **Clean links**: strip tracking params (`si`, `dlsi`, `nd`, `utm_*`, `from=`) before + storing any URL. +- **No AI-written artist prose.** Bios are entered verbatim from artist/organizer-supplied + text only. Structured facts (genre, origin, links) may be researched, but only to the + highest confidence — the source must unambiguously identify the specific band; a missing + field beats a wrong attribution. +- **Band follows are double opt-in** (`verified = 1` gates announcement email). Never revert + to auto-verify — it reopens an email-bombing vector. + +## PR rules + +- Branch from `main`; **rebase on `origin/main` before every push**, not just at PR open. +- Use the PR template (`.github/pull_request_template.md`); fill every section; `Closes #N` + on its own line in the body. One type label + one priority label at open time. +- **Nothing merges unless ALL CI checks are green** — required and non-required alike. + Never arm `gh pr merge --auto` while any check is still running. +- Track remaining/deferred work as GitHub issues, referenced from PRs — not chat threads. +- After addressing review comments: reply on the thread AND resolve it (rulesets block + merge on unresolved threads). +- When a change resolves a documented invariant/workaround, update `CLAUDE.md` (and this + file if the invariant is listed here) in the same PR. + +## Code style + +- Prettier + ESLint are law; `make gate` enforces both. Frontend: single quotes, no semis + (see `frontend/.prettierrc`); backend: double quotes, semis (root `.prettierrc`). +- Comments state constraints the code can't show — never narrate the next line or justify + a change to a reviewer. Reference issues (`#NNN`) when documenting a bug-class fix. +- Reference code locations as `file_path:line` in discussion. + +## Testing notes + +- Backend unit tests run from repo root (better-sqlite3-backed; may DLOPEN-fail on some + Apple Silicon setups — CI covers it, note it and move on). +- E2E requires a live wrangler server and env credentials (`ADMIN_EMAIL`/`ADMIN_PASSWORD` + are required for ANY Playwright invocation, including `--list`). `make e2e` handles this. +- E2E spec files are currently outside prettier/eslint scope (#622) — match their existing + single-quote style; don't reformat them piecemeal. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..06f3515f --- /dev/null +++ b/Makefile @@ -0,0 +1,81 @@ +# settimes.ca — canonical task runner for humans and agents. +# make propagates real exit codes; never wrap these in `cmd | tail` chains. +# E2E recipe mirrors .github/actions/e2e-env/action.yml. + +SHELL := /bin/sh +WRANGLER := ./frontend/node_modules/.bin/wrangler +E2E_STATE := .wrangler/e2e-state +E2E_PID := .wrangler/e2e-state/wrangler.pid +E2E_ADMIN_EMAIL ?= e2e-admin@test.local +E2E_ADMIN_PASSWORD ?= e2e-test-password-Xk9 + +.PHONY: help install build dev format format-check lint test test-backend test-frontend \ + gate validate-openapi schema-check e2e e2e-setup e2e-serve e2e-run e2e-clean + +help: ## List targets + @grep -E '^[a-zA-Z0-9_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' + +install: ## Install root + frontend dependencies + npm ci + cd frontend && npm ci + +build: ## Frontend production build (also what E2E serves) + npm run build --prefix frontend + +dev: ## Serve the built app + functions on :8788 against local D1 (foreground) + $(WRANGLER) pages dev frontend/dist --port 8788 --persist-to .wrangler/state + +format: ## Prettier --write, both stacks + npm run format + cd frontend && npx prettier --write "src/**/*.{js,jsx,json,css}" + +format-check: ## Prettier --check, both stacks (what CI runs) + npm run format:check + cd frontend && npm run format:check + +lint: ## ESLint, both stacks + npm run lint + cd frontend && npm run lint + +test-backend: ## Backend unit tests (better-sqlite3; may DLOPEN-fail on some Apple Silicon — CI covers it) + npm test + +test-frontend: ## Frontend unit tests + cd frontend && npm test -- --run + +test: test-backend test-frontend ## All unit tests + +gate: format format-check lint test build ## FULL pre-commit gate — run before every commit + +validate-openapi: ## Required when docs/api-spec.yaml changed + npm run validate:openapi + +schema-check: ## Required after touching migrations/ — regenerate + drift-check setup-complete.sql + node scripts/regenerate-setup-complete.mjs + node scripts/check-schema-drift.mjs + +e2e-setup: build ## Init + seed an isolated local D1 for E2E (safe to re-run) + rm -rf $(E2E_STATE) + $(WRANGLER) d1 execute settimes-production-db --local --persist-to $(E2E_STATE) --file=database/setup-complete.sql + $(WRANGLER) d1 execute settimes-production-db --local --persist-to $(E2E_STATE) --file=database/seed-test-data.sql + SEED_SQL=$$(node scripts/seed-e2e-admin.mjs --email "$(E2E_ADMIN_EMAIL)" --password "$(E2E_ADMIN_PASSWORD)"); \ + $(WRANGLER) d1 execute settimes-production-db --local --persist-to $(E2E_STATE) --command="$$SEED_SQL" + +e2e-serve: ## Start wrangler for E2E in the background (writes pidfile) + $(WRANGLER) pages dev frontend/dist --port 8788 --persist-to $(E2E_STATE) > $(E2E_STATE)/wrangler.log 2>&1 & \ + echo $$! > $(E2E_PID); \ + i=0; until curl -s -o /dev/null http://localhost:8788/ || [ $$i -ge 45 ]; do i=$$((i+1)); sleep 2; done; \ + curl -s -o /dev/null http://localhost:8788/ || { echo "wrangler failed to start; log:"; tail -20 $(E2E_STATE)/wrangler.log; exit 1; } + +e2e-run: ## Run Playwright (server must be up; credentials required even for --list) + ADMIN_EMAIL="$(E2E_ADMIN_EMAIL)" ADMIN_PASSWORD="$(E2E_ADMIN_PASSWORD)" npx playwright test $(SPEC) --reporter=list + +e2e-clean: ## Kill the E2E server by pidfile and remove isolated state + -[ -f $(E2E_PID) ] && kill $$(cat $(E2E_PID)) 2>/dev/null || true + rm -rf $(E2E_STATE) + +e2e: e2e-setup e2e-serve ## Full local E2E: setup, serve, run, always clean up + ADMIN_EMAIL="$(E2E_ADMIN_EMAIL)" ADMIN_PASSWORD="$(E2E_ADMIN_PASSWORD)" npx playwright test $(SPEC) --reporter=list; \ + status=$$?; \ + $(MAKE) e2e-clean; \ + exit $$status diff --git a/ground-truth.json b/ground-truth.json new file mode 100644 index 00000000..1d86980a --- /dev/null +++ b/ground-truth.json @@ -0,0 +1,87 @@ +{ + "_comment": "Machine-readable project canon for agents and humans. Slow-moving facts only — live data (lineups, band profiles, counts) belongs to the database, not this file. Update via PR when canon changes; bump verified_at when a human or agent re-verifies against production.", + "verified_at": "2026-07-17", + "project": { + "name": "SetTimes", + "domain": "https://settimes.ca", + "repo": "BreakableHoodie/settimesdotca", + "region": "Waterloo Region, Ontario, Canada (not Ottawa)", + "mission": "The best multi-venue / multi-artist event platform for Waterloo Region — fan-facing and admin tooling are equal priority", + "design_principles": [ + "lean and highly performant — media weight is a cost the site refuses", + "clean links — no referral/tracking params ever stored", + "no AI-written artist prose — bios are artist/organizer-supplied verbatim", + "researched facts at highest confidence only — a missing field beats a wrong attribution", + "SEO is a priority (structured data, titles, local discovery)" + ], + "roadmap": "docs/ROADMAP.md" + }, + "target_event": { + "name": "Long Weekend Band Crawl - Vol. 17", + "slug": "lwbc17", + "db_id": 21, + "date": "2026-08-02", + "doors": "18:30", + "show": "18:45", + "ages": "19+", + "band_count": 22, + "venue_street": "King St N, Waterloo" + }, + "venues_vol17": [ + { "db_id": 16, "name": "Blue Room (Inside Revive Karaoke)", "address": "28 King St N, Waterloo" }, + { "db_id": 8, "name": "Princess Cafe", "address": "46 King St N, Waterloo, ON, N2J 2W8" }, + { "db_id": 6, "name": "Prohibition Warehouse", "address": "56 King St N, Waterloo, ON, N2J 2X1" }, + { "db_id": 7, "name": "Revive Karaoke", "address": "28 King St N, Waterloo, ON, N2J 2W7" }, + { "db_id": 4, "name": "Room 47", "address": "47 King St N, Waterloo, ON, N2J 2W9" }, + { "db_id": 15, "name": "Roost", "address": "85 King St N, Waterloo, ON" } + ], + "venue_notes": { + "blue_room": "Blue Room is inside Revive Karaoke — both are distinct venues", + "jane_bond": "Jane Bond (db_id 17) is Vol-16 only", + "historical_2022": "Vol 1/2 venues (Dive Bar 85 King N, Revive Game Bar 90 King N, Pin Up Arcade Bar 8-247 King N) have no venue rows — they live in events.venue_info JSON; 85 King N is now Roost, Revive later moved to 28" + }, + "published_events": [ + { "db_id": 35, "slug": "blr3", "name": "Bad Livin' Roadshow 3", "date": "2026-07-10", "end_date": "2026-07-12" }, + { "db_id": 21, "slug": "lwbc17", "name": "Long Weekend Band Crawl - Vol. 17", "date": "2026-08-02" }, + { "db_id": 36, "slug": "buddiesfest2", "name": "Buddies Fest 2", "date": "2026-08-07", "end_date": "2026-08-09" } + ], + "archived_events": [ + { "db_id": 33, "slug": "lwbc01", "date": "2022-05-22" }, + { "db_id": 34, "slug": "lwbc02", "date": "2022-07-31" }, + { "db_id": 22, "slug": "lwbc03", "date": "2022-10-09" }, + { "db_id": 23, "slug": "lwbc04", "date": "2023-05-21" }, + { "db_id": 24, "slug": "lwbc05", "date": "2023-08-06" }, + { "db_id": 25, "slug": "lwbc06", "date": "2023-10-08" }, + { "db_id": 26, "slug": "lwbc07", "date": "2024-02-18" }, + { "db_id": 27, "slug": "lwbc08", "date": "2024-05-19" }, + { "db_id": 28, "slug": "lwbc09", "date": "2024-08-04" }, + { "db_id": 29, "slug": "lwbc10", "date": "2024-10-13" }, + { "db_id": 30, "slug": "lwbc11", "date": "2025-02-16" }, + { "db_id": 31, "slug": "lwbc12", "date": "2025-05-18" }, + { "db_id": 32, "slug": "lwbc13", "date": "2025-08-03" }, + { "db_id": 19, "slug": "lwbc14", "date": "2025-10-12" }, + { "db_id": 1, "slug": "lwbc15", "date": "2026-02-15" }, + { "db_id": 20, "slug": "lwbc16", "date": "2026-05-17" } + ], + "constants": { + "timezone": "America/Toronto", + "after_midnight_threshold_hour": 6, + "after_midnight_rule": "sets starting before 06:00 belong to the previous evening; canonical constant in frontend/src/utils/festivalDays.js", + "sqlite_datetime_format": "YYYY-MM-DD HH:MM:SS (space separator, never ISO T)", + "lucia_sessions_expires_at": "INTEGER Unix seconds — the one exception; compare with unixepoch()", + "social_link_keys": ["website", "instagram", "bandcamp", "facebook", "youtube", "spotify", "apple_music", "linktree"] + }, + "infrastructure": { + "hosting": "Cloudflare Pages + Pages Functions", + "database": { "product": "Cloudflare D1", "name": "settimes-production-db", "id": "c6c8b4a6-f951-47c1-a787-aab0a61ca09b" }, + "images": { "product": "Cloudflare R2", "bucket": "settimes-band-photos", "binding": "BAND_PHOTOS" }, + "migrations": "auto-apply on merge to main via cloudflare-pages.yml — never apply manually", + "e2e_port": 8788 + }, + "process": { + "merge_rule": "nothing merges unless ALL CI checks are green (required and non-required)", + "rebase_rule": "rebase on origin/main before every push", + "issue_tracking": "remaining/deferred work becomes a GitHub issue referenced from PRs", + "pre_commit_gate": "make gate" + } +} From 42bf567fbd5e4f1cb0f2a3b383911489db69d250 Mon Sep 17 00:00:00 2001 From: Andre Levesque <0sniffs_scaled@icloud.com> Date: Fri, 17 Jul 2026 15:00:20 -0400 Subject: [PATCH 2/3] feat: probe-band-links script + make target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Searching for band links is the time sink; validation is a 2-second owner yes/no. The probe checks canonical bandcamp subdomain patterns for every linkless band with plain HTTP (no search engine, no AI) and buckets results: AUTO (exact name match + Ontario location — doctrine- safe), REVIEW (page exists, identity unproven), NONE. Existence is not identity: unclaimed subdomains redirect to signup, and popular names are taken by unrelated acts. Co-Authored-By: Claude Fable 5 --- Makefile | 3 + scripts/probe-band-links.mjs | 137 +++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 scripts/probe-band-links.mjs diff --git a/Makefile b/Makefile index 06f3515f..3a491bc7 100644 --- a/Makefile +++ b/Makefile @@ -79,3 +79,6 @@ e2e: e2e-setup e2e-serve ## Full local E2E: setup, serve, run, always clean up status=$$?; \ $(MAKE) e2e-clean; \ exit $$status + +probe-links: ## Probe canonical bandcamp URLs for linkless bands (FILE=names.txt, newline-separated) + node scripts/probe-band-links.mjs --file $(FILE) diff --git a/scripts/probe-band-links.mjs b/scripts/probe-band-links.mjs new file mode 100644 index 00000000..6e143975 --- /dev/null +++ b/scripts/probe-band-links.mjs @@ -0,0 +1,137 @@ +#!/usr/bin/env node +/** + * Probe canonical bandcamp URL patterns for bands with no links on file. + * + * Searching is the expensive part of link collection; validation is cheap + * (owner does a 2-second yes/no). This script does the searching with plain + * HTTP — no search engine, no AI, no rate-limited APIs — and buckets results: + * + * AUTO exact band-name match on the page AND an Ontario location + * (safe to enter under the highest-confidence doctrine) + * REVIEW a real band page exists but identity is unproven + * (name variant, non-Ontario or missing location) — owner validates + * NONE no band page at any candidate URL + * + * Existence is NOT identity: bandcamp serves a signup page for unclaimed + * subdomains, and popular names are taken by unrelated acts (our Mixed + * Feelings lives at mixedfeelings2 because mixedfeelings was taken). + * + * Usage: + * node scripts/probe-band-links.mjs --file bands.txt # newline-separated names + * node scripts/probe-band-links.mjs --names "Band A,Band B" + * make probe-links FILE=bands.txt + */ + +import { readFileSync } from "node:fs"; + +const args = process.argv.slice(2); +function argValue(flag) { + const i = args.indexOf(flag); + return i >= 0 ? args[i + 1] : null; +} + +const file = argValue("--file"); +const namesArg = argValue("--names"); +let names = []; +if (file) { + names = readFileSync(file, "utf8") + .split("\n") + .map((l) => l.trim()) + .filter(Boolean); +} else if (namesArg) { + names = namesArg + .split(",") + .map((n) => n.trim()) + .filter(Boolean); +} else { + console.error("Usage: probe-band-links.mjs --file | --names 'A,B,C'"); + process.exit(1); +} + +/** Candidate bandcamp subdomain slugs for a band name, most-canonical first. */ +function candidateSlugs(name) { + const concat = name.toLowerCase().replace(/[^a-z0-9]/g, ""); + const noThe = name + .toLowerCase() + .replace(/^(the|a|an)\s+/, "") + .replace(/[^a-z0-9]/g, ""); + const hyphen = name + .toLowerCase() + .trim() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, ""); + return [...new Set([concat, noThe, hyphen])].filter((s) => s.length >= 3); +} + +const normalize = (s) => (s || "").toLowerCase().replace(/[^a-z0-9]/g, ""); + +/** Fetch a bandcamp subdomain; return null for dead/signup pages. */ +async function probe(slug) { + const url = `https://${slug}.bandcamp.com/`; + let res; + try { + res = await fetch(url, { + redirect: "follow", + signal: AbortSignal.timeout(12000), + headers: { "User-Agent": "settimes-link-probe/1.0 (band link discovery; settimes.ca)" }, + }); + } catch { + return null; + } + if (!res.ok) return null; + // Unclaimed subdomains redirect to bandcamp.com signup/discover. + if (!new URL(res.url).hostname.startsWith(slug + ".")) return null; + const html = (await res.text()).slice(0, 120000); + const title = (html.match(/([^<]*)<\/title>/i)?.[1] || "").trim(); + if (/^signup\b/i.test(title)) return null; + // Band pages title as "Music | Band Name" or "Release | Band Name". + const pageBandName = title.includes("|") ? title.split("|").pop().trim() : title; + // Location renders as <span class="location secondaryText">City, Region</span> + const location = (html.match(/class="location[^"]*"[^>]*>([^<]*)</i)?.[1] || "").trim(); + return { url, title, pageBandName, location }; +} + +const ONTARIO = /ontario|,\s*on\b/i; + +const results = { AUTO: [], REVIEW: [], NONE: [] }; +for (const name of names) { + let hit = null; + let hitSlug = null; + for (const slug of candidateSlugs(name)) { + // Sequential + polite: one candidate at a time, stop at the first live page. + const page = await probe(slug); + if (page) { + hit = page; + hitSlug = slug; + break; + } + } + if (!hit) { + results.NONE.push({ name }); + continue; + } + const nameMatches = normalize(hit.pageBandName) === normalize(name); + const inOntario = ONTARIO.test(hit.location); + const bucket = nameMatches && inOntario ? "AUTO" : "REVIEW"; + results[bucket].push({ + name, + url: hit.url, + page_band_name: hit.pageBandName, + location: hit.location || "(none shown)", + slug: hitSlug, + }); + console.error(`probed: ${name} → ${bucket} (${hit.url})`); +} + +console.log("\n## AUTO — exact name match + Ontario location (doctrine-safe to enter)\n"); +for (const r of results.AUTO) console.log(`- **${r.name}** → ${r.url} · "${r.page_band_name}" · ${r.location}`); +if (!results.AUTO.length) console.log("(none)"); + +console.log("\n## REVIEW — page exists, identity unproven (owner yes/no)\n"); +for (const r of results.REVIEW) + console.log(`- **${r.name}** → ${r.url} · page says "${r.page_band_name}" · ${r.location}`); +if (!results.REVIEW.length) console.log("(none)"); + +console.log("\n## NONE — no candidate URL is a live band page\n"); +for (const r of results.NONE) console.log(`- ${r.name}`); +if (!results.NONE.length) console.log("(none)"); From 581f69c3b191143e1fb22ccdd4b95e2894c651ad Mon Sep 17 00:00:00 2001 From: Andre Levesque <0sniffs_scaled@icloud.com> Date: Fri, 17 Jul 2026 15:07:19 -0400 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20probe-band-links=20v2=20=E2=80=94=20?= =?UTF-8?q?throttle,=20429-as-retry,=20full-body=20location?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unthrottled 93-band run tripped bandcamp's rate limiter: late 429s were silently bucketed as NONE (false negatives) and the location scan never got a fair test (it also only read the first 120KB — the location markup sits deep in the sidebar). Requests now space 2s apart with a 30s backoff on 429; rate-limited bands land in a RETRY bucket, never NONE. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- scripts/probe-band-links.mjs | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/scripts/probe-band-links.mjs b/scripts/probe-band-links.mjs index 6e143975..ed70a90a 100644 --- a/scripts/probe-band-links.mjs +++ b/scripts/probe-band-links.mjs @@ -66,6 +66,9 @@ function candidateSlugs(name) { const normalize = (s) => (s || "").toLowerCase().replace(/[^a-z0-9]/g, ""); /** Fetch a bandcamp subdomain; return null for dead/signup pages. */ +const THROTTLE_MS = 2000; // stay under bandcamp's rate limiter — a 429'd run misreads live pages as NONE +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + async function probe(slug) { const url = `https://${slug}.bandcamp.com/`; let res; @@ -78,28 +81,39 @@ async function probe(slug) { } catch { return null; } + if (res.status === 429) return { rateLimited: true }; // NEVER read a 429 as "no page" if (!res.ok) return null; // Unclaimed subdomains redirect to bandcamp.com signup/discover. if (!new URL(res.url).hostname.startsWith(slug + ".")) return null; - const html = (await res.text()).slice(0, 120000); + // Full body — the location markup sits deep in the sidebar, far past the head. + const html = await res.text(); const title = (html.match(/<title>([^<]*)<\/title>/i)?.[1] || "").trim(); if (/^signup\b/i.test(title)) return null; // Band pages title as "Music | Band Name" or "Release | Band Name". const pageBandName = title.includes("|") ? title.split("|").pop().trim() : title; - // Location renders as <span class="location secondaryText">City, Region</span> - const location = (html.match(/class="location[^"]*"[^>]*>([^<]*)</i)?.[1] || "").trim(); + // Location renders as <span class="location secondaryText">City, Region</span>; + // fall back to any og/meta description mention of a place-like "City, Region". + const location = (html.match(/class="location[^"]*"[^>]*>\s*([^<]+?)\s*</i)?.[1] || "").trim(); return { url, title, pageBandName, location }; } const ONTARIO = /ontario|,\s*on\b/i; -const results = { AUTO: [], REVIEW: [], NONE: [] }; +const results = { AUTO: [], REVIEW: [], NONE: [], RETRY: [] }; for (const name of names) { let hit = null; let hitSlug = null; + let sawRateLimit = false; for (const slug of candidateSlugs(name)) { // Sequential + polite: one candidate at a time, stop at the first live page. const page = await probe(slug); + await sleep(THROTTLE_MS); + if (page?.rateLimited) { + sawRateLimit = true; + console.error(`probed: ${name} → 429 on ${slug}, backing off 30s`); + await sleep(30000); + continue; + } if (page) { hit = page; hitSlug = slug; @@ -107,7 +121,7 @@ for (const name of names) { } } if (!hit) { - results.NONE.push({ name }); + results[sawRateLimit ? "RETRY" : "NONE"].push({ name }); continue; } const nameMatches = normalize(hit.pageBandName) === normalize(name); @@ -135,3 +149,7 @@ if (!results.REVIEW.length) console.log("(none)"); console.log("\n## NONE — no candidate URL is a live band page\n"); for (const r of results.NONE) console.log(`- ${r.name}`); if (!results.NONE.length) console.log("(none)"); + +console.log("\n## RETRY — rate-limited mid-probe; result unknown, re-run later\n"); +for (const r of results.RETRY) console.log(`- ${r.name}`); +if (!results.RETRY.length) console.log("(none)");