From 42344a08494fd207e370693c75888940dbde78e6 Mon Sep 17 00:00:00 2001 From: Chris Chudzicki Date: Tue, 21 Jul 2026 13:51:25 -0400 Subject: [PATCH 1/3] feat: tighten robots.txt to cut duplicate bot crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every ?resource= drawer URL canonicalizes to /search?resource=&resource_title=, and the resources sitemap enumerates every resource at exactly that URL — so crawling drawer overlays on /c/, /news, home, and faceted search is pure duplicate load (Applebot alone fans out over ~2.5M such URLs/week). Allow only the canonical drawer form and the bare /search landing, and disallow resource-carrying URLs site-wide, internal search results, and Next.js _rsc prefetch payloads (never part of rendered or indexed content). Also: - add /enrollmentcode/, /organization/, and /website_content/drafts to the app-only disallows - exempt link-preview fetchers (Facebook/Twitter/Slack/etc.) via their own group so og: cards keep working on drawer URLs - block AI-training crawlers (GPTBot, CCBot, meta-externalagent, Bytespider, ClaudeBot, Amazonbot) and opt out of training use of Googlebot/Applebot crawl data (Google-Extended, Applebot-Extended) The Allow rules win by longest-match precedence (RFC 9309) and are emitted first for naive first-match parsers. They are literal prefix matches, so `resource` must remain the first query param in canonical drawer URLs — documented on resourceDrawerSearch, whose output is already pinned by urls.test.ts. Co-Authored-By: Claude Fable 5 --- frontends/main/src/app/robots.test.ts | 76 +++++++++++++++++++++++++ frontends/main/src/app/robots.ts | 80 +++++++++++++++++++++++++-- frontends/main/src/common/urls.ts | 5 ++ 3 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 frontends/main/src/app/robots.test.ts diff --git a/frontends/main/src/app/robots.test.ts b/frontends/main/src/app/robots.test.ts new file mode 100644 index 0000000000..c6f9c57ee8 --- /dev/null +++ b/frontends/main/src/app/robots.test.ts @@ -0,0 +1,76 @@ +import robots from "./robots" + +describe("robots", () => { + const originalNoindex = process.env.MITOL_NOINDEX + + afterEach(() => { + process.env.MITOL_NOINDEX = originalNoindex + }) + + it("disallows everything when MITOL_NOINDEX is not 'false'", () => { + process.env.MITOL_NOINDEX = "true" + expect(robots()).toEqual({ + rules: { userAgent: "*", disallow: "/" }, + }) + }) + + it("emits the crawl rules when indexing is enabled", () => { + process.env.MITOL_NOINDEX = "false" + expect(robots()).toEqual({ + rules: [ + { + userAgent: "*", + // Canonical resource drawer URLs (the form the resources sitemap + // emits) and the bare search landing stay crawlable; these win + // over the disallows below by longest-match precedence. + allow: ["/search$", "/search?resource="], + disallow: [ + "/search", + "/*?resource=", + "/*&resource=", + "/*?_rsc=", + "/*&_rsc=", + "/dashboard/", + "/learningpaths/", + "/onboarding/", + "/cart/", + "/program_letter/", + "/enrollmentcode/", + "/organization/", + "/website_content/drafts", + ], + }, + { + userAgent: [ + "facebookexternalhit", + "Twitterbot", + "Slackbot", + "LinkedInBot", + "Discordbot", + "WhatsApp", + "TelegramBot", + ], + allow: "/", + }, + { + userAgent: [ + "GPTBot", + "CCBot", + "meta-externalagent", + "Google-Extended", + "Applebot-Extended", + "Bytespider", + "ClaudeBot", + "Amazonbot", + ], + disallow: "/", + }, + { + userAgent: "meta-externalads", + disallow: "/", + }, + ], + sitemap: "http://test.learn.odl.local:8062/sitemaps/sitemap-index.xml", + }) + }) +}) diff --git a/frontends/main/src/app/robots.ts b/frontends/main/src/app/robots.ts index 2a9028bcae..e089975d31 100644 --- a/frontends/main/src/app/robots.ts +++ b/frontends/main/src/app/robots.ts @@ -23,21 +23,89 @@ export default function robots(): MetadataRoute.Robots { rules: [ { userAgent: "*", - allow: "/", + /** + * Resource drawer: every ?resource= URL canonicalizes to + * /search?resource=&resource_title=, and the resources + * sitemap enumerates every resource at exactly that URL. Crawling + * drawer overlays anywhere else (/c/, /news, /, faceted search) is + * pure duplicate load, so allow only the canonical form (and the + * bare /search landing page) and block resource-carrying URLs + * site-wide. The allow rules win by longest-match precedence + * (RFC 9309); they are listed first for naive first-match parsers. + * + * NOTE: the allow rule is a literal prefix match, so it depends on + * `resource` being the FIRST query param in canonical drawer URLs + * (see resourceDrawerSearch in common/urls.ts). + */ + allow: ["/search$", "/search?resource="], disallow: [ + "/search", + "/*?resource=", + "/*&resource=", + // Next.js router-prefetch payloads — never part of rendered or + // indexed content + "/*?_rsc=", + "/*&_rsc=", + // Account / app-only areas "/dashboard/", "/learningpaths/", "/onboarding/", "/cart/", "/program_letter/", + "/enrollmentcode/", + "/organization/", + "/website_content/drafts", ], }, - // Meta's ad-preview crawler, not a real visitor -- driving disproportionate - // load against expensive, uncached SSR routes. robots.txt is advisory only - // (a non-compliant crawler can ignore it), so if this doesn't reduce its - // request volume, blocking it at the gateway/WAF layer is the follow-up. { - userAgent: "meta-externalads/1.1", + /** + * Link-preview fetchers: fetch exact shared URLs on demand (tiny + * volume) and must be able to see ?resource= URLs for og: cards. + * A named group opts them out of ALL default-group rules. + */ + userAgent: [ + "facebookexternalhit", + "Twitterbot", + "Slackbot", + "LinkedInBot", + "Discordbot", + "WhatsApp", + "TelegramBot", + ], + allow: "/", + }, + { + /** + * AI-training crawlers — blocking costs no search visibility. + * (Google-Extended / Applebot-Extended are opt-out tokens + * controlling training use of Googlebot/Applebot crawl data, not + * separate crawlers.) + */ + userAgent: [ + "GPTBot", + "CCBot", + "meta-externalagent", + "Google-Extended", + "Applebot-Extended", + "Bytespider", + "ClaudeBot", + "Amazonbot", + ], + disallow: "/", + }, + { + /** + * Meta's advertising/business crawler. Blocked 2026-07-21 (#3653) + * after it crawled ?resource=/_rsc= URL permutations at up to + * ~90k req/hr — the majority of all origin-reaching traffic during + * the incident. robots.txt is advisory only (this UA has never + * fetched /robots.txt here), so the gateway/WAF layer is the + * enforcement backstop if volume doesn't drop. + * + * NOTE: RFC 9309 user-agent tokens cannot contain "/" — matching + * requires the bare product token, not "meta-externalads/1.1". + */ + userAgent: "meta-externalads", disallow: "/", }, ], diff --git a/frontends/main/src/common/urls.ts b/frontends/main/src/common/urls.ts index d7f83d9a8d..091098cea3 100644 --- a/frontends/main/src/common/urls.ts +++ b/frontends/main/src/common/urls.ts @@ -155,6 +155,11 @@ export const absoluteUrl = (path: string): string => * /search?resource={id}[&resource_title={slug}] * `resource` is the authoritative id; `resource_title` is a cosmetic slug, * omitted when blank and ignored on lookup. + * + * `resource` MUST stay the first query param: robots.ts re-allows canonical + * drawer URLs via the literal prefix rule `Allow: /search?resource=`, so + * reordering the params would silently block them from crawlers (these are + * the URLs the resources sitemap emits). */ export const resourceDrawerSearch = ( resourceId: number, From 675b735f4f66d7fb6c9ea96ee8d0f5f28dbed55e Mon Sep 17 00:00:00 2001 From: Chris Chudzicki Date: Tue, 21 Jul 2026 14:16:17 -0400 Subject: [PATCH 2/3] test: pin robots allow rule to resourceDrawerSearch output The Allow rule for canonical drawer URLs is a literal prefix match, so it silently stops covering the sitemap/canonical URLs if resourceDrawerSearch ever changes its path or param order. Evaluate the emitted rules against the real builder output with a minimal RFC 9309 longest-match evaluator. Co-Authored-By: Claude Fable 5 --- frontends/main/src/app/robots.test.ts | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/frontends/main/src/app/robots.test.ts b/frontends/main/src/app/robots.test.ts index c6f9c57ee8..8f5ff82dfa 100644 --- a/frontends/main/src/app/robots.test.ts +++ b/frontends/main/src/app/robots.test.ts @@ -1,4 +1,39 @@ import robots from "./robots" +import { resourceDrawerSearch } from "@/common/urls" + +/** + * Minimal RFC 9309 rule evaluator: `*` matches any chars, `$` anchors the + * end, the longest matching pattern wins, and ties go to Allow. Used to pin + * rule *interactions* (which rule wins for a URL), not just the rule list. + */ +const escapeRegExp = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +const patternMatches = (pattern: string, path: string): boolean => { + const anchored = pattern.endsWith("$") + const body = anchored ? pattern.slice(0, -1) : pattern + const source = `^${body.split("*").map(escapeRegExp).join(".*")}${anchored ? "$" : ""}` + return new RegExp(source).test(path) +} +const asArray = (value: string | string[] | undefined): string[] => + value === undefined ? [] : Array.isArray(value) ? value : [value] +const isAllowed = ( + group: { allow?: string | string[]; disallow?: string | string[] }, + path: string, +): boolean => { + const matched = [ + ...asArray(group.allow).map((p) => ({ p, allow: true })), + ...asArray(group.disallow).map((p) => ({ p, allow: false })), + ].filter(({ p }) => patternMatches(p, path)) + matched.sort((a, b) => b.p.length - a.p.length || (a.allow ? -1 : 1)) + return matched[0]?.allow ?? true +} + +const getDefaultGroup = () => { + const rules = robots().rules + if (!Array.isArray(rules)) throw new Error("expected an array of rule groups") + const group = rules.find((r) => r.userAgent === "*") + if (!group) throw new Error("expected a default (*) rule group") + return group +} describe("robots", () => { const originalNoindex = process.env.MITOL_NOINDEX @@ -73,4 +108,40 @@ describe("robots", () => { sitemap: "http://test.learn.odl.local:8062/sitemaps/sitemap-index.xml", }) }) + + /** + * The Allow rule for drawer URLs is a literal prefix, so it only covers + * URLs exactly as resourceDrawerSearch emits them — the same builder the + * resources sitemap and canonical tags use. If the builder changes (path, + * param order), these fail rather than silently de-indexing every resource. + */ + describe("canonical drawer URLs stay crawlable", () => { + beforeEach(() => { + process.env.MITOL_NOINDEX = "false" + }) + + it("allows the URL form the resources sitemap emits", () => { + const group = getDefaultGroup() + expect( + isAllowed( + group, + resourceDrawerSearch(123, "Introduction to Algorithms"), + ), + ).toBe(true) + expect(isAllowed(group, resourceDrawerSearch(123, undefined))).toBe(true) + }) + + it("allows the bare search landing page but not faceted search", () => { + const group = getDefaultGroup() + expect(isAllowed(group, "/search")).toBe(true) + expect(isAllowed(group, "/search?q=physics")).toBe(false) + }) + + it("disallows drawer overlays anywhere else", () => { + const group = getDefaultGroup() + expect(isAllowed(group, "/?resource=123")).toBe(false) + expect(isAllowed(group, "/c/topic/physics?resource=123")).toBe(false) + expect(isAllowed(group, "/search?q=physics&resource=123")).toBe(false) + }) + }) }) From c6942e91ad8d82b286227f65e653cd64106b5449 Mon Sep 17 00:00:00 2001 From: Chris Chudzicki Date: Tue, 21 Jul 2026 14:25:25 -0400 Subject: [PATCH 3/3] fix: address review comments on robots rules and test env handling Drop the $ end-anchor: RFC 9309 does standardize $, but disallowing /search? instead of /search keeps the bare landing page crawlable even under legacy parsers that treat $ literally, and needs no anchor at all. Restore MITOL_NOINDEX by deletion when it was originally unset, since assigning undefined to process.env stores the string "undefined". Co-Authored-By: Claude Fable 5 --- frontends/main/src/app/robots.test.ts | 14 +++++++++----- frontends/main/src/app/robots.ts | 15 +++++++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/frontends/main/src/app/robots.test.ts b/frontends/main/src/app/robots.test.ts index 8f5ff82dfa..f7f3843e36 100644 --- a/frontends/main/src/app/robots.test.ts +++ b/frontends/main/src/app/robots.test.ts @@ -39,7 +39,11 @@ describe("robots", () => { const originalNoindex = process.env.MITOL_NOINDEX afterEach(() => { - process.env.MITOL_NOINDEX = originalNoindex + if (originalNoindex === undefined) { + delete process.env.MITOL_NOINDEX + } else { + process.env.MITOL_NOINDEX = originalNoindex + } }) it("disallows everything when MITOL_NOINDEX is not 'false'", () => { @@ -56,11 +60,11 @@ describe("robots", () => { { userAgent: "*", // Canonical resource drawer URLs (the form the resources sitemap - // emits) and the bare search landing stay crawlable; these win - // over the disallows below by longest-match precedence. - allow: ["/search$", "/search?resource="], + // emits) stay crawlable; this wins over the disallows below by + // longest-match precedence. + allow: ["/search?resource="], disallow: [ - "/search", + "/search?", "/*?resource=", "/*&resource=", "/*?_rsc=", diff --git a/frontends/main/src/app/robots.ts b/frontends/main/src/app/robots.ts index e089975d31..cf4bd894a6 100644 --- a/frontends/main/src/app/robots.ts +++ b/frontends/main/src/app/robots.ts @@ -28,18 +28,21 @@ export default function robots(): MetadataRoute.Robots { * /search?resource=&resource_title=, and the resources * sitemap enumerates every resource at exactly that URL. Crawling * drawer overlays anywhere else (/c/, /news, /, faceted search) is - * pure duplicate load, so allow only the canonical form (and the - * bare /search landing page) and block resource-carrying URLs - * site-wide. The allow rules win by longest-match precedence - * (RFC 9309); they are listed first for naive first-match parsers. + * pure duplicate load, so allow only the canonical form and block + * resource-carrying URLs site-wide. The bare /search landing page + * matches no disallow and stays crawlable by default. The allow + * rule wins by longest-match precedence (RFC 9309); it is listed + * first for naive first-match parsers. * * NOTE: the allow rule is a literal prefix match, so it depends on * `resource` being the FIRST query param in canonical drawer URLs * (see resourceDrawerSearch in common/urls.ts). */ - allow: ["/search$", "/search?resource="], + allow: ["/search?resource="], disallow: [ - "/search", + // Faceted/keyword search results (any query string except the + // canonical drawer form above) + "/search?", "/*?resource=", "/*&resource=", // Next.js router-prefetch payloads — never part of rendered or