Tighten robots.txt to cut duplicate bot crawl

frontends/main/src/app/robots.test.ts

@@ -0,0 +1,151 @@
+import robots from "./robots"
+import { resourceDrawerSearch } from "@/common/urls"
+
+/**
+ * Minimal RFC 9309 rule evaluator: `*` matches any chars, `$` anchors the
+ * end, the longest matching pattern wins, and ties go to Allow. Used to pin
+ * rule *interactions* (which rule wins for a URL), not just the rule list.
+ */
+const escapeRegExp = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+const patternMatches = (pattern: string, path: string): boolean => {
+  const anchored = pattern.endsWith("$")
+  const body = anchored ? pattern.slice(0, -1) : pattern
+  const source = `^${body.split("*").map(escapeRegExp).join(".*")}${anchored ? "$" : ""}`
+  return new RegExp(source).test(path)
+}
+const asArray = (value: string | string[] | undefined): string[] =>
+  value === undefined ? [] : Array.isArray(value) ? value : [value]
+const isAllowed = (
+  group: { allow?: string | string[]; disallow?: string | string[] },
+  path: string,
+): boolean => {
+  const matched = [
+    ...asArray(group.allow).map((p) => ({ p, allow: true })),
+    ...asArray(group.disallow).map((p) => ({ p, allow: false })),
+  ].filter(({ p }) => patternMatches(p, path))
+  matched.sort((a, b) => b.p.length - a.p.length || (a.allow ? -1 : 1))
+  return matched[0]?.allow ?? true
+}
+
+const getDefaultGroup = () => {
+  const rules = robots().rules
+  if (!Array.isArray(rules)) throw new Error("expected an array of rule groups")
+  const group = rules.find((r) => r.userAgent === "*")
+  if (!group) throw new Error("expected a default (*) rule group")
+  return group
+}
+
+describe("robots", () => {
+  const originalNoindex = process.env.MITOL_NOINDEX
+
+  afterEach(() => {
+    if (originalNoindex === undefined) {
+      delete process.env.MITOL_NOINDEX
+    } else {
+      process.env.MITOL_NOINDEX = originalNoindex
+    }
+  })
+
+  it("disallows everything when MITOL_NOINDEX is not 'false'", () => {
+    process.env.MITOL_NOINDEX = "true"
+    expect(robots()).toEqual({
+      rules: { userAgent: "*", disallow: "/" },
+    })
+  })
+
+  it("emits the crawl rules when indexing is enabled", () => {
+    process.env.MITOL_NOINDEX = "false"
+    expect(robots()).toEqual({
+      rules: [
+        {
+          userAgent: "*",
+          // Canonical resource drawer URLs (the form the resources sitemap
+          // emits) stay crawlable; this wins over the disallows below by
+          // longest-match precedence.
+          allow: ["/search?resource="],
+          disallow: [
+            "/search?",
+            "/*?resource=",
+            "/*&resource=",
+            "/*?_rsc=",
+            "/*&_rsc=",
+            "/dashboard/",
+            "/learningpaths/",
+            "/onboarding/",
+            "/cart/",
+            "/program_letter/",
+            "/enrollmentcode/",
+            "/organization/",
+            "/website_content/drafts",
+          ],
+        },
+        {
+          userAgent: [
+            "facebookexternalhit",
+            "Twitterbot",
+            "Slackbot",
+            "LinkedInBot",
+            "Discordbot",
+            "WhatsApp",
+            "TelegramBot",
+          ],
+          allow: "/",
+        },
+        {
+          userAgent: [
+            "GPTBot",
+            "CCBot",
+            "meta-externalagent",
+            "Google-Extended",
+            "Applebot-Extended",
+            "Bytespider",
+            "ClaudeBot",
+            "Amazonbot",
+          ],
+          disallow: "/",
+        },
+        {
+          userAgent: "meta-externalads",
+          disallow: "/",
+        },
+      ],
+      sitemap: "http://test.learn.odl.local:8062/sitemaps/sitemap-index.xml",
+    })
+  })
+
+  /**
+   * The Allow rule for drawer URLs is a literal prefix, so it only covers
+   * URLs exactly as resourceDrawerSearch emits them — the same builder the
+   * resources sitemap and canonical tags use. If the builder changes (path,
+   * param order), these fail rather than silently de-indexing every resource.
+   */
+  describe("canonical drawer URLs stay crawlable", () => {
+    beforeEach(() => {
+      process.env.MITOL_NOINDEX = "false"
+    })
+
+    it("allows the URL form the resources sitemap emits", () => {
+      const group = getDefaultGroup()
+      expect(
+        isAllowed(
+          group,
+          resourceDrawerSearch(123, "Introduction to Algorithms"),
+        ),
+      ).toBe(true)
+      expect(isAllowed(group, resourceDrawerSearch(123, undefined))).toBe(true)
+    })
+
+    it("allows the bare search landing page but not faceted search", () => {
+      const group = getDefaultGroup()
+      expect(isAllowed(group, "/search")).toBe(true)
+      expect(isAllowed(group, "/search?q=physics")).toBe(false)
+    })
+
+    it("disallows drawer overlays anywhere else", () => {
+      const group = getDefaultGroup()
+      expect(isAllowed(group, "/?resource=123")).toBe(false)
+      expect(isAllowed(group, "/c/topic/physics?resource=123")).toBe(false)
+      expect(isAllowed(group, "/search?q=physics&resource=123")).toBe(false)
+    })
+  })
+})

frontends/main/src/app/robots.ts

@@ -23,21 +23,92 @@ export default function robots(): MetadataRoute.Robots {
       rules: [
         {
           userAgent: "*",
-          allow: "/",
+          /**
+           * Resource drawer: every ?resource= URL canonicalizes to
+           * /search?resource=<id>&resource_title=<slug>, and the resources
+           * sitemap enumerates every resource at exactly that URL. Crawling
+           * drawer overlays anywhere else (/c/, /news, /, faceted search) is
+           * pure duplicate load, so allow only the canonical form and block
+           * resource-carrying URLs site-wide. The bare /search landing page
+           * matches no disallow and stays crawlable by default. The allow
+           * rule wins by longest-match precedence (RFC 9309); it is listed
+           * first for naive first-match parsers.
+           *
+           * NOTE: the allow rule is a literal prefix match, so it depends on
+           * `resource` being the FIRST query param in canonical drawer URLs
+           * (see resourceDrawerSearch in common/urls.ts).
+           */
+          allow: ["/search?resource="],
           disallow: [
+            // Faceted/keyword search results (any query string except the
+            // canonical drawer form above)
+            "/search?",
+            "/*?resource=",
+            "/*&resource=",
+            // Next.js router-prefetch payloads — never part of rendered or
+            // indexed content
+            "/*?_rsc=",
+            "/*&_rsc=",
+            // Account / app-only areas
             "/dashboard/",
             "/learningpaths/",
             "/onboarding/",
             "/cart/",
             "/program_letter/",
+            "/enrollmentcode/",
+            "/organization/",
+            "/website_content/drafts",
           ],
         },
-        // Meta's ad-preview crawler, not a real visitor -- driving disproportionate
-        // load against expensive, uncached SSR routes. robots.txt is advisory only
-        // (a non-compliant crawler can ignore it), so if this doesn't reduce its
-        // request volume, blocking it at the gateway/WAF layer is the follow-up.
         {
-          userAgent: "meta-externalads/1.1",
+          /**
+           * Link-preview fetchers: fetch exact shared URLs on demand (tiny
+           * volume) and must be able to see ?resource= URLs for og: cards.
+           * A named group opts them out of ALL default-group rules.
+           */
+          userAgent: [
+            "facebookexternalhit",
+            "Twitterbot",
+            "Slackbot",
+            "LinkedInBot",
+            "Discordbot",
+            "WhatsApp",
+            "TelegramBot",
+          ],
+          allow: "/",
+        },
+        {
+          /**
+           * AI-training crawlers — blocking costs no search visibility.
+           * (Google-Extended / Applebot-Extended are opt-out tokens
+           * controlling training use of Googlebot/Applebot crawl data, not
+           * separate crawlers.)
+           */
+          userAgent: [
+            "GPTBot",
+            "CCBot",
+            "meta-externalagent",
+            "Google-Extended",
+            "Applebot-Extended",
+            "Bytespider",
+            "ClaudeBot",
+            "Amazonbot",
+          ],
+          disallow: "/",
+        },
+        {
+          /**
+           * Meta's advertising/business crawler. Blocked 2026-07-21 (#3653)
+           * after it crawled ?resource=/_rsc= URL permutations at up to
+           * ~90k req/hr — the majority of all origin-reaching traffic during
+           * the incident. robots.txt is advisory only (this UA has never
+           * fetched /robots.txt here), so the gateway/WAF layer is the
+           * enforcement backstop if volume doesn't drop.
+           *
+           * NOTE: RFC 9309 user-agent tokens cannot contain "/" — matching
+           * requires the bare product token, not "meta-externalads/1.1".
+           */
+          userAgent: "meta-externalads",
           disallow: "/",
         },
       ],

frontends/main/src/common/urls.ts

@@ -155,6 +155,11 @@ export const absoluteUrl = (path: string): string =>
  *   /search?resource={id}[&resource_title={slug}]
  * `resource` is the authoritative id; `resource_title` is a cosmetic slug,
  * omitted when blank and ignored on lookup.
+ *
+ * `resource` MUST stay the first query param: robots.ts re-allows canonical
+ * drawer URLs via the literal prefix rule `Allow: /search?resource=`, so
+ * reordering the params would silently block them from crawlers (these are
+ * the URLs the resources sitemap emits).
  */
 export const resourceDrawerSearch = (
   resourceId: number,