From 7a48f7909a030b59402631a1d660239c56384e4a Mon Sep 17 00:00:00 2001
From: Claude Lin & Lay <eightborder@gmail.com>
Date: Sun, 26 Apr 2026 09:59:05 +0900
Subject: [PATCH] hotfix(poller): cap wiki ingestion subrequest fan-out (#130)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

W4 (#128 / PR #129) deploy 後、Cloudflare Worker logs に
"Too many subrequests by single Worker invocation" が大量発生。
1 cron invocation で 5 repos × N pages × 10 extension probe を
直列実行して per-invocation 1000 subrequest 上限を突破した。

修正:
- WIKI_EXTENSIONS を主流 2 種 (md / markdown) に縮小。
  mediawiki / org / rst / pod / textile / asciidoc / creole は
  per-page probe ごとに 1 subrequest を消費するので、bulk import
  worst case を予測可能にするため一旦削減。cached extension が
  ある page は影響なし (1 fetch で済む既存ロジック維持)。
  follow-up: 上記 8 種を per-repo opt-in flag 経由で再導入候補。

- MAX_WIKI_PAGES_PROBED_PER_REPO_PER_RUN = 20 を新設。
  pollWiki 内で per-repo の probe 数を硬く capping。bulk import を
  複数 cron run に自然分散させ、1 invocation の subrequest 総量を
  ~110 ((20 pages × 1-2 probes + wikiExists + listPages) × 5 repos)
  に抑える。Cloudflare Workers Paid (1000/invocation) で安全マージン。

挙動上の変更:
- 大規模 wiki (100+ pages) の初回 import は複数 cron run に分散
  (20 pages/run × 4 runs/hour = 80 pages/hour/repo)
- mediawiki / org / rst 等の wiki page は本 hotfix では index されない
  (該当する wiki がプロジェクト内に現状存在しないため実害ゼロ)

Closes #130
---
 src/poller.ts | 49 ++++++++++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/src/poller.ts b/src/poller.ts
index 2e98b54..86b9d3c 100644
--- a/src/poller.ts
+++ b/src/poller.ts
@@ -788,28 +788,37 @@ async function pollDocs(
 // ── Wiki doc poller ──────────────────────────────────────────
 
 /**
- * Wiki page markup file extensions that GitHub Wiki natively supports
- * (https://github.com/gollum/gollum). Sorted by popularity so the first hit
- * on a fresh page is usually the first probe.
+ * Wiki page markup file extensions probed when no cached extension is available.
+ *
+ * GitHub Wiki natively supports more formats (mediawiki / org / rst / pod /
+ * textile / asciidoc / creole) but the per-page extension probe is a pure
+ * subrequest cost — every miss eats 1 of the Worker's 1000-per-invocation
+ * subrequest budget (issue #130). Markdown is the dominant format in
+ * practice; restricting the probe set to `md` + `markdown` keeps the
+ * worst-case subrequest fan-out predictable while still covering every wiki
+ * we currently care about. Once a page is ingested the actual extension is
+ * cached on its WikiDocRecord and reused on subsequent polls regardless of
+ * this probe-set narrowing.
+ *
+ * Follow-up: re-introduce the rarer extensions behind a per-repo opt-in flag
+ * so the bulk-import worst case stays bounded. Tracked in #130 follow-up.
  */
-const WIKI_EXTENSIONS = [
-  "md",
-  "markdown",
-  "mediawiki",
-  "org",
-  "rst",
-  "rest",
-  "textile",
-  "pod",
-  "asciidoc",
-  "creole",
-] as const;
+const WIKI_EXTENSIONS = ["md", "markdown"] as const;
 
 /** Maximum wiki pages embedded per repo per cron run.
  *  Caps Workers AI embed budget the same way MAX_EMBEDDINGS_PER_RUN does for
  *  repository docs. Remaining changed pages are picked up on the next cron. */
 const MAX_WIKI_EMBEDDINGS_PER_RUN = 30;
 
+/** Maximum wiki pages whose content we *probe* (HTTP fetch) per repo per cron
+ *  run. Distinct from MAX_WIKI_EMBEDDINGS_PER_RUN because probing alone
+ *  consumes Worker subrequests even when the page has not changed (we still
+ *  fetch the raw content to compare hashes). On bulk import this is the
+ *  dominant subrequest cost — cap it so 5+ repos with deep wikis cannot
+ *  exhaust the per-invocation 1000-subrequest ceiling (issue #130). Pages
+ *  beyond the cap are deferred to the next cron run. */
+const MAX_WIKI_PAGES_PROBED_PER_REPO_PER_RUN = 20;
+
 /**
  * Probe whether a repo has a wiki at all.
  *
@@ -971,6 +980,7 @@ async function pollWiki(
   let skipped = 0;
   let failed = 0;
   let removed = 0;
+  let probed = 0;
 
   for (const pageName of pageSlugs) {
     if (embedded >= MAX_WIKI_EMBEDDINGS_PER_RUN) {
@@ -981,7 +991,16 @@ async function pollWiki(
       break;
     }
 
+    if (probed >= MAX_WIKI_PAGES_PROBED_PER_REPO_PER_RUN) {
+      console.warn(
+        `Wiki probe batch limit reached for ${repo} (${MAX_WIKI_PAGES_PROBED_PER_REPO_PER_RUN}). ` +
+          `Each probe consumes a Worker subrequest; bulk imports spread across multiple cron runs.`,
+      );
+      break;
+    }
+
     const prior = existingMap.get(pageName);
+    probed++;
     const fetched = await fetchWikiContent(repo, pageName, prior?.extension);
     if (!fetched) {
       // The slug was discovered in `_pages` but no extension served. Treat as