From 8fb17f8823a1eb8e19c46b0775ef048700db1e21 Mon Sep 17 00:00:00 2001 From: Claude Lin & Lay Date: Mon, 27 Apr 2026 18:48:12 +0900 Subject: [PATCH] fix(poller): cap pollComments fetches per repo per invocation (#134) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pollComments の per-repo fetch 数に上限を追加し、Cloudflare Worker の 1 起動あたり 1000 subrequest の上限を超過する問題を修正する。 これまでは MAX_COMMENT_BACKFILL_PARENTS=20 と MAX_COMMENTS_EMBEDDED_PER_REPO=30 の 2 段で制限していたが、各 parent は 3 endpoint (issue comments / PR reviews / PR review comments) に fan-out するため、最悪ケースで 1 repo あたり 60 fetch を 発行する。dipper_ai のように comment が多い repo では、他の poller (diff / wiki / issue) と合算して budget を使い切り、subrequest exhaustion で fetch が落ちる 事象が観測された (issue #134, 2026-04-26 cron)。 #130 / #131 の wiki 側修正と同じパターンで、embed cap とは独立した「probe(fetch) cap」を導入する。MAX_COMMENT_FETCHES_PER_REPO_PER_RUN = 30 を追加し、per-parent loop 内で fetch 発行のたびに counter を increment、上限到達で loop を打ち切る。 打ち切り時は warn log を出して観測しやすくする。 - 4 つの稼働中 repo (github-rag-mcp / github-webhook-mcp / liplus-language / liplus-desktop) は parent 数 × 3 が 30 を下回るため挙動退行はない。 - dipper_ai のように cap に到達する repo では、未処理 parent は次の cron で 自然に拾われる。 Closes #134 --- src/poller.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/poller.ts b/src/poller.ts index 5498a93..227523d 100644 --- a/src/poller.ts +++ b/src/poller.ts @@ -59,6 +59,18 @@ const MAX_COMMENT_BACKFILL_PARENTS = 20; * Workers AI embed calls are the dominant cost for the comment surface. */ const MAX_COMMENTS_EMBEDDED_PER_REPO = 30; +/** Maximum number of GitHub API *fetches* the comment poller issues per repo + * per cron run. Distinct from MAX_COMMENTS_EMBEDDED_PER_REPO because each + * parent fans out to up to 3 endpoints (issue comments + PR reviews + PR + * review comments) and every fetch consumes 1 of the Worker's + * 1000-subrequest-per-invocation budget — even on parents whose comments are + * unchanged and embed-skipped. With MAX_COMMENT_BACKFILL_PARENTS = 20 the + * worst-case fan-out is 60 fetches per repo, which combined with diff / wiki + * / issue pollers exhausts the budget on busy repos (issue #134, observed on + * Liplus-Project/dipper_ai). Capping fetches keeps the comment surface's + * worst-case bounded; remaining parents are picked up on the next cron. */ +const MAX_COMMENT_FETCHES_PER_REPO_PER_RUN = 30; + /** Maximum number of commits fetched in the forward (webhook-redundancy) phase * of the diff poller per repo per run. * Forward is normally a no-op because the webhook path already indexes new @@ -1190,15 +1202,24 @@ async function pollComments( let reviewCommentsSkipped = 0; let reviewCommentsFiltered = 0; let fetchFailures = 0; + let fetchesIssued = 0; + let fetchBudgetExhausted = false; const embedBudget = (): boolean => commentsEmbedded + reviewsEmbedded + reviewCommentsEmbedded < MAX_COMMENTS_EMBEDDED_PER_REPO; + const fetchBudget = (): boolean => fetchesIssued < MAX_COMMENT_FETCHES_PER_REPO_PER_RUN; + for (const parent of parents) { if (!embedBudget()) break; + if (!fetchBudget()) { + fetchBudgetExhausted = true; + break; + } // Top-level comments (issues and PRs both route through /issues/{N}/comments) try { + fetchesIssued++; const comments = await fetchIssueComments(repo, parent.number, env.GITHUB_TOKEN); for (const c of comments) { if (!embedBudget()) break; @@ -1219,8 +1240,13 @@ async function pollComments( if (!isPullRequestRecord(parent)) continue; if (!embedBudget()) break; + if (!fetchBudget()) { + fetchBudgetExhausted = true; + break; + } try { + fetchesIssued++; const reviews = await fetchPRReviews(repo, parent.number, env.GITHUB_TOKEN); for (const r of reviews) { if (!embedBudget()) break; @@ -1238,8 +1264,13 @@ async function pollComments( } if (!embedBudget()) break; + if (!fetchBudget()) { + fetchBudgetExhausted = true; + break; + } try { + fetchesIssued++; const inline = await fetchPRReviewComments(repo, parent.number, env.GITHUB_TOKEN); for (const rc of inline) { if (!embedBudget()) break; @@ -1257,8 +1288,17 @@ async function pollComments( } } + if (fetchBudgetExhausted) { + console.warn( + `pollComments: fetch budget reached for ${repo} ` + + `(${MAX_COMMENT_FETCHES_PER_REPO_PER_RUN} fetches). Each parent fans out to up ` + + `to 3 endpoints; remaining parents are deferred to the next cron run.`, + ); + } + console.log( `${repo} comments: scanned ${parents.length} parents, ` + + `fetches_issued=${fetchesIssued}/${MAX_COMMENT_FETCHES_PER_REPO_PER_RUN}, ` + `top-level [embedded=${commentsEmbedded}, skipped=${commentsSkipped}, filtered=${commentsFiltered}], ` + `reviews [embedded=${reviewsEmbedded}, skipped=${reviewsSkipped}, filtered=${reviewsFiltered}], ` + `inline [embedded=${reviewCommentsEmbedded}, skipped=${reviewCommentsSkipped}, filtered=${reviewCommentsFiltered}], ` +