From 2caab525abdf2aad0cef4724cf00dcd7647c1cf4 Mon Sep 17 00:00:00 2001
From: liplus-lin-lay <liplus-lin-lay@users.noreply.github.com>
Date: Sun, 5 Apr 2026 22:50:06 +0900
Subject: [PATCH] fix: cap pagination per cron run to prevent CPU time limit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fetchAllIssues に maxPages パラメータを追加し、1回の cron 実行で
取得するページ数を MAX_PAGES_PER_RUN=5（500件）に制限。
900+ issue のリポジトリで初回同期時に Cloudflare Worker の
CPU 時間制限を超過する問題を解決する。

ページ上限到達時は watermark を pollStartTime ではなく
最後に取得した issue の updated_at に設定し、次回 cron で
続きから取得できるようにした。

Refs #30

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 src/poller.ts | 61 ++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 51 insertions(+), 10 deletions(-)

diff --git a/src/poller.ts b/src/poller.ts
index 1f5ddc0..cf2b618 100644
--- a/src/poller.ts
+++ b/src/poller.ts
@@ -35,6 +35,13 @@ const PER_PAGE = 100;
  *  Remaining issues are stored with empty bodyHash and retried next cron. */
 const MAX_EMBEDDINGS_PER_RUN = 50;
 
+/** Maximum number of API pages to fetch per single cron run.
+ *  Prevents Cloudflare Worker CPU time limit on large repos (e.g. 900+ issues initial sync).
+ *  At PER_PAGE=100, this caps a single run at 500 issues.
+ *  When capped, the watermark is set to the last fetched issue's updated_at
+ *  so the next cron continues from where it left off. */
+const MAX_PAGES_PER_RUN = 5;
+
 /**
  * Compute SHA-256 hash of title + body for change detection.
  * Returns hex-encoded hash string.
@@ -106,14 +113,17 @@ async function fetchIssuePage(
 }
 
 /**
- * Fetch all issues (with pagination) from GitHub API since a given timestamp.
- * For initial sync (no since), fetches all issues.
+ * Fetch issues (with pagination) from GitHub API since a given timestamp.
+ * When maxPages is provided, stops after that many pages to stay within
+ * Cloudflare Worker CPU time limits. Returns a `capped` flag indicating
+ * whether pagination was truncated before exhausting all results.
  */
 async function fetchAllIssues(
   repo: string,
   token: string,
   since?: string,
-): Promise<GitHubIssue[]> {
+  maxPages?: number,
+): Promise<{ issues: GitHubIssue[]; capped: boolean }> {
   const allIssues: GitHubIssue[] = [];
   let page = 1;
 
@@ -127,14 +137,23 @@ async function fetchAllIssues(
     if (!hasMore) break;
     page++;
 
-    // Safety: cap at 50 pages (5000 issues) to prevent runaway loops
+    // Stop if we've reached the per-run page cap
+    if (maxPages && page > maxPages) {
+      console.warn(
+        `Pagination capped at ${maxPages} pages (${allIssues.length} issues) for ${repo}. ` +
+        `Remaining issues will be fetched in subsequent cron runs.`,
+      );
+      return { issues: allIssues, capped: true };
+    }
+
+    // Safety: absolute cap to prevent runaway loops
     if (page > 50) {
-      console.warn(`Pagination cap reached for ${repo} at page ${page}`);
-      break;
+      console.warn(`Absolute pagination cap reached for ${repo} at page ${page}`);
+      return { issues: allIssues, capped: true };
     }
   }
 
-  return allIssues;
+  return { issues: allIssues, capped: false };
 }
 
 /**
@@ -311,8 +330,13 @@ async function pollRepo(
   // Record poll start time before fetching (to avoid missing updates during fetch)
   const pollStartTime = new Date().toISOString();
 
-  // Fetch issues from GitHub API
-  const issues = await fetchAllIssues(repo, env.GITHUB_TOKEN, since);
+  // Fetch issues from GitHub API (with per-run page cap)
+  const { issues, capped } = await fetchAllIssues(
+    repo,
+    env.GITHUB_TOKEN,
+    since,
+    MAX_PAGES_PER_RUN,
+  );
 
   if (issues.length === 0) {
     console.log(`No updates for ${repo}`);
@@ -330,12 +354,29 @@ async function pollRepo(
   // Process issues (embedding + store)
   const stats = await processIssues(issues, repo, env, storeStub);
 
+  // Watermark strategy:
+  // - If all pages were fetched (not capped): use pollStartTime so next run
+  //   picks up anything updated during this fetch.
+  // - If pagination was capped: use the updated_at of the last fetched issue
+  //   (sorted by updated asc) so the next cron continues from where we left off.
+  //   Using pollStartTime here would skip the remaining unfetched issues.
+  let nextWatermark: string;
+  if (capped) {
+    const lastIssue = issues[issues.length - 1];
+    nextWatermark = lastIssue.updated_at;
+    console.log(
+      `${repo}: pagination was capped — watermark set to last fetched issue updated_at: ${nextWatermark}`,
+    );
+  } else {
+    nextWatermark = pollStartTime;
+  }
+
   // Update watermark after successful processing
   await storeStub.fetch(
     new Request("http://store/watermark", {
       method: "POST",
       headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ repo, lastPolledAt: pollStartTime }),
+      body: JSON.stringify({ repo, lastPolledAt: nextWatermark }),
     }),
   );