From 381939c79fffb9bd74b0a2c107463e5d56f45763 Mon Sep 17 00:00:00 2001 From: Claude Lin & Lay Date: Mon, 3 Aug 2026 12:14:01 +0900 Subject: [PATCH] fix(backfill): cap the Vectorize id batch at the getByIds limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VECTOR_BATCH_SIZE を 50 から 20 に下げ、根拠コメントを実測の上限に 書き直した。1000 は upsert 側の上限であり、getByIds は 20 で `VECTOR_GET_ERROR (code = 40007)` を返す。get / upsert 共通の 1 定数 なので、狭いほうの get に合わせれば両方を満たす。 stale が 20 を超える repo は最初の getByIds batch で throw し、D1 に 触れる前に呼び出し全体が失敗していた（本番の liplus-language / github-rag-mcp で発生）。 テスト側は getByIds stub を実物と同じく 21 件以上で 40007 を投げる ようにし、batch 分割の期待値を 20 刻みに合わせた上で、上限超過を 名指しで見張る回帰テストを 1 本追加した。 Closes #213 Co-Authored-By: Claude Opus 5 --- src/backfill-issue-state.test.ts | 26 +++++++++++++++++++++++++- src/backfill-issue-state.ts | 12 ++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/backfill-issue-state.test.ts b/src/backfill-issue-state.test.ts index 270f6d8..50731fa 100644 --- a/src/backfill-issue-state.test.ts +++ b/src/backfill-issue-state.test.ts @@ -8,6 +8,9 @@ import { const REPO = "acme/widgets"; +/** Vectorize's documented `getByIds` payload cap. */ +const VECTORIZE_GET_BY_IDS_MAX = 20; + /** Stub the GitHub `state=open` listing, paginating at 100 like the real API. */ function stubOpenListing(openNumbers: number[]) { const pages: Array> = []; @@ -84,6 +87,14 @@ function mkEnv( getByIds: vi.fn(async (ids: string[]) => { getBatchSizes.push(ids.length); if (opts.vectorizeThrows) throw new Error("vectorize down"); + // The real binding rejects the call outright past 20 IDs, so the stub does + // too — an oversized batch must fail the test that produced it (#213). + if (ids.length > VECTORIZE_GET_BY_IDS_MAX) { + throw new Error( + `VECTOR_GET_ERROR (code = 40007): too many ids in payload; ` + + `max id count is ${VECTORIZE_GET_BY_IDS_MAX}, got ${ids.length}`, + ); + } return ids .filter((id) => !(opts.missingVectors ?? []).includes(id)) .map((id) => ({ id, values: [0.1, 0.2], metadata: { repo: REPO, state: "open" } })); @@ -248,7 +259,20 @@ describe("backfill-issue-state: per-call budget", () => { await backfillIssueState(REPO, env, { limit: 200 }); - expect(getBatchSizes).toEqual([50, 50, 20]); + expect(getBatchSizes).toEqual([20, 20, 20, 20, 20, 20]); + }); + + it("never sends more IDs to getByIds than Vectorize accepts", async () => { + // `getByIds` caps the payload at 20 IDs (error 40007). A larger batch made the + // whole call throw before D1 was touched, so any repo with more than 20 stale + // rows was unrepairable (#213). + stubOpenListing([]); + const { env, getBatchSizes } = mkEnv(openRows(53)); + + await backfillIssueState(REPO, env, { limit: 200 }); + + expect(Math.max(...getBatchSizes)).toBeLessThanOrEqual(VECTORIZE_GET_BY_IDS_MAX); + expect(getBatchSizes.reduce((a, b) => a + b, 0)).toBe(53); }); }); diff --git a/src/backfill-issue-state.ts b/src/backfill-issue-state.ts index d94847e..ced5135 100644 --- a/src/backfill-issue-state.ts +++ b/src/backfill-issue-state.ts @@ -37,10 +37,14 @@ export const DEFAULT_ISSUE_STATE_LIMIT = 200; /** Hard ceiling on the per-call row budget a caller may request. */ export const MAX_ISSUE_STATE_LIMIT = 1000; -/** Vector IDs per `getByIds` / `upsert` call. Well inside the documented 1000-vector - * batch cap, and small enough that one batch's payload (values + metadata) stays - * modest for a 1024-dimension index. */ -const VECTOR_BATCH_SIZE = 50; +/** Vector IDs per `getByIds` / `upsert` call. + * + * Set by the *smaller* of the two caps this loop touches: `getByIds` rejects more + * than 20 IDs per call (`VECTOR_GET_ERROR (code = 40007): too many ids in payload; + * max id count is 20`), while the 1000-vector batch cap applies to `upsert`. One + * constant serves both calls, so it has to satisfy the tighter one — a repo with + * more than 20 stale rows failed on its first `getByIds` batch otherwise (#213). */ +const VECTOR_BATCH_SIZE = 20; /** Items per page of the GitHub open-item listing. */ const OPEN_LIST_PER_PAGE = 100;