From 3d24277039aa00764a0d0c4b0b3d50b73a9cad11 Mon Sep 17 00:00:00 2001 From: ankit-thebigred Date: Sun, 26 Jul 2026 12:58:20 +0400 Subject: [PATCH] fix(db): stop rebuilding the FTS index on every start, and page the rebuild Closes #37. Two compounding bugs made the gateway unable to start on a large database. 1. The "is the index already populated?" probe could never return true. messages_fts is created as fts5(text_content, content=''), a contentless table, so SQLite stores no column values and reading one back always returns NULL regardless of how many rows are indexed. `!sample?.text_content` was therefore always true, the full-rebuild branch ran on every launch, and the fts_version check directly above it never had any effect. The first thing that branch does is DROP TABLE, so a healthy index was destroyed each start. Reproduction, no app required: CREATE VIRTUAL TABLE t USING fts5(text_content, content=''); INSERT INTO t(rowid, text_content) VALUES (1, 'hello world'); SELECT text_content FROM t LIMIT 1; -- NULL, looks empty SELECT COUNT(*) FROM t; -- 1 Fixed by counting rows. 2. The rebuild materialised the entire history in one .all(). Every user and assistant message, content JSON included, was loaded into a single array before indexing. On a large history that exhausts the V8 heap, and because the table was already dropped, each respawn restarted from an empty index and OOMed again. The incremental path had the same shape. Fixed by paging both paths through a shared helper with a keyset cursor, so peak memory is flat regardless of history size. Paging rather than .iterate() is deliberate: better-sqlite3 refuses to run a write while an iterator is open on the same connection ("This database connection is busy executing a query"), so an open iterator with batched inserts throws. Verified against the bundled better-sqlite3. Verified on a scratch database with the same schema: a 5,000 message rebuild spanning three pages indexes 5,000 rows, non user/assistant types stay excluded, a second call re-indexes nothing, ten newly inserted messages are picked up incrementally, and MATCH queries still return the expected rowids. --- src/db.ts | 89 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 32 deletions(-) diff --git a/src/db.ts b/src/db.ts index 785ef79..464aa14 100644 --- a/src/db.ts +++ b/src/db.ts @@ -257,53 +257,78 @@ export function backfillFtsIndex(): void { const currentVersion = versionRow ? parseInt(versionRow.value, 10) : 0; const needsRebuild = currentVersion < FTS_VERSION; - const sample = d.prepare("SELECT text_content FROM messages_fts LIMIT 1").get() as { text_content: string | null } | undefined; + // messages_fts is a contentless FTS5 table (content=''), so SQLite does not store the + // column values and reading one back always yields NULL no matter how many rows are + // indexed. Probing the column therefore reported "empty" on every start, forced a full + // rebuild each time and left the fts_version check above unreachable. Count rows. + const sample = d.prepare('SELECT COUNT(*) AS n FROM messages_fts').get() as { n: number } | undefined; - if (needsRebuild || !sample?.text_content) { + if (needsRebuild || !sample?.n) { // drop and recreate for clean rebuild d.exec('DROP TABLE IF EXISTS messages_fts'); d.exec("CREATE VIRTUAL TABLE messages_fts USING fts5(text_content, content='', tokenize='porter unicode61')"); console.error(`[db] rebuilding FTS index (v${FTS_VERSION})...`); - const rows = d.prepare('SELECT id, content, type FROM messages WHERE type IN (\'user\', \'assistant\') ORDER BY id').all() as { id: number; content: string; type: string }[]; - - const insert = d.prepare('INSERT INTO messages_fts(rowid, text_content) VALUES (?, ?)'); - const tx = d.transaction(() => { - let indexed = 0; - for (const row of rows) { - const text = extractMessageText(row.content); - if (!text || text.length < 5) continue; - insert.run(row.id, text); - indexed++; - } - d.prepare("INSERT OR REPLACE INTO db_meta (key, value) VALUES ('fts_version', ?)").run(String(FTS_VERSION)); - console.error(`[db] FTS rebuild complete (v${FTS_VERSION}): ${indexed}/${rows.length} messages indexed`); - }); - tx(); + const total = indexMessagesInPages( + "SELECT id, content FROM messages WHERE type IN ('user', 'assistant') AND id > ? ORDER BY id LIMIT ?" + ); + d.prepare("INSERT OR REPLACE INTO db_meta (key, value) VALUES ('fts_version', ?)").run(String(FTS_VERSION)); + console.error(`[db] FTS rebuild complete (v${FTS_VERSION}): ${total} messages indexed`); return; } // incremental: index any messages missing from FTS (only user + assistant, skip result) - const missing = d.prepare(` - SELECT m.id, m.content, m.type FROM messages m - WHERE m.type IN ('user', 'assistant') - AND m.id NOT IN (SELECT rowid FROM messages_fts) - ORDER BY m.id - `).all() as { id: number; content: string; type: string }[]; - - if (missing.length === 0) return; + const indexed = indexMessagesInPages( + `SELECT m.id, m.content FROM messages m + WHERE m.type IN ('user', 'assistant') + AND m.id > ? + AND m.id NOT IN (SELECT rowid FROM messages_fts) + ORDER BY m.id + LIMIT ?` + ); + + if (indexed > 0) { + console.error(`[db] incremental FTS backfill complete: ${indexed} messages indexed`); + } +} - console.error(`[db] incremental FTS backfill: ${missing.length} messages to index...`); +/** + * Index messages into messages_fts one bounded page at a time. + * + * pageSql must select id and content, take a keyset cursor as its first parameter and a + * page size as its second, and order by id. Paging keeps peak memory flat regardless of + * history size. The previous code materialised every user and assistant message in one + * .all(), which on a large database exhausted the V8 heap and crash-looped the gateway. + * + * This pages rather than streaming with .iterate() on purpose: better-sqlite3 refuses to + * run a write while an iterator is open on the same connection, throwing "This database + * connection is busy executing a query", so an open iterator plus batched inserts is not + * an option here. + */ +function indexMessagesInPages(pageSql: string, pageSize = 2000): number { + const d = getDb(); + const page = d.prepare(pageSql); const insert = d.prepare('INSERT INTO messages_fts(rowid, text_content) VALUES (?, ?)'); - const tx = d.transaction(() => { - let indexed = 0; - for (const row of missing) { + + const writePage = d.transaction((rows: { id: number; content: string }[]): number => { + let n = 0; + for (const row of rows) { const text = extractMessageText(row.content); if (!text || text.length < 5) continue; insert.run(row.id, text); - indexed++; + n++; } - console.error(`[db] incremental FTS backfill complete: ${indexed}/${missing.length} messages indexed`); + return n; }); - tx(); + + let cursor = 0; + let indexed = 0; + for (;;) { + const rows = page.all(cursor, pageSize) as { id: number; content: string }[]; + if (rows.length === 0) break; + indexed += writePage(rows); + cursor = rows[rows.length - 1].id; + if (rows.length < pageSize) break; + } + return indexed; }