perf: shrink the lossless search index

plugins/codex-lcm/src/maintenance.ts

@@ -16,6 +16,7 @@ import {
   indexEventInTransaction,
   invalidateRawLogState,
   recordRawLogState,
+  SEARCH_INDEX_VACUUM_KEY,
   segmentsNeedRawJsonClearing,
 } from "./storage-persistence.ts";
 import { registerStoredEventReader } from "./stored-event.ts";
@@ -155,9 +156,24 @@ function maintenanceNeeded(config: LcmConfig): boolean {
   return manifest.migration?.complete === false
     || manifest.segments.some((record) => !record.compressed)
     || archivedPayloadClearingNeeded(config, manifest.segments.map((record) => record.id))
+    || searchIndexVacuumNeeded(config)
     || (config.retentionDays !== undefined && config.configError === undefined);
 }
 
+function searchIndexVacuumNeeded(config: LcmConfig): boolean {
+  if (!fs.existsSync(config.indexPath)) return false;
+  let db: DatabaseSync | undefined;
+  try {
+    db = new DatabaseSync(config.indexPath, { readOnly: true });
+    return db.prepare("SELECT 1 FROM index_metadata WHERE key = ?1").get(SEARCH_INDEX_VACUUM_KEY) !== undefined;
+  } catch (error) {
+    if (error instanceof Error) return true;
+    throw error;
+  } finally {
+    db?.close();
+  }
+}
+
 function archivedPayloadClearingNeeded(config: LcmConfig, segmentIds: readonly string[]): boolean {
   if (segmentIds.length === 0 || !fs.existsSync(config.indexPath)) return false;
   let db: DatabaseSync | undefined;
@@ -252,9 +268,11 @@ function maintainSegments(config: LcmConfig, now: () => Date): MaintenanceReport
     }
     const retention = expireSegments(config, db, now());
     errors.push(...retention.errors);
-    if (db && (cleared > 0 || retention.expired > 0)) {
+    const vacuumSearchIndex = db?.prepare("SELECT 1 FROM index_metadata WHERE key = ?1").get(SEARCH_INDEX_VACUUM_KEY) !== undefined;
+    if (db && (cleared > 0 || retention.expired > 0 || vacuumSearchIndex)) {
       db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
       db.exec("VACUUM");
+      if (vacuumSearchIndex) db.prepare("DELETE FROM index_metadata WHERE key = ?1").run(SEARCH_INDEX_VACUUM_KEY);
     }
     if (db && errors.length === 0) {
       const state = withRawLogLock(config.rawLogPath, () => segmentedRawLogState(config));
@@ -282,7 +300,7 @@ function expireSegments(
     try {
       invalidateRawLogState(db);
       for (const record of expired) {
-        db.prepare("DELETE FROM event_fts WHERE event_id IN (SELECT event_id FROM events WHERE segment_id = ?1)").run(record.id);
+        db.prepare("DELETE FROM event_fts WHERE rowid IN (SELECT rowid FROM events WHERE segment_id = ?1)").run(record.id);
         db.prepare("DELETE FROM file_refs WHERE observed_event_id IN (SELECT event_id FROM events WHERE segment_id = ?1)").run(record.id);
         db.prepare("DELETE FROM events WHERE segment_id = ?1").run(record.id);
       }

plugins/codex-lcm/src/storage-pack.ts

@@ -111,7 +111,7 @@ function searchContextEvents(db: DatabaseSync | undefined, rawLogPath: string, a
   const statement = db.prepare(`
     SELECT lcm_raw_json(e.raw_json, e.segment_id, e.raw_offset, e.raw_length) AS raw_json
     FROM event_fts f
-    JOIN events e ON e.event_id = f.event_id
+    JOIN events e ON e.rowid = f.rowid
     WHERE event_fts MATCH ?1
       AND (?2 IS NULL OR e.cwd = ?2)
       AND e.hook_event IN ${SUMMARY_SOURCE_HOOKS}

plugins/codex-lcm/src/storage-persistence.ts

@@ -8,21 +8,28 @@ import { extractFileReferences } from "./file-refs.ts";
 import { overflowReferenceFromEvent } from "./overflow.ts";
 import { rawLogState, rawLogStat, readRawEventIds, readRawEvents, segmentedRawLogState, type RawEventLocation, type RawLogState } from "./raw-log.ts";
 import { eventSearchText } from "./storage-context.ts";
-import { recordValue } from "./storage-rows.ts";
-import { initializeStorageSchema } from "./storage-schema.ts";
+import { recordValue, rowToSessionMemorySummary, rowToSummaryNode } from "./storage-rows.ts";
+import { createSearchIndexTables, initializeStorageSchema } from "./storage-schema.ts";
 import { segmentStorageHealth } from "./raw-segments.ts";
 import { STORED_EVENT_JSON_SQL } from "./stored-event.ts";
 import { getSummaryBackfillSessionIds, rebuildSessionMemorySummary, shouldRebuildSessionMemorySummary } from "./storage-summaries.ts";
 import { extractEventMetadata, extractSessionMetadata, isCodexLcmToolEvent, isSearchIndexEvent, maxNullable, scalar, summarizeSessions } from "./storage-sessions.ts";
 import type { Health, IndexCleanupReport } from "./storage-types.ts";
-import { SUMMARY_ALGORITHM_VERSION, SUMMARY_NODE_VERSION, isSummarySourceEvent } from "./summary.ts";
+import {
+  SUMMARY_ALGORITHM_VERSION,
+  SUMMARY_NODE_VERSION,
+  isSummarySourceEvent,
+  summaryNodeSearchText,
+  summarySearchText,
+} from "./summary.ts";
 
 const SUMMARY_SOURCE_HOOKS = "('UserPromptSubmit', 'Note', 'Stop', 'PreCompact', 'PostCompact')";
 const FILE_REF_BACKFILL_KEY = "file_refs_backfilled_v1";
 const DELEGATION_PARENT_BACKFILL_KEY = "delegation_parent_backfilled_v1";
 const EVENT_METADATA_BACKFILL_KEY = "event_metadata_backfilled_v1";
 const EVENT_LOCATOR_METADATA_BACKFILL_KEY = "event_locator_metadata_backfilled_v1";
 const RAW_LOG_INDEX_STATE_KEY = "raw_log_index_state_v1";
+export const SEARCH_INDEX_VACUUM_KEY = "search_index_vacuum_v1";
 
 export type IndexEventResult = { readonly inserted: boolean; readonly summaryTouched: boolean };
 export type RawEventIdCache = {
@@ -122,12 +129,14 @@ export function previewCleanupReport(indexPath: string, inspection: CleanupInspe
 
 export function replaceCleanupSearchIndex(db: DatabaseSync, searchableEvents: readonly NormalizedEvent[]): void {
   db.prepare("DELETE FROM event_fts").run();
+  const selectRowId = db.prepare("SELECT rowid FROM events WHERE event_id = ?1");
   const insertSearchEvent = db.prepare(`
-    INSERT INTO event_fts (event_id, session_id, cwd, repo_root, hook_event, content)
-    VALUES (?1, ?2, ?3, ?4, ?5, ?6)
+    INSERT INTO event_fts (rowid, event_id, session_id, cwd, repo_root, hook_event, content)
+    VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
   `);
   for (const event of searchableEvents) {
-    insertSearchEvent.run(event.event_id, event.session_id, event.cwd, event.repo_root ?? "", event.hook_event, eventSearchText(event));
+    const rowId = Number(recordValue(selectRowId.get(event.event_id)).rowid);
+    insertSearchEvent.run(rowId, event.event_id, event.session_id, event.cwd, event.repo_root ?? "", event.hook_event, eventSearchText(event));
   }
   db.prepare("UPDATE events SET text = '' WHERE text <> ''").run();
 }
@@ -272,6 +281,70 @@ export function initializeIndex(db: DatabaseSync): void {
   const { backfillSessionMetadata } = initializeStorageSchema(db);
   backfillExistingEventMetadata(db);
   if (backfillSessionMetadata) backfillExistingSessionMetadata(db);
+  migrateSearchIndexes(db);
+}
+
+function migrateSearchIndexes(db: DatabaseSync): boolean {
+  const names = ["event_fts", "session_summary_fts", "summary_node_fts"] as const;
+  const schemas = db.prepare(`
+    SELECT name, sql FROM sqlite_master
+    WHERE type = 'table' AND name IN ('event_fts', 'session_summary_fts', 'summary_node_fts')
+  `).all();
+  if (names.every((name) => schemas.some((row) => {
+    const record = recordValue(row);
+    return record.name === name && String(record.sql).includes("contentless_delete=1");
+  }))) return false;
+
+  db.exec("BEGIN IMMEDIATE");
+  try {
+    const eventRows = db.prepare(`SELECT rowid, event_id, ${STORED_EVENT_JSON_SQL} AS raw_json FROM events ORDER BY rowid`).all();
+    db.exec("DROP TABLE event_fts; DROP TABLE session_summary_fts; DROP TABLE summary_node_fts");
+    createSearchIndexTables(db);
+    const insertEvent = db.prepare(`
+      INSERT INTO event_fts (rowid, event_id, session_id, cwd, repo_root, hook_event, content)
+      VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
+    `);
+    for (const row of eventRows) {
+      const record = recordValue(row);
+      const event = decodePersistedEvent(String(record.raw_json));
+      if (event.event_id !== String(record.event_id)) {
+        throw new Error(`Stored event locator mismatch for ${String(record.event_id)}.`);
+      }
+      if (!isSearchIndexEvent(event) || isCodexLcmToolEvent(event)) continue;
+      insertEvent.run(
+        Number(record.rowid), event.event_id, event.session_id, event.cwd,
+        event.repo_root ?? "", event.hook_event, eventSearchText(event),
+      );
+    }
+    const insertSummary = db.prepare(`
+      INSERT INTO session_summary_fts (rowid, session_id, cwd, repo_root, content)
+      VALUES (?1, ?2, ?3, ?4, ?5)
+    `);
+    for (const row of db.prepare("SELECT rowid, * FROM session_summaries ORDER BY rowid").all()) {
+      const summary = rowToSessionMemorySummary(row);
+      insertSummary.run(
+        Number(recordValue(row).rowid), summary.session_id, summary.cwd,
+        summary.repo_root ?? "", summarySearchText(summary),
+      );
+    }
+    const insertNode = db.prepare(`
+      INSERT INTO summary_node_fts (rowid, node_id, session_id, cwd, repo_root, depth, content)
+      VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
+    `);
+    for (const row of db.prepare("SELECT rowid, * FROM summary_nodes ORDER BY rowid").all()) {
+      const node = rowToSummaryNode(row);
+      insertNode.run(
+        Number(recordValue(row).rowid), node.node_id, node.session_id, node.cwd,
+        node.repo_root ?? "", String(node.depth), summaryNodeSearchText(node),
+      );
+    }
+    db.prepare("INSERT OR REPLACE INTO index_metadata (key, value) VALUES (?1, 'pending')").run(SEARCH_INDEX_VACUUM_KEY);
+    db.exec("COMMIT");
+    return true;
+  } catch (error) {
+    if (db.isTransaction) db.exec("ROLLBACK");
+    throw error;
+  }
 }
 
 export function indexEventInTransaction(
@@ -329,9 +402,9 @@ export function indexEventInTransaction(
   );
   if (isSearchIndexEvent(event) && !isCodexLcmToolEvent(event)) {
     db.prepare(`
-      INSERT INTO event_fts (event_id, session_id, cwd, repo_root, hook_event, content)
-      VALUES (?1, ?2, ?3, ?4, ?5, ?6)
-    `).run(event.event_id, event.session_id, event.cwd, event.repo_root ?? "", event.hook_event, eventSearchText(event));
+      INSERT INTO event_fts (rowid, event_id, session_id, cwd, repo_root, hook_event, content)
+      VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
+    `).run(insert.lastInsertRowid, event.event_id, event.session_id, event.cwd, event.repo_root ?? "", event.hook_event, eventSearchText(event));
   }
   indexFileRefsForEvent(db, event);
   const summaryTouched = isSummarySourceEvent(event);

plugins/codex-lcm/src/storage-schema.ts

@@ -46,15 +46,6 @@ export function initializeStorageSchema(db: DatabaseSync): SchemaInitialization
       agent_id TEXT,
       overflow_sha256 TEXT
     );
-    CREATE VIRTUAL TABLE IF NOT EXISTS event_fts USING fts5(
-      event_id UNINDEXED,
-      session_id,
-      cwd,
-      repo_root,
-      hook_event,
-      content
-    );
-
     CREATE TABLE IF NOT EXISTS session_summaries (
       session_id TEXT PRIMARY KEY,
       summary_version INTEGER NOT NULL DEFAULT ${SUMMARY_ALGORITHM_VERSION},
@@ -71,12 +62,6 @@ export function initializeStorageSchema(db: DatabaseSync): SchemaInitialization
       source_event_ids_json TEXT NOT NULL,
       summary_text TEXT NOT NULL
     );
-    CREATE VIRTUAL TABLE IF NOT EXISTS session_summary_fts USING fts5(
-      session_id UNINDEXED,
-      cwd,
-      repo_root,
-      content
-    );
     CREATE TABLE IF NOT EXISTS summary_nodes (
       node_id TEXT PRIMARY KEY,
       session_id TEXT NOT NULL,
@@ -112,20 +97,13 @@ export function initializeStorageSchema(db: DatabaseSync): SchemaInitialization
       key TEXT PRIMARY KEY,
       value TEXT NOT NULL
     );
-    CREATE VIRTUAL TABLE IF NOT EXISTS summary_node_fts USING fts5(
-      node_id UNINDEXED,
-      session_id,
-      cwd,
-      repo_root,
-      depth,
-      content
-    );
     CREATE INDEX IF NOT EXISTS idx_session_summaries_updated ON session_summaries(updated_at);
     CREATE INDEX IF NOT EXISTS idx_summary_nodes_session_depth_latest ON summary_nodes(session_id, depth, latest_at);
     CREATE INDEX IF NOT EXISTS idx_summary_nodes_session_latest ON summary_nodes(session_id, latest_at);
     CREATE INDEX IF NOT EXISTS idx_file_refs_session_time ON file_refs(session_id, timestamp);
     CREATE INDEX IF NOT EXISTS idx_file_refs_path ON file_refs(path);
   `);
+  createSearchIndexTables(db);
   ensureColumn(db, "events", "turn_id", "TEXT");
   ensureColumn(db, "events", "tool_use_id", "TEXT");
   ensureColumn(db, "events", "segment_id", "TEXT");
@@ -161,6 +139,39 @@ export function initializeStorageSchema(db: DatabaseSync): SchemaInitialization
   return { backfillSessionMetadata };
 }
 
+export function createSearchIndexTables(db: DatabaseSync): void {
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS event_fts USING fts5(
+      event_id UNINDEXED,
+      session_id,
+      cwd,
+      repo_root,
+      hook_event,
+      content,
+      content='',
+      contentless_delete=1
+    );
+    CREATE VIRTUAL TABLE IF NOT EXISTS session_summary_fts USING fts5(
+      session_id UNINDEXED,
+      cwd,
+      repo_root,
+      content,
+      content='',
+      contentless_delete=1
+    );
+    CREATE VIRTUAL TABLE IF NOT EXISTS summary_node_fts USING fts5(
+      node_id UNINDEXED,
+      session_id,
+      cwd,
+      repo_root,
+      depth,
+      content,
+      content='',
+      contentless_delete=1
+    );
+  `);
+}
+
 function ensureColumn(db: DatabaseSync, table: string, column: string, type: string): boolean {
   const tableName = sqlIdentifier(table);
   const columnName = sqlIdentifier(column);

plugins/codex-lcm/src/storage-search.ts

@@ -338,7 +338,7 @@ export function searchStoredSessions(db: DatabaseSync | undefined, rawLogPath: s
            e.timestamp AS match_timestamp, 1 AS match_weight,
            'event' AS match_kind, e.event_id AS match_event_id
     FROM event_fts f
-    JOIN events e ON e.event_id = f.event_id
+    JOIN events e ON e.rowid = f.rowid
     JOIN sessions s ON s.session_id = e.session_id
     WHERE event_fts MATCH ?1
       AND (?2 IS NULL OR s.cwd = ?2)
@@ -353,7 +353,7 @@ export function searchStoredSessions(db: DatabaseSync | undefined, rawLogPath: s
            'session_summary' AS match_kind, ss.topics_json AS match_topics_json,
            ss.source_event_ids_json AS match_source_event_ids_json
     FROM session_summary_fts f
-    JOIN session_summaries ss ON ss.session_id = f.session_id
+    JOIN session_summaries ss ON ss.rowid = f.rowid
     JOIN sessions s ON s.session_id = ss.session_id
     WHERE session_summary_fts MATCH ?1
       AND (?2 IS NULL OR s.cwd = ?2)
@@ -369,7 +369,7 @@ export function searchStoredSessions(db: DatabaseSync | undefined, rawLogPath: s
            n.source_event_ids_json AS match_source_event_ids_json,
            n.source_token_count AS match_source_token_count
     FROM summary_node_fts f
-    JOIN summary_nodes n ON n.node_id = f.node_id
+    JOIN summary_nodes n ON n.rowid = f.rowid
     JOIN sessions s ON s.session_id = n.session_id
     WHERE summary_node_fts MATCH ?1
       AND (?2 IS NULL OR s.cwd = ?2)

plugins/codex-lcm/src/storage-summaries.ts

@@ -158,7 +158,7 @@ export function searchSummaryNodes(db: DatabaseSync | undefined, args: SummaryNo
            n.source_type, n.source_ids_json, n.source_event_ids_json, n.earliest_at, n.latest_at,
            n.created_at, n.cwd, n.repo_root, n.git_branch, n.topics_json
     FROM summary_node_fts f
-    JOIN summary_nodes n ON n.node_id = f.node_id
+    JOIN summary_nodes n ON n.rowid = f.rowid
     WHERE summary_node_fts MATCH ?1
       AND (?2 IS NULL OR n.cwd = ?2)
       AND (?3 IS NULL OR n.repo_root = ?3)
@@ -297,14 +297,14 @@ export function rebuildSessionMemorySummary(db: DatabaseSync | undefined, sessio
   if (!db) return;
   const events = getSummaryEventsForSession(db, sessionId);
   if (events.length === 0) {
-    db.prepare("DELETE FROM session_summary_fts WHERE session_id = ?1").run(sessionId);
+    db.prepare("DELETE FROM session_summary_fts WHERE rowid IN (SELECT rowid FROM session_summaries WHERE session_id = ?1)").run(sessionId);
     db.prepare("DELETE FROM session_summaries WHERE session_id = ?1").run(sessionId);
     rebuildSummaryNodes(db, sessionId);
     return;
   }
   const summary = buildSessionMemorySummary(events);
   const summaryText = summarySearchText(summary);
-  db.prepare("DELETE FROM session_summary_fts WHERE session_id = ?1").run(sessionId);
+  db.prepare("DELETE FROM session_summary_fts WHERE rowid IN (SELECT rowid FROM session_summaries WHERE session_id = ?1)").run(sessionId);
   db.prepare(`
     INSERT INTO session_summaries
       (session_id, summary_version, updated_at, cwd, repo_root, git_branch, title, overview, topics_json,
@@ -341,8 +341,8 @@ export function rebuildSessionMemorySummary(db: DatabaseSync | undefined, sessio
     summaryText,
   );
   db.prepare(`
-    INSERT INTO session_summary_fts (session_id, cwd, repo_root, content)
-    VALUES (?1, ?2, ?3, ?4)
+    INSERT INTO session_summary_fts (rowid, session_id, cwd, repo_root, content)
+    SELECT rowid, ?1, ?2, ?3, ?4 FROM session_summaries WHERE session_id = ?1
   `).run(summary.session_id, summary.cwd, summary.repo_root ?? "", summaryText);
   rebuildSummaryNodes(db, sessionId);
 }
@@ -370,7 +370,7 @@ function rebuildSummaryNodes(db: DatabaseSync, sessionId: string): void {
     existing.set(String(record.node_id), Number(record.summary_version));
   }
   const nextIds = new Set(nodes.map((node) => node.node_id));
-  const deleteFts = db.prepare("DELETE FROM summary_node_fts WHERE node_id = ?1");
+  const deleteFts = db.prepare("DELETE FROM summary_node_fts WHERE rowid IN (SELECT rowid FROM summary_nodes WHERE node_id = ?1)");
   const deleteNode = db.prepare("DELETE FROM summary_nodes WHERE node_id = ?1");
   for (const nodeId of existing.keys()) {
     if (nextIds.has(nodeId)) continue;
@@ -428,8 +428,8 @@ function insertSummaryNode(db: DatabaseSync, node: SummaryNode): void {
     JSON.stringify(node.topics),
   );
   db.prepare(`
-    INSERT INTO summary_node_fts (node_id, session_id, cwd, repo_root, depth, content)
-    VALUES (?1, ?2, ?3, ?4, ?5, ?6)
+    INSERT INTO summary_node_fts (rowid, node_id, session_id, cwd, repo_root, depth, content)
+    SELECT rowid, ?1, ?2, ?3, ?4, ?5, ?6 FROM summary_nodes WHERE node_id = ?1
   `).run(
     node.node_id,
     node.session_id,