diff --git a/packages/cli/src/commands/parse-cursor.ts b/packages/cli/src/commands/parse-cursor.ts index c3eef1e..4dfebe8 100644 --- a/packages/cli/src/commands/parse-cursor.ts +++ b/packages/cli/src/commands/parse-cursor.ts @@ -11,6 +11,7 @@ export interface CursorImportOptions { platform?: string now: number cursor: CursorCursor | null + onProgress?: (current: number, total: number, recordsFound: number) => void } export interface CursorImportResult { @@ -102,15 +103,30 @@ export function runParseCursor( db: Database.Database, options: CursorImportOptions, ): CursorImportResult { - const { dbPath, device, deviceInstanceId, platform, now, cursor } = options + const { dbPath, device, deviceInstanceId, platform, now, cursor, onProgress } = options const records: StatsRecord[] = [] const toolCalls: ToolCallRecord[] = [] const errors: string[] = [] let lastCursor: CursorCursor | null = null - // Fetch all composerData entries with content + // Gracefully skip databases without the expected table (e.g. residual dbs + // left behind by old/uninstalled Cursor versions) + const hasTable = db + .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'cursorDiskKV'`) + .get() + if (!hasTable) { + return { records, toolCalls, nextCursor: lastCursor, errors } + } + + // Fetch all composerData entries with content. + // Use an index-friendly range scan instead of LIKE: SQLite's LIKE is + // case-insensitive by default and cannot use the `key TEXT UNIQUE` index, + // so every LIKE prefix query degenerates into a full-table scan, which is + // fatal on multi-GB state.vscdb files. Cursor writes keys with exact + // lowercase prefixes, so binary range comparison matches the same rows. + // (';' is the ASCII character right after ':') const composerRows = db - .prepare(`SELECT key, CAST(value AS TEXT) as value FROM cursorDiskKV WHERE key LIKE 'composerData:%' AND length(value) > 100`) + .prepare(`SELECT key, CAST(value AS TEXT) as value FROM cursorDiskKV WHERE key >= 'composerData:' AND key < 'composerData;' AND length(value) > 100`) .all() as ComposerRow[] // Parse and filter composerData entries @@ -152,17 +168,25 @@ export function runParseCursor( return { records, toolCalls, nextCursor: lastCursor, errors } } - // Prepare statement to fetch all bubbles for a composer + // Prepare statement to fetch all bubbles for a composer. + // Range bounds instead of LIKE so the key index is used (see above); + // otherwise each composer triggers a full-table scan. const bubbleStmt = db.prepare( - `SELECT CAST(value AS TEXT) as value FROM cursorDiskKV WHERE key LIKE ?`, + `SELECT CAST(value AS TEXT) as value FROM cursorDiskKV WHERE key >= ? AND key < ?`, ) + let processed = 0 for (const composer of filtered) { lastCursor = { lastCreatedAt: composer.createdAt, lastId: composer.composerId } + processed += 1 + onProgress?.(processed, filtered.length, records.length) try { // Fetch all bubble entries for this conversation - const bubbleRows = bubbleStmt.all(`bubbleId:${composer.composerId}:%`) as { value: string }[] + const bubbleRows = bubbleStmt.all( + `bubbleId:${composer.composerId}:`, + `bubbleId:${composer.composerId};`, + ) as { value: string }[] let totalInputTokens = 0 let totalOutputTokens = 0 diff --git a/packages/cli/src/commands/parse.ts b/packages/cli/src/commands/parse.ts index edbbe4d..3f0442f 100644 --- a/packages/cli/src/commands/parse.ts +++ b/packages/cli/src/commands/parse.ts @@ -948,6 +948,7 @@ export async function runParse(db: Database.Database, filterTool?: string, optio try { const cursorDb = new Database(cursorDbPath, { readonly: true }) try { + let cursorProgressFired = false const result = runParseCursor(cursorDb, { dbPath: cursorDbPath, device, @@ -955,6 +956,10 @@ export async function runParse(db: Database.Database, filterTool?: string, optio platform: devicePlatform, now: Date.now(), cursor: wm.getCursorCursor(), + onProgress: (current, total, recordsFound) => { + cursorProgressFired = true + onProgress({ phase: 'Parsing SQLite', tool: 'cursor', current, total, records: parsedCount + recordsFound, toolCalls: toolCallCount }) + }, }) for (const record of result.records) insertRecord(db, record) @@ -966,7 +971,9 @@ export async function runParse(db: Database.Database, filterTool?: string, optio parsedCount += result.records.length toolCallCount += result.toolCalls.length errors.push(...result.errors) - onProgress({ phase: 'Parsing SQLite', tool: 'cursor', current: 1, total: 1, records: parsedCount, toolCalls: toolCallCount }) + if (!cursorProgressFired) { + onProgress({ phase: 'Parsing SQLite', tool: 'cursor', current: 1, total: 1, records: parsedCount, toolCalls: toolCallCount }) + } } finally { cursorDb.close() } diff --git a/packages/cli/tests/commands/parse-cursor.test.ts b/packages/cli/tests/commands/parse-cursor.test.ts index 5d6a42c..cf1c2e9 100644 --- a/packages/cli/tests/commands/parse-cursor.test.ts +++ b/packages/cli/tests/commands/parse-cursor.test.ts @@ -275,6 +275,78 @@ describe('runParseCursor', () => { expect(result.nextCursor).toBeNull() expect(result.errors).toHaveLength(0) }) + + it('skips gracefully when cursorDiskKV table is missing', () => { + // Residual dbs from old/uninstalled Cursor versions may lack the table; + // the parser must degrade gracefully instead of throwing. + const emptyDb = new Database(':memory:') + try { + const result = runParseCursor(emptyDb, BASE_OPTIONS) + + expect(result.records).toHaveLength(0) + expect(result.toolCalls).toHaveLength(0) + expect(result.nextCursor).toBeNull() + expect(result.errors).toHaveLength(0) + } finally { + emptyDb.close() + } + }) + + it('ignores unrelated key prefixes', () => { + // Large dbs contain many other prefixes (agentKv, checkpointId, ...); + // none of them may leak into the stats. + db.prepare(`INSERT INTO cursorDiskKV (key, value) VALUES (?, ?)`).run( + 'agentKv:some-agent-key', + JSON.stringify({ type: 2, tokenCount: { inputTokens: 999999, outputTokens: 999999 } }), + ) + db.prepare(`INSERT INTO cursorDiskKV (key, value) VALUES (?, ?)`).run( + 'checkpointId:cp-1', + JSON.stringify({ type: 2, tokenCount: { inputTokens: 888888, outputTokens: 888888 } }), + ) + insertComposer(db, 'conv-1', 1779400000000, {}) + insertBubble(db, 'conv-1', 'b-1', 2, 1000, 100) + + const result = runParseCursor(db, BASE_OPTIONS) + + expect(result.records).toHaveLength(1) + expect(result.records[0].inputTokens).toBe(1000) + expect(result.records[0].outputTokens).toBe(100) + }) + + it('reports progress per processed conversation', () => { + insertComposer(db, 'conv-1', 1779400000000, {}) + insertBubble(db, 'conv-1', 'b-1', 2, 1000, 100) + insertComposer(db, 'conv-2', 1779410000000, {}) + insertBubble(db, 'conv-2', 'b-2', 2, 2000, 200) + insertComposer(db, 'conv-3', 1779420000000, {}) + insertBubble(db, 'conv-3', 'b-3', 2, 3000, 300) + + const calls: [number, number, number][] = [] + const result = runParseCursor(db, { + ...BASE_OPTIONS, + onProgress: (current, total, recordsFound) => calls.push([current, total, recordsFound]), + }) + + expect(result.records).toHaveLength(3) + expect(calls.length).toBe(3) + expect(calls.map(([current]) => current)).toEqual([1, 2, 3]) + expect(calls.every(([, total]) => total === 3)).toBe(true) + }) + + it('does not report progress when there is nothing new to process', () => { + insertComposer(db, 'conv-old', 1779400000000, {}) + insertBubble(db, 'conv-old', 'b-1', 2, 1000, 100) + + const calls: [number, number, number][] = [] + const result = runParseCursor(db, { + ...BASE_OPTIONS, + cursor: { lastCreatedAt: 1779400000000, lastId: 'conv-old' }, + onProgress: (current, total, recordsFound) => calls.push([current, total, recordsFound]), + }) + + expect(result.records).toHaveLength(0) + expect(calls).toHaveLength(0) + }) }) describe('runParse with cursor', () => {