From 7e37e30d579353c7f07ef4360d362e6a59bd623d Mon Sep 17 00:00:00 2001 From: Julian <1041324235@qq.com> Date: Wed, 1 Jul 2026 15:22:12 +0800 Subject: [PATCH 1/2] feat: detect and parse CodeBuddy IDE usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing codebuddy parser only covered ~/.codebuddy/projects JSONL, so the Tencent CodeBuddy CN IDE went undetected — its per-message JSON logs live under CodeBuddyExtension/Data/**/CodeBuddyIDE/**/history/ //messages/. Add a new codebuddy-ide source (reported under the existing CodeBuddy tool) with a cross-platform probe, a dedicated parser that emits one record per conversation from the cumulative statsSnapshot, and a watermark cursor to skip already-imported conversations. Override the path with AIUSAGE_CODEBUDDY_IDE_PATH. --- CHANGELOG.md | 5 + CHANGELOG.zh-CN.md | 5 + packages/cli/src/commands/parse-codebuddy.ts | 246 ++++++++++++++++++ packages/cli/src/commands/parse.ts | 28 ++ packages/cli/src/discovery.ts | 58 ++++- packages/cli/src/watermark.ts | 11 +- .../tests/commands/parse-codebuddy.test.ts | 132 ++++++++++ 7 files changed, 483 insertions(+), 2 deletions(-) create mode 100644 packages/cli/src/commands/parse-codebuddy.ts create mode 100644 packages/cli/tests/commands/parse-codebuddy.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7764a358..f98bac4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- **CodeBuddy IDE support** — detect and parse the Tencent CodeBuddy IDE (including the CN variant), whose per-message JSON logs live under `CodeBuddyExtension/Data/**/CodeBuddyIDE/**/history///messages/`. The existing `codebuddy` JSONL parser only covered `~/.codebuddy/projects`, so IDE usage went undetected. Token usage is read from each conversation's cumulative `statsSnapshot` (cache-miss input, cached input, output). Reported under the existing **CodeBuddy** tool via a new `codebuddy-ide` source; override the path with `AIUSAGE_CODEBUDDY_IDE_PATH`. + ## [1.5.7] - 2026-06-25 ### Added diff --git a/CHANGELOG.zh-CN.md b/CHANGELOG.zh-CN.md index 446a358d..6b01b16a 100644 --- a/CHANGELOG.zh-CN.md +++ b/CHANGELOG.zh-CN.md @@ -5,6 +5,11 @@ 格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/)， 并遵循 [语义化版本控制](https://semver.org/lang/zh-CN/)。 +## [未发布] + +### 新增 +- **CodeBuddy IDE 支持** — 检测并解析腾讯 CodeBuddy IDE（含 CN 变体）。其逐条消息的 JSON 日志位于 `CodeBuddyExtension/Data/**/CodeBuddyIDE/**/history/<会话>/<对话>/messages/`。原有的 `codebuddy` JSONL 解析器仅覆盖 `~/.codebuddy/projects`，因此 IDE 用量此前无法被检测到。用量数据取自每个对话累计的 `statsSnapshot`（未命中缓存的输入、缓存输入、输出）。归入现有的 **CodeBuddy** 工具，新增 `codebuddy-ide` 数据源；可用 `AIUSAGE_CODEBUDDY_IDE_PATH` 覆盖路径。 + ## [1.5.7] - 2026-06-25 ### 新增 diff --git a/packages/cli/src/commands/parse-codebuddy.ts b/packages/cli/src/commands/parse-codebuddy.ts new file mode 100644 index 00000000..386abaf5 --- /dev/null +++ b/packages/cli/src/commands/parse-codebuddy.ts @@ -0,0 +1,246 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs' +import { join, basename, dirname } from 'node:path' +import type { StatsRecord } from '@aiusage/core' +import { generateRecordId, inferProvider, calculateCost } from '@aiusage/core' + +export interface CodeBuddyImportOptions { + dataDir: string // .../CodeBuddyExtension/Data + device: string + deviceInstanceId: string + platform?: string + now: number + cursor?: number // Unix ms; skip conversations last active at/before this + exchangeRate?: number +} + +export interface CodeBuddyImportResult { + records: StatsRecord[] + nextCursor: number + errors: string[] +} + +function num(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) ? value : 0 +} + +function toMs(iso: unknown): number { + if (typeof iso !== 'string') return 0 + const ms = Date.parse(iso) + return Number.isFinite(ms) ? ms : 0 +} + +/** + * The CodeBuddy IDE stores each chat message as its own JSON file: + * /CodeBuddyIDE//history///messages/*.json + * Only the final assistant message of each agent turn carries usage stats, under a + * stringified `extra` field. `statsSnapshot` is a running (conversation-cumulative) + * snapshot of input/cache tokens, so we take the latest snapshot per conversation and + * emit a single record for it to avoid double-counting across turns. + */ +interface MessageFile { + role: string + message?: string + extra?: string + createdAt?: string +} + +interface ConversationUsage { + ts: number + model: string + inputTokens: number + outputTokens: number + cacheReadTokens: number + cacheWriteTokens: number + thinkingTokens: number + cwd?: string +} + +const WORKSPACE_FOLDER_RE = /Workspace Folder:\s*(.+)/ + +/** Extract the plain text of a user message (its `message` field is stringified JSON). */ +function userMessageText(raw: string): string { + try { + const inner = JSON.parse(raw) as { content?: Array<{ type?: string; text?: string }> } + if (Array.isArray(inner.content)) { + return inner.content + .filter((block) => block?.type === 'text' && typeof block.text === 'string') + .map((block) => block.text) + .join('\n') + } + } catch { + // Fall back to the raw string if it is not JSON. + } + return raw +} + +/** Recursively collect every `messages` directory under a CodeBuddyIDE data root. */ +function findMessagesDirs(root: string, depth = 0): string[] { + if (depth > 12) return [] + let entries + try { + entries = readdirSync(root, { withFileTypes: true }) + } catch { + return [] + } + const out: string[] = [] + for (const entry of entries) { + if (!entry.isDirectory()) continue + const full = join(root, entry.name) + if (entry.name === 'messages') { + out.push(full) + continue + } + out.push(...findMessagesDirs(full, depth + 1)) + } + return out +} + +/** Read and summarize one conversation's messages directory into a single usage record. */ +function readConversation(messagesDir: string): ConversationUsage | null { + let files: string[] + try { + files = readdirSync(messagesDir).filter((f) => f.endsWith('.json')) + } catch { + return null + } + if (files.length === 0) return null + + const parsed: MessageFile[] = [] + for (const file of files) { + try { + const data = JSON.parse(readFileSync(join(messagesDir, file), 'utf-8')) as MessageFile + if (data && typeof data === 'object') parsed.push(data) + } catch { + // Skip unreadable/partial message files. + } + } + if (parsed.length === 0) return null + + // Workspace folder (cwd) is embedded in the user prompt's block. + let cwd: string | undefined + for (const msg of parsed) { + if (msg.role !== 'user' || typeof msg.message !== 'string') continue + const match = WORKSPACE_FOLDER_RE.exec(userMessageText(msg.message)) + if (match) { + cwd = match[1].trim() + break + } + } + + // Latest assistant message carrying a stats snapshot wins (cumulative snapshot). + let latest: { ts: number; extra: Record } | null = null + for (const msg of parsed) { + if (msg.role !== 'assistant' || typeof msg.extra !== 'string') continue + let extra: Record + try { + extra = JSON.parse(msg.extra) as Record + } catch { + continue + } + const hasStats = extra.statsSnapshot != null || extra.lastStepInputTokens != null + if (!hasStats) continue + const ts = toMs(msg.createdAt) + if (!latest || ts >= latest.ts) latest = { ts, extra } + } + if (!latest) return null + + const extra = latest.extra + const snapshot = (extra.statsSnapshot ?? {}) as Record + + // Prefer the cumulative snapshot; fall back to per-step fields when absent. + const cacheReadTokens = num(snapshot.cachedInputTokens ?? extra.lastStepCachedInputTokens) + const inputTokens = snapshot.cacheMissTokens != null + ? num(snapshot.cacheMissTokens) + : Math.max(0, num(extra.lastStepInputTokens) - cacheReadTokens) + const outputTokens = num(snapshot.lastOutputTokens ?? extra.lastStepOutputTokens) + const cacheWriteTokens = num(snapshot.cacheWriteTokens) + const thinkingTokens = num(snapshot.thinkingTokens) + + const model = typeof extra.modelId === 'string' && extra.modelId.trim() + ? extra.modelId.trim() + : 'unknown' + + return { + ts: latest.ts, + model, + inputTokens, + outputTokens, + cacheReadTokens, + cacheWriteTokens, + thinkingTokens, + cwd, + } +} + +export function runParseCodeBuddy(options: CodeBuddyImportOptions): CodeBuddyImportResult { + const { dataDir, device, deviceInstanceId, platform, now, cursor, exchangeRate } = options + const records: StatsRecord[] = [] + const errors: string[] = [] + let nextCursor = cursor ?? 0 + + if (!existsSync(dataDir)) { + return { records, nextCursor, errors } + } + + let messagesDirs: string[] + try { + messagesDirs = findMessagesDirs(dataDir) + } catch (e) { + return { records, nextCursor, errors: [String(e)] } + } + + for (const messagesDir of messagesDirs) { + const conversationDir = dirname(messagesDir) + const sessionId = basename(conversationDir) + let usage: ConversationUsage | null + try { + usage = readConversation(messagesDir) + } catch (e) { + errors.push(`${messagesDir}: ${e instanceof Error ? e.message : e}`) + continue + } + if (!usage) continue + if (usage.inputTokens + usage.outputTokens + usage.cacheReadTokens === 0) continue + + // Skip conversations already imported (last activity at or before the cursor). + if (cursor && usage.ts <= cursor) continue + if (usage.ts > nextCursor) nextCursor = usage.ts + + const provider = inferProvider(usage.model) + const tokenArgs = { + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + cacheReadTokens: usage.cacheReadTokens, + cacheWriteTokens: usage.cacheWriteTokens, + thinkingTokens: usage.thinkingTokens, + } + const cost = calculateCost(usage.model, tokenArgs, exchangeRate) + const recordId = generateRecordId(deviceInstanceId, conversationDir, usage.ts) + + records.push({ + id: recordId, + ts: usage.ts, + ingestedAt: now, + updatedAt: now, + lineOffset: 0, + tool: 'codebuddy', + model: usage.model, + provider, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + cacheReadTokens: usage.cacheReadTokens, + cacheWriteTokens: usage.cacheWriteTokens, + thinkingTokens: usage.thinkingTokens, + cost, + costSource: cost > 0 ? 'pricing' : 'unknown', + sessionId, + sourceFile: conversationDir, + cwd: usage.cwd, + device, + deviceInstanceId, + platform, + }) + } + + return { records, nextCursor, errors } +} diff --git a/packages/cli/src/commands/parse.ts b/packages/cli/src/commands/parse.ts index 9f8a9513..2a1d1709 100644 --- a/packages/cli/src/commands/parse.ts +++ b/packages/cli/src/commands/parse.ts @@ -22,6 +22,7 @@ import { runParseZed } from './parse-zed.js' import { runParseKiro } from './parse-kiro.js' import { runParseZcode } from './parse-zcode.js' import { runParseTrae } from './parse-trae.js' +import { runParseCodeBuddy } from './parse-codebuddy.js' import type { ProgressInfo } from '../progress.js' interface ParseResult { @@ -1054,6 +1055,33 @@ export async function runParse(db: Database.Database, filterTool?: string, optio onProgress({ phase: 'Parsing SQLite', tool: 'trae', current: 1, total: 1, records: parsedCount, toolCalls: toolCallCount }) } + // CodeBuddy IDE: per-message JSON files under CodeBuddyExtension/Data. + const codeBuddyIdeDir = getDbPath('codebuddy-ide') ?? '' + if ((!filterTool || filterTool === 'codebuddy') && existsSync(codeBuddyIdeDir)) { + try { + const cbCursor = wm.getCodeBuddyIdeCursor() + const result = runParseCodeBuddy({ + dataDir: codeBuddyIdeDir, + device, + deviceInstanceId, + platform: devicePlatform, + now: Date.now(), + cursor: cbCursor, + exchangeRate, + }) + for (const record of result.records) insertRecord(db, record) + parsedCount += result.records.length + errors.push(...result.errors) + if (result.nextCursor > cbCursor) { + wm.setCodeBuddyIdeCursor(result.nextCursor) + wm.save() + } + onProgress({ phase: 'Parsing logs', tool: 'codebuddy', current: 1, total: 1, records: parsedCount, toolCalls: toolCallCount }) + } catch (e) { + errors.push(`${codeBuddyIdeDir}: ${e instanceof Error ? e.message : e}`) + } + } + // Fix historical records that were parsed before init created state.json. // If the current device UUID is known, backfill any records with 'unknown' device_instance_id. if (deviceInstanceId !== 'unknown') { diff --git a/packages/cli/src/discovery.ts b/packages/cli/src/discovery.ts index 2e299197..92aea009 100644 --- a/packages/cli/src/discovery.ts +++ b/packages/cli/src/discovery.ts @@ -352,6 +352,59 @@ function probeCodeBuddy(ctx: ProbeContext): string | null { return existsSync(dir) ? dir : null } +/** CodeBuddy IDE (Tencent) stores per-message JSON under CodeBuddyExtension/Data. */ +function codeBuddyIdeRoots(ctx: ProbeContext): string[] { + const roots: string[] = [] + if (platform() === 'darwin') { + roots.push(join(ctx.home, 'Library', 'Application Support', 'CodeBuddyExtension', 'Data')) + } else if (platform() === 'win32') { + const appData = ctx.env.APPDATA ?? join(ctx.home, 'AppData', 'Roaming') + roots.push(join(appData, 'CodeBuddyExtension', 'Data')) + } else { + const config = ctx.env.XDG_CONFIG_HOME ?? join(ctx.home, '.config') + roots.push(join(config, 'CodeBuddyExtension', 'Data')) + } + return unique(roots) +} + +function probeCodeBuddyIde(ctx: ProbeContext): string | null { + const override = envOverride('codebuddy-ide', ctx.env) + if (override) return override + const legacy = ctx.legacySources?.['codebuddy-ide'] + if (legacy) return legacy + for (const root of codeBuddyIdeRoots(ctx)) { + if (existsSync(root)) return root + } + return null +} + +/** Count CodeBuddy IDE conversations that contain at least one message file. */ +function countCodeBuddyIdeConversations(dir: string): number { + let count = 0 + const walk = (root: string, depth: number): void => { + if (depth > 12) return + let entries + try { + entries = readdirSync(root, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + if (!entry.isDirectory()) continue + const full = join(root, entry.name) + if (entry.name === 'messages') { + try { + if (readdirSync(full).some((f) => f.endsWith('.json'))) count++ + } catch {} + continue + } + walk(full, depth + 1) + } + } + walk(dir, 0) + return count +} + function probeKiro(ctx: ProbeContext): string | null { const override = envOverride('kiro', ctx.env) if (override) return override @@ -607,6 +660,7 @@ const TOOL_REGISTRY: readonly ToolEntry[] = [ { tool: 'gemini', sourceKey: 'gemini', label: 'Gemini CLI', probe: probeGemini }, { tool: 'kimi', sourceKey: 'kimi', label: 'Kimi Code', probe: probeKimi }, { tool: 'codebuddy', sourceKey: 'codebuddy', label: 'CodeBuddy', probe: probeCodeBuddy }, + { tool: 'codebuddy', sourceKey: 'codebuddy-ide', label: 'CodeBuddy (IDE)', probe: probeCodeBuddyIde }, { tool: 'kiro', sourceKey: 'kiro', label: 'Kiro', probe: probeKiro }, { tool: 'grok', sourceKey: 'grok', label: 'Grok Build', probe: probeGrok }, { tool: 'antigravity', sourceKey: 'antigravity', label: 'Antigravity', probe: probeAntigravity }, @@ -654,7 +708,9 @@ export function discoverTools(env: NodeJS.ProcessEnv = process.env): DetectedToo try { const stat = statSync(detectedPath) if (stat.isDirectory()) { - if (entry.sourceKey === 'roocode' || entry.sourceKey === 'kilocode') { + if (entry.sourceKey === 'codebuddy-ide') { + fileCount += countCodeBuddyIdeConversations(detectedPath) + } else if (entry.sourceKey === 'roocode' || entry.sourceKey === 'kilocode') { fileCount += findJsonFiles(detectedPath).filter((p) => basename(p) === 'ui_messages.json').length } else if (entry.sourceKey === 'kelivo') { fileCount += findJsonFiles(detectedPath).filter((p) => basename(p) === 'chats.json').length diff --git a/packages/cli/src/watermark.ts b/packages/cli/src/watermark.ts index a65ed024..2abb68a7 100644 --- a/packages/cli/src/watermark.ts +++ b/packages/cli/src/watermark.ts @@ -55,6 +55,7 @@ export interface WatermarkState { zcode?: ZcodeCursor | null zcodeTools?: ZcodeToolCursor | null trae?: number | null + codebuddyIde?: number | null } /** @deprecated Use FileWatermarkData instead */ @@ -110,7 +111,7 @@ export class WatermarkManager { if (parsed && typeof parsed === 'object' && !('files' in parsed)) { return { files: { ...defaultFileData(), ...parsed } } } - return { files: { ...defaultFileData(), ...(parsed.files ?? {}) }, opencode: parsed.opencode ?? null, hermes: parsed.hermes ?? null, qoder: parsed.qoder ?? null, cursor: parsed.cursor ?? null, goose: parsed.goose ?? null, zed: parsed.zed ?? null, kiro: parsed.kiro ?? null, zcode: parsed.zcode ?? null, zcodeTools: parsed.zcodeTools ?? null, trae: parsed.trae ?? null } + return { files: { ...defaultFileData(), ...(parsed.files ?? {}) }, opencode: parsed.opencode ?? null, hermes: parsed.hermes ?? null, qoder: parsed.qoder ?? null, cursor: parsed.cursor ?? null, goose: parsed.goose ?? null, zed: parsed.zed ?? null, kiro: parsed.kiro ?? null, zcode: parsed.zcode ?? null, zcodeTools: parsed.zcodeTools ?? null, trae: parsed.trae ?? null, codebuddyIde: parsed.codebuddyIde ?? null } } catch { return { files: defaultFileData() } } @@ -221,4 +222,12 @@ export class WatermarkManager { setTraeLastImported(ts: number): void { this.data.trae = ts } + + getCodeBuddyIdeCursor(): number { + return this.data.codebuddyIde ?? 0 + } + + setCodeBuddyIdeCursor(ts: number): void { + this.data.codebuddyIde = ts + } } diff --git a/packages/cli/tests/commands/parse-codebuddy.test.ts b/packages/cli/tests/commands/parse-codebuddy.test.ts new file mode 100644 index 00000000..25a17bae --- /dev/null +++ b/packages/cli/tests/commands/parse-codebuddy.test.ts @@ -0,0 +1,132 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import Database from 'better-sqlite3' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { mkdirSync, writeFileSync, rmSync } from 'node:fs' +import { initializeDatabase } from '../../src/db/index.js' + +vi.mock('node:os', async () => { + const actual = await vi.importActual('node:os') + return { + ...actual, + homedir: () => join(tmpdir(), 'aiusage-parse-codebuddy-test'), + } +}) + +const { runParse } = await import('../../src/commands/parse.js') + +const testDir = join(tmpdir(), 'aiusage-parse-codebuddy-test') +const dataDir = join(testDir, 'cb-data') + +/** Build a conversation messages/ directory with the given message files. */ +function writeConversation(convId: string, messages: Record[]): void { + const messagesDir = join(dataDir, 'default', 'CodeBuddyIDE', 'ws', 'history', 'session-1', convId, 'messages') + mkdirSync(messagesDir, { recursive: true }) + messages.forEach((msg, i) => { + writeFileSync(join(messagesDir, `msg-${i}.json`), JSON.stringify(msg)) + }) +} + +function userMessage(text: string, cwd: string): Record { + return { + role: 'user', + message: JSON.stringify({ + role: 'user', + content: [{ type: 'text', text: `\nWorkspace Folder: ${cwd}\n\n\n\n${text}\n` }], + }), + id: 'u1', + createdAt: '2026-07-01T06:58:18.147Z', + } +} + +function assistantFinal(createdAt: string, extra: Record): Record { + return { role: 'assistant', message: '{}', id: 'a-final', extra: JSON.stringify(extra), createdAt } +} + +describe('runParse with CodeBuddy IDE data', () => { + let cacheDb: Database.Database + + beforeEach(() => { + rmSync(testDir, { recursive: true, force: true }) + mkdirSync(join(testDir, '.aiusage'), { recursive: true }) + writeFileSync(join(testDir, '.aiusage', 'watermark.json'), '{}') + writeFileSync(join(testDir, '.aiusage', 'config.json'), JSON.stringify({ + sources: { 'codebuddy-ide': dataDir }, + })) + cacheDb = new Database(':memory:') + initializeDatabase(cacheDb) + }) + + afterEach(() => { + cacheDb.close() + rmSync(testDir, { recursive: true, force: true }) + }) + + it('imports one record per conversation from the final statsSnapshot', async () => { + writeConversation('conv-1', [ + userMessage('帮我分析当前项目结构', '/Users/tjh/claude-projects/test'), + // Intermediate assistant/tool messages carry no usage stats. + { role: 'assistant', message: '{}', id: 'a1', extra: JSON.stringify({ modelId: 'deepseek-v4-flash' }), createdAt: '2026-07-01T06:58:21.819Z' }, + { role: 'tool', message: '{}', id: 't1', extra: '{}', createdAt: '2026-07-01T06:58:21.827Z' }, + assistantFinal('2026-07-01T06:58:36.818Z', { + modelId: 'deepseek-v4-flash', + lastStepInputTokens: 44551, + lastStepOutputTokens: 683, + lastStepCachedInputTokens: 28928, + statsSnapshot: { + cachedInputTokens: 66688, + cacheWriteTokens: 0, + cacheMissTokens: 44472, + thinkingTokens: 0, + lastOutputTokens: 683, + credit: 1.5, + }, + }), + ]) + + const result = await runParse(cacheDb, 'codebuddy') + + expect(result.errors).toHaveLength(0) + expect(result.parsedCount).toBe(1) + const row = cacheDb.prepare('SELECT tool, model, provider, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, thinking_tokens, session_id, cwd FROM records').get() as any + expect(row).toMatchObject({ + tool: 'codebuddy', + model: 'deepseek-v4-flash', + provider: 'deepseek', + input_tokens: 44472, + output_tokens: 683, + cache_read_tokens: 66688, + cache_write_tokens: 0, + thinking_tokens: 0, + session_id: 'conv-1', + cwd: '/Users/tjh/claude-projects/test', + }) + }) + + it('skips conversations with no usage stats', async () => { + writeConversation('empty-conv', [ + userMessage('hi', '/tmp/x'), + { role: 'assistant', message: '{}', id: 'a1', extra: JSON.stringify({ modelId: 'deepseek-v4-flash' }), createdAt: '2026-07-01T06:58:21.819Z' }, + ]) + + const result = await runParse(cacheDb, 'codebuddy') + + expect(result.parsedCount).toBe(0) + expect(cacheDb.prepare('SELECT COUNT(*) AS n FROM records').get()).toMatchObject({ n: 0 }) + }) + + it('is idempotent and does not double-count on re-parse', async () => { + writeConversation('conv-1', [ + userMessage('q', '/tmp/x'), + assistantFinal('2026-07-01T06:58:36.818Z', { + modelId: 'deepseek-v4-flash', + statsSnapshot: { cachedInputTokens: 100, cacheMissTokens: 200, cacheWriteTokens: 0, thinkingTokens: 0, lastOutputTokens: 50 }, + }), + ]) + + await runParse(cacheDb, 'codebuddy') + await runParse(cacheDb, 'codebuddy') + + expect(cacheDb.prepare('SELECT COUNT(*) AS n FROM records').get()).toMatchObject({ n: 1 }) + }) +}) From e8bc6e65686000a72a48ec62f59c756e944697a9 Mon Sep 17 00:00:00 2001 From: Julian <1041324235@qq.com> Date: Wed, 1 Jul 2026 15:39:22 +0800 Subject: [PATCH 2/2] fix: correct CodeBuddy CLI cached-token double-counting CodeBuddy CLI logs usage under both message.usage (Anthropic-shaped field names but OpenAI-style semantics, where input_tokens already includes cached tokens) and providerData.rawUsage. The generic parser assumed Anthropic semantics and added cache_read_input_tokens on top of the cache-inclusive input_tokens, inflating input by the cached amount (~100x on cache-heavy turns). Read the clean prompt_cache_hit/miss decomposition from rawUsage, falling back to message.usage with cache reads subtracted from input. Verified against real glm-5.1 CLI logs (input 27537 -> 273, 28923 -> 251) and covered by a core regression test. --- CHANGELOG.md | 3 + CHANGELOG.zh-CN.md | 3 + packages/core/src/parsers/generic-jsonl.ts | 51 +++++++++++++- packages/core/tests/codebuddy-cli.test.ts | 77 ++++++++++++++++++++++ 4 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 packages/core/tests/codebuddy-cli.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f98bac4c..9ba40bd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **CodeBuddy IDE support** — detect and parse the Tencent CodeBuddy IDE (including the CN variant), whose per-message JSON logs live under `CodeBuddyExtension/Data/**/CodeBuddyIDE/**/history///messages/`. The existing `codebuddy` JSONL parser only covered `~/.codebuddy/projects`, so IDE usage went undetected. Token usage is read from each conversation's cumulative `statsSnapshot` (cache-miss input, cached input, output). Reported under the existing **CodeBuddy** tool via a new `codebuddy-ide` source; override the path with `AIUSAGE_CODEBUDDY_IDE_PATH`. +### Fixed +- **CodeBuddy CLI double-counted cached tokens** — the CLI logs usage under both `message.usage` (Anthropic-shaped field names but OpenAI-style semantics, where `input_tokens` already includes cached tokens) and `providerData.rawUsage`. The generic parser assumed Anthropic semantics and added `cache_read_input_tokens` on top of the cache-inclusive `input_tokens`, inflating input by the cached amount (~100× on cache-heavy turns). CodeBuddy now reads the clean `prompt_cache_hit/miss` decomposition from `rawUsage` (falling back to `message.usage` with cache reads subtracted). + ## [1.5.7] - 2026-06-25 ### Added diff --git a/CHANGELOG.zh-CN.md b/CHANGELOG.zh-CN.md index 6b01b16a..ef805737 100644 --- a/CHANGELOG.zh-CN.md +++ b/CHANGELOG.zh-CN.md @@ -10,6 +10,9 @@ ### 新增 - **CodeBuddy IDE 支持** — 检测并解析腾讯 CodeBuddy IDE（含 CN 变体）。其逐条消息的 JSON 日志位于 `CodeBuddyExtension/Data/**/CodeBuddyIDE/**/history/<会话>/<对话>/messages/`。原有的 `codebuddy` JSONL 解析器仅覆盖 `~/.codebuddy/projects`，因此 IDE 用量此前无法被检测到。用量数据取自每个对话累计的 `statsSnapshot`（未命中缓存的输入、缓存输入、输出）。归入现有的 **CodeBuddy** 工具，新增 `codebuddy-ide` 数据源；可用 `AIUSAGE_CODEBUDDY_IDE_PATH` 覆盖路径。 +### 修复 +- **CodeBuddy CLI 缓存 token 重复计数** — CLI 同时把用量写在 `message.usage`（字段名是 Anthropic 风格，但语义是 OpenAI 风格——`input_tokens` 已包含缓存 token）和 `providerData.rawUsage` 里。通用解析器按 Anthropic 语义处理，在已含缓存的 `input_tokens` 之上又加了一遍 `cache_read_input_tokens`，导致输入被缓存量虚高（缓存密集的轮次可达约 100 倍）。现在 codebuddy 改为读取 `rawUsage` 中干净的 `prompt_cache_hit/miss` 分解（缺失时回退到 `message.usage` 并减去缓存读取部分）。 + ## [1.5.7] - 2026-06-25 ### 新增 diff --git a/packages/core/src/parsers/generic-jsonl.ts b/packages/core/src/parsers/generic-jsonl.ts index f3bbf6da..b9874336 100644 --- a/packages/core/src/parsers/generic-jsonl.ts +++ b/packages/core/src/parsers/generic-jsonl.ts @@ -30,13 +30,58 @@ function sanitizeModel(value: unknown, fallback: string): string { return model || fallback } -function usageFromAny(parsed: any): { +interface Usage { inputTokens: number outputTokens: number cacheReadTokens: number cacheWriteTokens: number thinkingTokens: number -} | null { +} + +/** + * CodeBuddy CLI logs usage under both `message.usage` (Anthropic-shaped field names + * but with OpenAI-style semantics — `input_tokens` INCLUDES cached tokens) and + * `providerData.rawUsage` (clean prompt_cache_hit/miss decomposition). The generic + * extractor assumes Anthropic semantics (input excludes cache), so it double-counts + * the cached tokens. Prefer rawUsage; otherwise subtract cache reads from input. + */ +function codeBuddyUsage(parsed: any): Usage | null { + const ru = parsed?.providerData?.rawUsage + if (ru && typeof ru === 'object') { + const hit = num(ru.prompt_cache_hit_tokens ?? ru.prompt_tokens_details?.cached_tokens) + const miss = ru.prompt_cache_miss_tokens + const prompt = num(ru.prompt_tokens ?? ru.input_tokens) + return { + inputTokens: miss != null ? num(miss) : Math.max(0, prompt - hit), + outputTokens: num(ru.completion_tokens ?? ru.output_tokens), + cacheReadTokens: hit, + cacheWriteTokens: num(ru.prompt_cache_write_tokens ?? ru.cache_creation_input_tokens), + thinkingTokens: num(ru.completion_thinking_tokens ?? ru.completion_tokens_details?.reasoning_tokens), + } + } + + const mu = parsed?.message?.usage + if (mu && typeof mu === 'object') { + const cacheRead = num(mu.cache_read_input_tokens ?? mu.prompt_tokens_details?.cached_tokens) + const rawInput = num(mu.input_tokens ?? mu.prompt_tokens) + return { + inputTokens: Math.max(0, rawInput - cacheRead), + outputTokens: num(mu.output_tokens ?? mu.completion_tokens), + cacheReadTokens: cacheRead, + cacheWriteTokens: num(mu.cache_creation_input_tokens ?? mu.cache_write_input_tokens), + thinkingTokens: num(mu.thinking_tokens ?? mu.reasoning_tokens), + } + } + + return null +} + +function usageFromAny(parsed: any, tool?: Tool): Usage | null { + if (tool === 'codebuddy') { + const cb = codeBuddyUsage(parsed) + if (cb) return cb + } + const usage = parsed?.message?.usage ?? parsed?.usage @@ -144,7 +189,7 @@ export class GenericJsonlParser implements Parser { const normalized = parsed?.type === 'context.append_loop_event' && parsed.event ? { ...parsed.event, time: parsed.time } : parsed if (!shouldAccept(this.tool, normalized)) return null - const usage = usageFromAny(normalized) + const usage = usageFromAny(normalized, this.tool) if (!usage) return null const total = usage.inputTokens + usage.outputTokens + usage.cacheReadTokens + usage.cacheWriteTokens + usage.thinkingTokens if (total === 0) return null diff --git a/packages/core/tests/codebuddy-cli.test.ts b/packages/core/tests/codebuddy-cli.test.ts new file mode 100644 index 00000000..794bae49 --- /dev/null +++ b/packages/core/tests/codebuddy-cli.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect } from 'vitest' +import { GenericJsonlParser } from '../src/parsers/generic-jsonl.js' +import type { ParseContext } from '../src/types.js' + +// CodeBuddy CLI writes usage under both `message.usage` (Anthropic-shaped names but +// OpenAI-style semantics — input_tokens INCLUDES cached tokens) and +// `providerData.rawUsage` (clean prompt_cache_hit/miss decomposition). The parser must +// not double-count the cached tokens as both input and cache-read. +describe('GenericJsonlParser codebuddy CLI usage', () => { + const parser = new GenericJsonlParser('codebuddy', 'codebuddy-unknown') + const ctx: ParseContext = { + sourceFile: '/tmp/cb.jsonl', + lineOffset: 0, + sessionId: 'session-1', + tool: 'codebuddy', + now: 1776738085700, + device: 'test-device', + deviceInstanceId: 'device-123', + } + + it('uses rawUsage hit/miss decomposition (no double-count of cached input)', () => { + const line = JSON.stringify({ + type: 'message', + role: 'assistant', + timestamp: '2026-07-01T07:28:00.000Z', + message: { + // Cache-inclusive input_tokens; naive parsing double-counts the 27264 cached. + usage: { input_tokens: 27537, output_tokens: 552, total_tokens: 28089, cache_read_input_tokens: 27264 }, + }, + providerData: { + model: 'glm-5.1', + rawUsage: { + prompt_tokens: 27537, + completion_tokens: 552, + prompt_cache_hit_tokens: 27264, + prompt_cache_miss_tokens: 273, + prompt_cache_write_tokens: 0, + completion_thinking_tokens: 0, + prompt_tokens_details: { cached_tokens: 27264 }, + }, + }, + }) + + const result = parser.parseLine(line, ctx) + expect(result).not.toBeNull() + expect(result!.record).toMatchObject({ + model: 'glm-5.1', + provider: 'zhipu', + inputTokens: 273, // non-cached only (prompt_cache_miss_tokens) + outputTokens: 552, + cacheReadTokens: 27264, // cached, counted once + cacheWriteTokens: 0, + thinkingTokens: 0, + }) + }) + + it('falls back to message.usage and subtracts cache reads from input', () => { + const line = JSON.stringify({ + type: 'message', + role: 'assistant', + timestamp: '2026-07-01T07:28:00.000Z', + providerData: { model: 'glm-5.1' }, + message: { + usage: { input_tokens: 28923, output_tokens: 106, cache_read_input_tokens: 28672 }, + }, + }) + + const result = parser.parseLine(line, ctx) + expect(result).not.toBeNull() + expect(result!.record).toMatchObject({ + model: 'glm-5.1', + inputTokens: 251, // 28923 - 28672 + outputTokens: 106, + cacheReadTokens: 28672, + }) + }) +})