diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 7ab71c9..5de1c56 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -222,15 +222,25 @@ Configure at two levels: The controller automatically detects and scans all installed tools: -| Tool | Provider | Log Location | -|------|----------|-------------| -| Claude Code | Anthropic | `~/.claude/projects/` | -| Codex CLI | OpenAI | `~/.codex/sessions/` | -| Copilot CLI | GitHub | `~/.copilot/session-state/` | +| Tool | Provider | Usage Source | +|------|----------|--------------| +| Claude Code | Anthropic / compatible wrappers | `~/.config/claude/projects/`, `~/.claude/projects/` | +| Codex CLI | OpenAI | `~/.codex/sessions/`, `~/.codex/archived_sessions/` | +| Cursor | Cursor | Local `state.vscdb` credential + Cursor usage CSV API | +| Copilot CLI | GitHub | `~/.copilot/otel/`, `~/.copilot/session-state/` | +| Copilot for VS Code | GitHub | VS Code logs and `User/workspaceStorage/**/chatSessions/*.{json,jsonl}` | | Gemini CLI | Google | `~/.gemini/tmp/` | -| Amp | Sourcegraph | `~/.local/share/amp/threads/` | -| Kimi Code | Moonshot | `$KIMI_CODE_HOME/sessions/`（默认 `~/.kimi-code/sessions/`） | -| Kimi CLI（旧版） | Moonshot | `~/.kimi/sessions/` | -| Qwen Code | Alibaba | `~/.qwen/tmp/` | -| Droid | — | `~/.factory/sessions/` | -| OpenCode | — | `~/.local/share/opencode/` | +| Antigravity | Google | `~/.gemini/antigravity/` interaction metadata | +| Amp | Model provider (Sourcegraph product) | `~/.local/share/amp/threads/` | +| Kimi Code | Moonshot | `$KIMI_CODE_HOME/sessions/` (default `~/.kimi-code/sessions/`) | +| Kimi CLI (legacy) | Moonshot | `~/.kimi/sessions/` | +| Qwen Code | Alibaba | `~/.qwen/projects/`, legacy `~/.qwen/tmp/` | +| Droid | Factory | `~/.factory/sessions/*.settings.json` | +| OpenCode | OpenCode | `~/.local/share/opencode/opencode.db`, legacy `storage/message/` | +| Pi / Oh My Pi | Multiple | `~/.pi/agent/sessions/`, `~/.omp/agent/sessions/` | + +The scanner implementations are compatibility-audited against the overlapping +MIT-licensed [tokscale](https://github.com/junhoyeo/tokscale) parsers. Some +legacy IDE artifacts expose interaction timestamps but no reliable token +counters; AIUsage keeps those as zero-token activity instead of estimating +billable usage from conversation content. diff --git a/packages/cli/README.md b/packages/cli/README.md index ff771b4..23febd4 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -9,12 +9,31 @@ - scheduling automatic sync to an AIUsage Worker - diagnosing configuration and connectivity issues -Kimi scanning supports both legacy Kimi CLI data in `~/.kimi/sessions/` and -new Kimi Code data in `$KIMI_CODE_HOME/sessions/` (default: -`~/.kimi-code/sessions/`). It reads only token counters and session metadata -from `wire.jsonl`; conversation content is not uploaded. The new wire-format -parser references the MIT-licensed [tokscale](https://github.com/junhoyeo/tokscale) -implementation. +The local scanners have been compatibility-audited against the overlapping +parsers in the MIT-licensed [tokscale](https://github.com/junhoyeo/tokscale) +project. AIUsage keeps tool-specific safeguards where the source semantics +differ, rather than treating every local record as billable token usage. + +## Local scanner coverage + +| Tool | Sources and compatibility behavior | +|------|------------------------------------| +| Claude Code | `~/.config/claude/projects/` and `~/.claude/projects/` JSONL; deduplicates parent/sidechain replays, merges streaming snapshots per token field, and honors wrapper providers. Aggregate `stats-cache.json` is intentionally not converted into guessed per-message usage. | +| Codex CLI | Active and archived `~/.codex` sessions; fork-aware replay boundaries, inherited baselines, `last_token_usage`, and total-delta fallback. | +| Cursor | Reads the local `state.vscdb` credential and requests Cursor's token-strategy usage CSV; the database is snapshotted when locked. | +| Copilot CLI | OpenTelemetry JSONL under `~/.copilot/otel/` plus `session-state` shutdown totals; granular inference spans supersede same-trace aggregates. | +| Copilot for VS Code | Chat logs, legacy session JSON, and modern CRDT `workspaceStorage/**/chatSessions/*.jsonl`; modern sessions contribute real token fields, while legacy records remain interaction-only. | +| Gemini CLI | Session JSON/JSONL, headless stats, and `$set.messages` updates under `~/.gemini/tmp/`, with last-write-wins request deduplication. | +| Antigravity | `~/.gemini/antigravity/brain` and browser-recording metadata; currently reports interaction counts because those artifacts do not expose reliable token counters. | +| Amp | `~/.local/share/amp/threads/`; reconciles the usage ledger with message usage so partial ledgers are completed without double-counting. | +| Kimi CLI / Kimi Code | Legacy `~/.kimi/sessions/` and `$KIMI_CODE_HOME/sessions/` (default `~/.kimi-code/sessions/`) `wire.jsonl`; handles progressive status snapshots and nested agent sessions. | +| Qwen Code | Current `~/.qwen/projects/` and legacy `~/.qwen/tmp/` chat JSONL, with session/position deduplication and cache-aware input accounting. | +| Droid | `~/.factory/sessions/*.settings.json`; uses persisted token totals first and the transcript only as a model fallback. | +| OpenCode | OpenCode 1.2+ `opencode.db` through read-only `sqlite3`, plus legacy `storage/message/*.json`. | +| Pi / Oh My Pi | `~/.pi/agent/sessions/` and `~/.omp/agent/sessions/` JSONL, including provider, cache-write, and session metadata. | + +Only token counters and session metadata are aggregated or uploaded. Conversation +content and local credentials are never uploaded. ## Install diff --git a/packages/cli/package.json b/packages/cli/package.json index 439e4d2..cd746d5 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@aiusage/cli", - "version": "1.7.3", + "version": "1.7.4", "description": "Track and sync AI tool token usage across devices.", "license": "MIT", "private": false, diff --git a/packages/cli/src/doctor.ts b/packages/cli/src/doctor.ts index e5b63e9..8840711 100644 --- a/packages/cli/src/doctor.ts +++ b/packages/cli/src/doctor.ts @@ -92,7 +92,7 @@ async function countFiles(dir: string, exts: string[], cap = 1000): Promise { const g3 = s.groupTools; const home = homedir(); const tools: ToolDef[] = [ - { dir: join(home, '.claude', 'projects'), label: 'Claude Code', exts: ['.jsonl'] }, - { dir: join(home, '.codex'), label: 'Codex CLI', exts: ['.jsonl'] }, - { dir: join(home, 'Library', 'Application Support', 'Cursor', 'User', 'globalStorage'), label: 'Cursor', exts: ['.vscdb'] }, - { dir: join(home, '.copilot', 'session-state'), label: 'Copilot CLI', exts: ['.jsonl'] }, - { dir: join(home, 'Library', 'Application Support', 'Code', 'logs'), label: 'Copilot VS Code', exts: ['.log'] }, - { dir: join(home, '.gemini', 'tmp'), label: 'Gemini CLI', exts: ['.json'] }, - { dir: join(home, '.gemini', 'antigravity'), label: 'Antigravity', exts: ['.json'] }, - { dir: join(home, '.qwen', 'tmp'), label: 'Qwen Code', exts: ['.jsonl'] }, - { dir: join(resolveKimiCodeHome(home), 'sessions'), label: 'Kimi Code', exts: ['.jsonl'] }, - { dir: join(home, '.kimi', 'sessions'), label: 'Kimi CLI (legacy)', exts: ['.jsonl'] }, - { dir: join(home, '.local', 'share', 'amp', 'threads'), label: 'Amp', exts: ['.json'] }, - { dir: join(home, '.factory', 'sessions'), label: 'Droid', exts: ['.jsonl', '.json'] }, - { dir: join(home, '.local', 'share', 'opencode'), label: 'OpenCode', exts: ['.json'] }, - { dir: join(home, '.pi', 'agent', 'sessions'), label: 'Pi', exts: ['.jsonl'] }, + { dirs: [join(home, '.config', 'claude', 'projects'), join(home, '.claude', 'projects')], label: 'Claude Code', exts: ['.jsonl'] }, + { dirs: [join(home, '.codex')], label: 'Codex CLI', exts: ['.jsonl'] }, + { dirs: [join(home, 'Library', 'Application Support', 'Cursor', 'User', 'globalStorage')], label: 'Cursor', exts: ['.vscdb'] }, + { dirs: [join(home, '.copilot', 'session-state'), join(home, '.copilot', 'otel')], label: 'Copilot CLI', exts: ['.jsonl'] }, + { + dirs: [ + join(home, 'Library', 'Application Support', 'Code', 'logs'), + join(home, 'Library', 'Application Support', 'Code', 'User', 'workspaceStorage'), + ], + label: 'Copilot VS Code', + exts: ['.log', '.json', '.jsonl'], + }, + { dirs: [join(home, '.gemini', 'tmp')], label: 'Gemini CLI', exts: ['.json', '.jsonl'] }, + { dirs: [join(home, '.gemini', 'antigravity')], label: 'Antigravity', exts: ['.json'] }, + { dirs: [join(home, '.qwen', 'projects'), join(home, '.qwen', 'tmp')], label: 'Qwen Code', exts: ['.jsonl'] }, + { dirs: [join(resolveKimiCodeHome(home), 'sessions')], label: 'Kimi Code', exts: ['.jsonl'] }, + { dirs: [join(home, '.kimi', 'sessions')], label: 'Kimi CLI (legacy)', exts: ['.jsonl'] }, + { dirs: [join(home, '.local', 'share', 'amp', 'threads')], label: 'Amp', exts: ['.json'] }, + { dirs: [join(home, '.factory', 'sessions')], label: 'Droid', exts: ['.settings.json'] }, + { dirs: [join(home, '.local', 'share', 'opencode')], label: 'OpenCode', exts: ['.db', '.json'] }, + { dirs: [join(home, '.pi', 'agent', 'sessions'), join(home, '.omp', 'agent', 'sessions')], label: 'Pi / OMP', exts: ['.jsonl'] }, ]; for (const tool of tools) { - try { - await stat(tool.dir); - } catch { + let installed = false; + let n = 0; + for (const dir of tool.dirs) { + try { + await stat(dir); + installed = true; + n += await countFiles(dir, tool.exts); + } catch { + // 检查同一工具的其他兼容目录。 + } + } + if (!installed) { checks.push({ group: g3, name: tool.label, status: 'warn', message: s.notInstalled }); continue; } - const n = await countFiles(tool.dir, tool.exts); if (n > 0) { checks.push({ group: g3, name: tool.label, status: 'ok', message: s.hasData(n) }); } else { diff --git a/packages/cli/src/report.ts b/packages/cli/src/report.ts index a27b713..761eb23 100644 --- a/packages/cli/src/report.ts +++ b/packages/cli/src/report.ts @@ -1,11 +1,12 @@ import { readdir, readFile } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { basename, join } from 'node:path'; +import { basename, dirname, join } from 'node:path'; import type { IngestBreakdown } from '@aiusage/shared'; import { calculateCost, PRICING_VERSION, type PricingCatalog } from '@aiusage/shared'; import { scanDates } from './scan.js'; -import { parseTs, dateKey } from './scanners/utils.js'; +import { parseTs, dateKey, fileModifiedTs } from './scanners/utils.js'; import { resolveKimiCodeHome } from './scanners/kimi.js'; +import { discoverOpenCodeUsageDates } from './scanners/opencode.js'; import type { PricingInfo } from './pricing.js'; export type ReportRange = '7d' | '1m' | '3m' | 'all' | 'today'; @@ -176,14 +177,19 @@ async function discoverAllDates(): Promise { discoverCopilotVscodeDates(dates), discoverAntigravityDates(dates), discoverGenericJsonlDates(join(home, '.copilot', 'session-state'), dates), + discoverGenericJsonlDates(join(home, '.copilot', 'otel'), dates), discoverGenericJsonlDates(join(home, '.qwen', 'tmp'), dates), + discoverGenericJsonlDates(join(home, '.qwen', 'projects'), dates), discoverGenericJsonlDates(join(home, '.kimi', 'sessions'), dates), discoverGenericJsonlDates(join(resolveKimiCodeHome(home), 'sessions'), dates), discoverGenericJsonDates(join(home, '.local', 'share', 'amp', 'threads'), dates), - discoverGenericJsonlDates(join(home, '.factory', 'sessions'), dates), - discoverGenericJsonDates(join(home, '.local', 'share', 'opencode'), dates), + discoverGenericJsonDates(join(home, '.factory', 'sessions'), dates), + discoverOpenCodeUsageDates().then(found => found.forEach(date => dates.add(date))), discoverGenericJsonlDates(join(home, '.pi', 'agent', 'sessions'), dates), + discoverGenericJsonlDates(join(home, '.omp', 'agent', 'sessions'), dates), ]); + const explicitCopilotOtel = process.env.COPILOT_OTEL_FILE_EXPORTER_PATH?.trim(); + if (explicitCopilotOtel) await discoverJsonlFileDates(explicitCopilotOtel, dates); return [...dates].sort(); } @@ -194,14 +200,28 @@ async function discoverGenericJsonlDates(baseDir: string, dates: Set): P for (const filePath of files) { const content = await safeReadUtf8(filePath); if (!content) continue; + let foundDate = false; for (const line of content.split('\n')) { if (!line.trim()) continue; - let record: { timestamp?: string | number; time?: string | number; created_at?: string | number }; + let record: Record; try { record = JSON.parse(line); } catch { continue; } - const ts = parseTs(record.timestamp ?? record.time ?? record.created_at); - if (ts) dates.add(dateKey(ts)); + foundDate = collectRecordDates(record, dates) || foundDate; } + if (!foundDate) await addFileModifiedDate(filePath, dates); + } +} + +async function discoverJsonlFileDates(filePath: string, dates: Set): Promise { + const content = await safeReadUtf8(filePath); + if (!content) return; + let foundDate = false; + for (const line of content.split('\n')) { + if (!line.trim()) continue; + try { + foundDate = collectRecordDates(JSON.parse(line) as Record, dates) || foundDate; + } catch { /* skip */ } } + if (!foundDate) await addFileModifiedDate(filePath, dates); } /** 通用：递归扫描 .json 文件，从顶层或 messages 提取 timestamp */ @@ -213,18 +233,62 @@ async function discoverGenericJsonDates(baseDir: string, dates: Set): Pr if (!content) continue; let data: any; try { data = JSON.parse(content); } catch { continue; } - // 顶层 timestamp - const topTs = parseTs(data.timestamp ?? data.createTime); - if (topTs) dates.add(dateKey(topTs)); - // messages 数组 - const msgs = data.messages ?? data.history ?? []; - if (Array.isArray(msgs)) { - for (const msg of msgs) { - const ts = parseTs(msg.timestamp ?? msg.createTime); - if (ts) dates.add(dateKey(ts)); - } + const foundDate = Array.isArray(data) + ? data.reduce((found, row) => collectRecordDates(row, dates) || found, false) + : collectRecordDates(data, dates); + if (!foundDate) await addFileModifiedDate(filePath, dates); + } +} + +function collectRecordDates(record: Record | undefined, dates: Set): boolean { + if (!record || typeof record !== 'object') return false; + let found = false; + const candidates = [ + record.timestamp, record.time, record.created_at, record.createTime, record.startTime, + record.lastUpdated, record.created, record.providerLockTimestamp, record.endTime, + record.hrTime, record._hrTime, record.observedTimestamp, record.timeUnixNano, + record.time?.created, + ]; + for (const value of candidates) { + const ts = parseStructuredTs(value); + if (ts) { + dates.add(dateKey(ts)); + found = true; } } + const nestedRows = [ + ...(Array.isArray(record.messages) ? record.messages : []), + ...(Array.isArray(record.history) ? record.history : []), + ...(Array.isArray(record.data?.messages) ? record.data.messages : []), + ...(Array.isArray(record.data?.history) ? record.data.history : []), + ...(Array.isArray(record.$set?.messages) ? record.$set.messages : []), + ...(Array.isArray(record.usageLedger?.events) ? record.usageLedger.events : []), + ]; + for (const row of nestedRows) found = collectRecordDates(row, dates) || found; + return found; +} + +function parseStructuredTs(value: unknown): Date | null { + if (Array.isArray(value) && value.length > 0) { + const seconds = Number(value[0]); + const nanos = Number(value[1] ?? 0); + if (Number.isFinite(seconds) && Number.isFinite(nanos)) { + return parseTs(seconds * 1_000 + nanos / 1_000_000); + } + } + if (typeof value === 'number' || (typeof value === 'string' && /^\d+(?:\.\d+)?$/.test(value))) { + const raw = Number(value); + if (!Number.isFinite(raw) || raw <= 0) return null; + const abs = Math.abs(raw); + const millis = abs >= 1e17 ? raw / 1e6 : abs >= 1e14 ? raw / 1e3 : raw; + return parseTs(millis); + } + return parseTs(value as string | number | undefined); +} + +async function addFileModifiedDate(filePath: string, dates: Set): Promise { + const timestamp = await fileModifiedTs(filePath); + if (timestamp) dates.add(dateKey(timestamp)); } async function walkForFiles(dir: string, ext: string, result: string[]): Promise { @@ -242,64 +306,10 @@ async function walkForFiles(dir: string, ext: string, result: string[]): Promise async function discoverGeminiDates(dates: Set): Promise { const baseDir = join(homedir(), '.gemini', 'tmp'); - const files: string[] = []; - await walkForGeminiJsonl(baseDir, files); - - for (const filePath of files) { - const content = await safeReadUtf8(filePath); - if (!content) continue; - - let session: - | { timestamp?: string | number; createTime?: string | number; startTime?: string | number; messages?: { timestamp?: string | number; createTime?: string | number }[]; history?: { timestamp?: string | number; createTime?: string | number }[]; data?: { createTime?: string | number; messages?: { timestamp?: string | number; createTime?: string | number }[]; history?: { timestamp?: string | number; createTime?: string | number }[] } } - | Array<{ timestamp?: string | number }>; - try { - session = JSON.parse(content); - } catch { - continue; - } - - if (Array.isArray(session)) { - for (const row of session) { - const ts = parseTs(row.timestamp); - if (ts) dates.add(dateKey(ts)); - } - continue; - } - - const topLevelTs = parseTs(session.timestamp ?? session.createTime ?? session.startTime ?? session.data?.createTime); - if (topLevelTs) dates.add(dateKey(topLevelTs)); - - const messages = [ - ...(session.messages ?? []), - ...(session.history ?? []), - ...(session.data?.messages ?? []), - ...(session.data?.history ?? []), - ]; - for (const msg of messages) { - const ts = parseTs(msg.timestamp ?? msg.createTime); - if (ts) { - dates.add(dateKey(ts)); - } - } - } -} - -async function walkForGeminiJsonl(dir: string, result: string[]): Promise { - let entries; - try { - entries = await readdir(dir, { withFileTypes: true }); - } catch { - return; - } - - for (const entry of entries) { - const fullPath = join(dir, entry.name); - if (entry.isDirectory()) { - await walkForGeminiJsonl(fullPath, result); - } else if (entry.name.endsWith('.json')) { - result.push(fullPath); - } - } + await Promise.all([ + discoverGenericJsonDates(baseDir, dates), + discoverGenericJsonlDates(baseDir, dates), + ]); } async function discoverCopilotVscodeDates(dates: Set): Promise { @@ -320,7 +330,7 @@ async function discoverCopilotVscodeDates(dates: Set): Promise { const sessionFiles: string[] = []; await walkForFiles(join(home, 'Library', 'Application Support', 'Code', 'User', 'workspaceStorage'), '.json', sessionFiles); for (const filePath of sessionFiles) { - if (!filePath.includes('/chatSessions/')) continue; + if (!isChatSessionFile(filePath)) continue; const content = await safeReadUtf8(filePath); if (!content) continue; let session: { requests?: Array<{ timestamp?: string | number; response?: unknown[]; result?: { errorDetails?: { responseIsIncomplete?: boolean } } }> }; @@ -336,6 +346,56 @@ async function discoverCopilotVscodeDates(dates: Set): Promise { if (ts) dates.add(dateKey(ts)); } } + + const crdtFiles: string[] = []; + await walkForFiles( + join(home, 'Library', 'Application Support', 'Code', 'User', 'workspaceStorage'), + '.jsonl', + crdtFiles, + ); + for (const filePath of crdtFiles) { + if (!isChatSessionFile(filePath)) continue; + const content = await safeReadUtf8(filePath); + if (!content) continue; + for (const line of content.split('\n')) { + if (!line.trim()) continue; + let record: { kind?: number; k?: unknown[]; v?: unknown }; + try { + record = JSON.parse(line) as { kind?: number; k?: unknown[]; v?: unknown }; + } catch { + continue; + } + const root = record.v && typeof record.v === 'object' && !Array.isArray(record.v) + ? record.v as { requests?: unknown[] } + : undefined; + const requests = record.kind === 0 && Array.isArray(root?.requests) + ? root.requests + : record.kind === 2 && record.k?.[0] === 'requests' && Array.isArray(record.v) + ? record.v + : []; + for (const value of requests) { + if (!value || typeof value !== 'object' || Array.isArray(value)) continue; + const request = value as { + timestamp?: string | number; + modelId?: string; + promptTokens?: number; + completionTokens?: number; + result?: { metadata?: { promptTokens?: number; outputTokens?: number; resolvedModel?: string } }; + }; + const metadata = request.result?.metadata; + const isCopilot = Boolean(metadata?.resolvedModel) || request.modelId?.startsWith('copilot/'); + const hasTokens = (request.promptTokens ?? metadata?.promptTokens ?? 0) > 0 + || (request.completionTokens ?? metadata?.outputTokens ?? 0) > 0; + if (!isCopilot || !hasTokens) continue; + const ts = parseTs(request.timestamp); + if (ts) dates.add(dateKey(ts)); + } + } + } +} + +function isChatSessionFile(filePath: string): boolean { + return basename(dirname(filePath)) === 'chatSessions'; } async function discoverAntigravityDates(dates: Set): Promise { diff --git a/packages/cli/src/scanners/__tests__/amp.test.ts b/packages/cli/src/scanners/__tests__/amp.test.ts new file mode 100644 index 0000000..fe65ad6 --- /dev/null +++ b/packages/cli/src/scanners/__tests__/amp.test.ts @@ -0,0 +1,65 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { scanAmpDates } from '../amp.js'; + +let tmpDir: string; + +beforeEach(async () => { + tmpDir = join(tmpdir(), `aiusage-amp-${Date.now()}`); + await mkdir(tmpDir, { recursive: true }); +}); + +afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }); +}); + +async function writeThread(name: string, value: unknown): Promise { + await writeFile(join(tmpDir, name), JSON.stringify(value)); +} + +describe('scanAmpDates', () => { + it('部分 ledger 与 message usage 对账：匹配项不双计，缺失项仍保留', async () => { + await writeThread('T-partial.json', { + id: 'thread-1', + created: Date.parse('2026-07-18T08:00:00Z'), + usageLedger: { events: [{ + timestamp: '2026-07-18T09:00:00Z', + model: 'claude-sonnet-4', + toMessageId: 1, + tokens: { input: 100, output: 20 }, + }] }, + messages: [ + { role: 'assistant', messageId: 1, usage: { model: 'claude-sonnet-4', inputTokens: 100, outputTokens: 20, cacheReadInputTokens: 30 } }, + { role: 'assistant', messageId: 2, usage: { model: 'claude-sonnet-4', inputTokens: 50, outputTokens: 10, cacheCreationInputTokens: 5 } }, + ], + }); + + const rows = (await scanAmpDates(['2026-07-18'], tmpDir)).get('2026-07-18')!; + expect(rows).toEqual([ + expect.objectContaining({ + provider: 'anthropic', + product: 'amp', + eventCount: 2, + inputTokens: 150, + cachedInputTokens: 30, + cacheWriteTokens: 5, + outputTokens: 30, + }), + ]); + }); + + it('完整 ledger 与无 ID 的同 token 消息按指纹去重', async () => { + await writeThread('T-full.json', { + created: '2026-07-18T08:00:00Z', + usageLedger: { events: [{ model: 'gemini-2.5-pro', tokens: { input: 10, output: 2 } }] }, + messages: [{ role: 'assistant', usage: { model: 'gemini-2.5-pro', inputTokens: 10, outputTokens: 2 } }], + }); + + const rows = (await scanAmpDates(['2026-07-18'], tmpDir)).get('2026-07-18')!; + expect(rows[0]).toEqual(expect.objectContaining({ + provider: 'google', eventCount: 1, inputTokens: 10, outputTokens: 2, + })); + }); +}); diff --git a/packages/cli/src/scanners/__tests__/claude.test.ts b/packages/cli/src/scanners/__tests__/claude.test.ts index a21d162..b970024 100644 --- a/packages/cli/src/scanners/__tests__/claude.test.ts +++ b/packages/cli/src/scanners/__tests__/claude.test.ts @@ -43,18 +43,6 @@ function claudeRecord(opts: { }; } -function makeStatsCache(opts: { - dailyModelTokens: Array<{ date: string; tokensByModel: Record }>; - modelUsage?: Record; -}): string { - return JSON.stringify({ - version: '1', - lastComputedDate: '2026-01-31', - dailyModelTokens: opts.dailyModelTokens, - modelUsage: opts.modelUsage ?? {}, - }); -} - let tmpDir: string; // baseDirs that getClaudeProjectDirs() will use: tmpDir/projects // stats-cache lives at: tmpDir/stats-cache.json @@ -96,7 +84,7 @@ describe('JSONL scanning', () => { expect(b.cacheWriteTokens).toBe(3000); }); - it('deduplicates repeated records with the same messageId+requestId (first-seen wins)', async () => { + it('deduplicates repeated records with the same messageId+requestId', async () => { const projectDir = join(tmpDir, 'projects', '-Users-test-project'); // Same messageId+requestId in 3 files (session replay pattern): identical token counts await writeJsonl(projectDir, 'session1.jsonl', [ @@ -116,184 +104,90 @@ describe('JSONL scanning', () => { expect(b.eventCount).toBe(1); // single event despite 3 files }); - it('returns empty array for a date with no JSONL data and no stats-cache', async () => { - const result = await scanClaudeDates(['2025-11-01'], join(tmpDir, 'projects')); - expect(result.get('2025-11-01')).toEqual([]); - }); -}); - -// ─── Stats-cache fallback ──────────────────────────────────────────────────── - -describe('stats-cache fallback', () => { - it('fills a date with no JSONL data from stats-cache dailyModelTokens', async () => { - await writeFile( - join(tmpDir, 'stats-cache.json'), - makeStatsCache({ - dailyModelTokens: [ - { date: '2025-12-25', tokensByModel: { 'claude-opus-4-5-20251101': 100_000 } }, - ], - modelUsage: { - 'claude-opus-4-5-20251101': { inputTokens: 800_000, outputTokens: 200_000 }, // 80/20 ratio - }, - }), - ); - - const result = await scanClaudeDates(['2025-12-25'], join(tmpDir, 'projects')); - const breakdowns = result.get('2025-12-25')!; - expect(breakdowns).toHaveLength(1); - const b = breakdowns[0]; - expect(b.model).toBe('claude-opus-4-5'); - expect(b.provider).toBe('anthropic'); - expect(b.product).toBe('claude-code'); - // 80% input, 20% output - expect(b.inputTokens).toBe(80_000); - expect(b.outputTokens).toBe(20_000); - // cache fields are unknown from stats-cache - expect(b.cachedInputTokens).toBe(0); - expect(b.cacheWriteTokens).toBe(0); - }); - - it('uses 70/30 default ratio when modelUsage has no entry for the model', async () => { - await writeFile( - join(tmpDir, 'stats-cache.json'), - makeStatsCache({ - dailyModelTokens: [ - { date: '2025-12-26', tokensByModel: { 'claude-unknown-model-20251201': 50_000 } }, - ], - modelUsage: {}, // no entry for this model - }), - ); - - const result = await scanClaudeDates(['2025-12-26'], join(tmpDir, 'projects')); - const [b] = result.get('2025-12-26')!; - expect(b.inputTokens).toBe(35_000); // 70% of 50000 - expect(b.outputTokens).toBe(15_000); // 30% of 50000 - }); - - it('does NOT use stats-cache for a date that already has JSONL data', async () => { - // Write JSONL data for Dec 27 + it('缺少 requestId 时按 message.id 去重，并合并流式快照的字段最大值', async () => { const projectDir = join(tmpDir, 'projects', '-Users-test-project'); await writeJsonl(projectDir, 'session.jsonl', [ - claudeRecord({ - timestamp: '2025-12-27T10:00:00.000Z', - requestId: 'req_999', - model: 'claude-opus-4-5-20251101', - inputTokens: 1_000, - outputTokens: 100, - }), + { + type: 'assistant', timestamp: '2026-01-15T10:00:00.000Z', cwd: '/Users/test/project', + message: { id: 'msg_stream', model: 'claude-opus-4-8', usage: { + input_tokens: 100, cache_read_input_tokens: 300, output_tokens: 20, + } }, + }, + { + type: 'assistant', timestamp: '2026-01-15T10:00:01.000Z', cwd: '/Users/test/project', + message: { id: 'msg_stream', model: 'claude-opus-4-8', usage: { + input_tokens: 150, cache_read_input_tokens: 250, output_tokens: 40, + } }, + }, ]); - // Stats-cache also has Dec 27 with different (much larger) values - await writeFile( - join(tmpDir, 'stats-cache.json'), - makeStatsCache({ - dailyModelTokens: [ - { date: '2025-12-27', tokensByModel: { 'claude-opus-4-5-20251101': 999_999 } }, - ], - modelUsage: { 'claude-opus-4-5-20251101': { inputTokens: 7, outputTokens: 3 } }, - }), - ); - const result = await scanClaudeDates(['2025-12-27'], join(tmpDir, 'projects')); - const [b] = result.get('2025-12-27')!; - // Should come from JSONL, not stats-cache - expect(b.inputTokens).toBe(1_000); - expect(b.outputTokens).toBe(100); + const [b] = (await scanClaudeDates(['2026-01-15'], join(tmpDir, 'projects'))).get('2026-01-15')!; + expect(b).toEqual(expect.objectContaining({ + eventCount: 1, + inputTokens: 150, + cachedInputTokens: 300, + outputTokens: 40, + })); }); - it('handles missing stats-cache.json gracefully', async () => { - // No stats-cache.json written - const result = await scanClaudeDates(['2025-12-28'], join(tmpDir, 'projects')); - expect(result.get('2025-12-28')).toEqual([]); - }); - - it('handles malformed stats-cache.json gracefully', async () => { - await writeFile(join(tmpDir, 'stats-cache.json'), 'not valid json {{{{'); - - const result = await scanClaudeDates(['2025-12-29'], join(tmpDir, 'projects')); - expect(result.get('2025-12-29')).toEqual([]); - }); - - it('covers multiple missing dates from a single stats-cache', async () => { - await writeFile( - join(tmpDir, 'stats-cache.json'), - makeStatsCache({ - dailyModelTokens: [ - { date: '2025-12-25', tokensByModel: { 'claude-opus-4-5-20251101': 100_000 } }, - { date: '2025-12-26', tokensByModel: { 'claude-opus-4-5-20251101': 200_000 } }, - { date: '2025-12-28', tokensByModel: { 'claude-opus-4-5-20251101': 50_000 } }, // not requested - ], - modelUsage: { 'claude-opus-4-5-20251101': { inputTokens: 6, outputTokens: 4 } }, // 60/40 - }), - ); + it('缓存写入总量使用权威字段，并随流式快照更新明细', async () => { + const projectDir = join(tmpDir, 'projects', '-Users-test-project'); + await writeJsonl(projectDir, 'session.jsonl', [ + { + type: 'assistant', timestamp: '2026-01-15T10:00:00.000Z', cwd: '/Users/test/project', + message: { id: 'msg_cache', model: 'claude-opus-4-8', usage: { + input_tokens: 10, output_tokens: 20, cache_creation_input_tokens: 100, + cache_creation: { ephemeral_5m_input_tokens: 60, ephemeral_1h_input_tokens: 30 }, + } }, + }, + { + type: 'assistant', timestamp: '2026-01-15T10:00:01.000Z', cwd: '/Users/test/project', + message: { id: 'msg_cache', model: 'claude-opus-4-8', usage: { + input_tokens: 10, output_tokens: 20, cache_creation_input_tokens: 120, + cache_creation: { ephemeral_5m_input_tokens: 70, ephemeral_1h_input_tokens: 50 }, + } }, + }, + ]); - const result = await scanClaudeDates( - ['2025-12-25', '2025-12-26'], - join(tmpDir, 'projects'), - ); - expect(result.get('2025-12-25')![0].inputTokens).toBe(60_000); - expect(result.get('2025-12-26')![0].inputTokens).toBe(120_000); - expect(result.has('2025-12-28')).toBe(false); // not requested + const [b] = (await scanClaudeDates(['2026-01-15'], join(tmpDir, 'projects'))).get('2026-01-15')!; + expect(b).toEqual(expect.objectContaining({ + eventCount: 1, + cacheWriteTokens: 120, + cacheWrite5mTokens: 70, + cacheWrite1hTokens: 50, + })); }); - it('normalises model names from stats-cache (strips date suffix)', async () => { - await writeFile( - join(tmpDir, 'stats-cache.json'), - makeStatsCache({ - dailyModelTokens: [ - { date: '2025-12-30', tokensByModel: { 'claude-sonnet-4-5-20250929': 40_000 } }, - ], - }), - ); - - const result = await scanClaudeDates(['2025-12-30'], join(tmpDir, 'projects')); - const [b] = result.get('2025-12-30')!; - expect(b.model).toBe('claude-sonnet-4-5'); // date suffix stripped - }); -}); - -// ─── Integration: JSONL + stats-cache mixed ────────────────────────────────── - -describe('Integration: JSONL and stats-cache for different dates', () => { - it('returns JSONL data for covered dates and stats-cache data for uncovered dates', async () => { - // JSONL covers Jan 15 + it('从显式字段或模型前缀识别兼容模型供应商', async () => { const projectDir = join(tmpDir, 'projects', '-Users-test-project'); await writeJsonl(projectDir, 'session.jsonl', [ - claudeRecord({ - timestamp: '2026-01-15T10:00:00.000Z', - requestId: 'req_100', - model: 'claude-opus-4-6', - inputTokens: 2_000, - outputTokens: 300, - }), + { + type: 'assistant', timestamp: '2026-01-15T10:00:00.000Z', cwd: '/Users/test/project', + message: { id: 'msg_custom', model: 'model_api/experimental_0630', usage: { + input_tokens: 100, output_tokens: 20, + } }, + }, ]); - // Stats-cache covers Dec 25 (before JSONL rotation) - await writeFile( - join(tmpDir, 'stats-cache.json'), - makeStatsCache({ - dailyModelTokens: [ - { date: '2025-12-25', tokensByModel: { 'claude-opus-4-5-20251101': 500_000 } }, - ], - modelUsage: { 'claude-opus-4-5-20251101': { inputTokens: 623, outputTokens: 377 } }, - }), - ); + const [b] = (await scanClaudeDates(['2026-01-15'], join(tmpDir, 'projects'))).get('2026-01-15')!; + expect(b.provider).toBe('model_api'); + }); - const result = await scanClaudeDates( - ['2026-01-15', '2025-12-25'], - join(tmpDir, 'projects'), - ); + it('returns empty array for a date with no JSONL data and no stats-cache', async () => { + const result = await scanClaudeDates(['2025-11-01'], join(tmpDir, 'projects')); + expect(result.get('2025-11-01')).toEqual([]); + }); +}); - // Jan 15: from JSONL - const jan15 = result.get('2026-01-15')!; - expect(jan15).toHaveLength(1); - expect(jan15[0].model).toBe('claude-opus-4-6'); - expect(jan15[0].inputTokens).toBe(2_000); +describe('stats-cache safety', () => { + it('does not fabricate per-model input/output from aggregate stats-cache data', async () => { + await writeFile(join(tmpDir, 'stats-cache.json'), JSON.stringify({ + dailyModelTokens: [ + { date: '2025-12-25', tokensByModel: { 'claude-opus-4-5': 500_000 } }, + ], + })); - // Dec 25: from stats-cache, with 62.3/37.7 ratio - const dec25 = result.get('2025-12-25')!; - expect(dec25).toHaveLength(1); - expect(dec25[0].model).toBe('claude-opus-4-5'); - expect(dec25[0].inputTokens).toBe(311_500); // round(500000 * 623/1000) - expect(dec25[0].cachedInputTokens).toBe(0); // cache unknown from stats-cache + const result = await scanClaudeDates(['2025-12-25'], join(tmpDir, 'projects')); + expect(result.get('2025-12-25')).toEqual([]); }); }); diff --git a/packages/cli/src/scanners/__tests__/codex.test.ts b/packages/cli/src/scanners/__tests__/codex.test.ts index 78fa368..ea69faa 100644 --- a/packages/cli/src/scanners/__tests__/codex.test.ts +++ b/packages/cli/src/scanners/__tests__/codex.test.ts @@ -361,6 +361,126 @@ describe('Fix 2: deduplication of duplicate events', () => { expect(results[0].eventCount).toBe(2); // 两个会话各 1 笔真实 turn，全零不计 expect(results[0].outputTokens).toBe(200); // 80 + 120 }); + + it('不会因两个独立 session 的累计 token 完全相同而误杀其中一个', async () => { + const day = '2025-10-16'; + const dir = join(tmpDir, 'sessions', '2025', '10', '16'); + for (const id of ['session-a', 'session-b']) { + await writeSession(dir, `${id}.jsonl`, [ + { type: 'session_meta', payload: { id, cwd: '/same-project' } }, + { type: 'turn_context', payload: { model: 'gpt-5-codex', cwd: '/same-project' } }, + { + type: 'event_msg', timestamp: `${day}T10:00:00.000Z`, + payload: { type: 'token_count', info: { + total_token_usage: { input_tokens: 100, cached_input_tokens: 50, output_tokens: 10 }, + last_token_usage: { input_tokens: 100, cached_input_tokens: 50, output_tokens: 10 }, + } }, + }, + ]); + } + + const results = await scanCodex(day, tmpDir); + expect(results[0]).toEqual(expect.objectContaining({ + eventCount: 2, inputTokens: 100, cachedInputTokens: 100, outputTokens: 20, + })); + }); + + it('同一 session 在活动目录和归档目录各一份时仍只统计一次', async () => { + const day = '2025-10-16'; + const lines = [ + { type: 'session_meta', payload: { id: 'same-session', cwd: '/same-project' } }, + { type: 'turn_context', payload: { model: 'gpt-5-codex', cwd: '/same-project' } }, + { + type: 'event_msg', timestamp: `${day}T10:00:00.000Z`, + payload: { type: 'token_count', info: { + total_token_usage: { input_tokens: 100, cached_input_tokens: 50, output_tokens: 10 }, + last_token_usage: { input_tokens: 100, cached_input_tokens: 50, output_tokens: 10 }, + } }, + }, + ]; + await writeSession(join(tmpDir, 'sessions', '2025', '10', '16'), 'active.jsonl', lines); + const archivedLines = lines.map((line) => JSON.parse( + JSON.stringify(line).replaceAll('/same-project', '/stale-archived-project'), + )); + await writeSession(join(tmpDir, 'archived_sessions', 'nested'), 'archived.jsonl', archivedLines); + + const results = await scanCodex(day, tmpDir); + expect(results[0]).toEqual(expect.objectContaining({ + project: '/same-project', eventCount: 1, outputTokens: 10, + })); + }); + + it('session_meta 的初始 workspace 优先于后续 turn_context cwd', async () => { + const day = '2025-10-16'; + await writeSession(join(tmpDir, 'sessions', '2025', '10', '16'), 'workspace.jsonl', [ + { type: 'session_meta', payload: { id: 'workspace-session', cwd: '/origin-project' } }, + { type: 'turn_context', payload: { model: 'gpt-5-codex', cwd: '/later-cwd' } }, + { + type: 'event_msg', timestamp: `${day}T10:00:00.000Z`, + payload: { type: 'token_count', info: { + total_token_usage: { input_tokens: 100, cached_input_tokens: 50, output_tokens: 10 }, + last_token_usage: { input_tokens: 100, cached_input_tokens: 50, output_tokens: 10 }, + } }, + }, + ]); + + const results = await scanCodex(day, tmpDir); + expect(results[0]).toEqual(expect.objectContaining({ project: '/origin-project' })); + }); + + it('子会话跳过复制的父历史，只统计超过继承基线的自身 turn', async () => { + const day = '2025-10-16'; + const dir = join(tmpDir, 'sessions', '2025', '10', '16'); + const parentId = '019e5b00-0000-7000-8000-000000000001'; + const childId = '019e5c03-0000-7000-8000-000000000001'; + await writeSession(dir, 'child.jsonl', [ + { + type: 'session_meta', + payload: { + id: childId, + forked_from_id: parentId, + source: { subagent: { thread_spawn: { parent_thread_id: parentId } } }, + cwd: '/child-project', + }, + }, + { type: 'session_meta', payload: { id: parentId, cwd: '/parent-project' } }, + { type: 'turn_context', payload: { turn_id: parentId, model: 'gpt-5-codex', cwd: '/parent-project' } }, + { + type: 'event_msg', timestamp: `${day}T10:00:00.000Z`, + payload: { type: 'token_count', info: { + total_token_usage: { input_tokens: 300, cached_input_tokens: 200, output_tokens: 30, total_tokens: 330 }, + last_token_usage: { input_tokens: 300, cached_input_tokens: 200, output_tokens: 30, total_tokens: 330 }, + } }, + }, + { + type: 'event_msg', timestamp: `${day}T10:00:00.500Z`, + payload: { type: 'task_started', turn_id: childId }, + }, + { type: 'turn_context', payload: { turn_id: childId, model: 'gpt-5-codex' } }, + { + type: 'event_msg', timestamp: `${day}T10:00:01.000Z`, + payload: { type: 'token_count', info: { + total_token_usage: { input_tokens: 300, cached_input_tokens: 200, output_tokens: 30, total_tokens: 330 }, + last_token_usage: { input_tokens: 250, cached_input_tokens: 180, output_tokens: 25, total_tokens: 275 }, + } }, + }, + { + type: 'event_msg', timestamp: `${day}T10:00:02.000Z`, + payload: { type: 'token_count', info: { + total_token_usage: { input_tokens: 320, cached_input_tokens: 210, output_tokens: 32, total_tokens: 352 }, + last_token_usage: { input_tokens: 20, cached_input_tokens: 10, output_tokens: 2, total_tokens: 22 }, + } }, + }, + ]); + + const results = await scanCodex(day, tmpDir); + expect(results).toEqual([ + expect.objectContaining({ + project: '/child-project', eventCount: 1, inputTokens: 10, + cachedInputTokens: 10, outputTokens: 2, + }), + ]); + }); }); // ─── Fix 3: fallback to delta when last_token_usage is absent ─── diff --git a/packages/cli/src/scanners/__tests__/copilot-vscode.test.ts b/packages/cli/src/scanners/__tests__/copilot-vscode.test.ts index 59f7be6..5d9f10a 100644 --- a/packages/cli/src/scanners/__tests__/copilot-vscode.test.ts +++ b/packages/cli/src/scanners/__tests__/copilot-vscode.test.ts @@ -54,6 +54,40 @@ async function writeWorkspaceSession(opts: { ); } +async function writeWorkspaceJsonl(opts: { + rootDir: string; + workspaceId: string; + folderUri: string; + sessionId: string; + timestamp: number; + resolvedModel?: string; +}): Promise { + const workspaceDir = join(opts.rootDir, opts.workspaceId); + await mkdir(join(workspaceDir, 'chatSessions'), { recursive: true }); + await writeFile(join(workspaceDir, 'workspace.json'), JSON.stringify({ folder: opts.folderUri })); + await writeFile(join(workspaceDir, 'chatSessions', `${opts.sessionId}.jsonl`), [ + JSON.stringify({ kind: 0, v: { requests: [] } }), + JSON.stringify({ + kind: 2, + k: ['requests'], + v: [{ + requestId: 'request-jsonl', + timestamp: opts.timestamp, + modelId: 'copilot/auto', + promptTokens: 5_000, + completionTokens: 200, + result: { metadata: { + resolvedModel: opts.resolvedModel ?? 'claude-sonnet-4-6', + toolCallRounds: [ + { thinking: { tokens: 30 } }, + { thinking: { tokens: 70 } }, + ], + } }, + }], + }), + ].join('\n')); +} + describe('scanCopilotVscodeDates', () => { it('counts successful chat completions as IDE usage events', async () => { await writeCopilotLog([ @@ -172,4 +206,54 @@ describe('scanCopilotVscodeDates', () => { }), ]); }); + + it('读取新版 chatSessions JSONL 中的真实 token 与 resolvedModel', async () => { + const workspaceStorageDir = join(tmpDir, 'workspaceStorage'); + await writeWorkspaceJsonl({ + rootDir: workspaceStorageDir, + workspaceId: 'ws-jsonl', + folderUri: 'file:///Users/test/JSONL%20Project', + sessionId: 'session-jsonl', + timestamp: Date.parse('2026-02-12T12:00:00.000Z'), + resolvedModel: 'claude-sonnet-4.6-20260201', + }); + + const [row] = (await scanCopilotVscodeDates( + ['2026-02-12'], + tmpDir, + { '/Users/test/JSONL Project': 'JSONL Project' }, + workspaceStorageDir, + )).get('2026-02-12')!; + + expect(row).toEqual(expect.objectContaining({ + provider: 'anthropic', + model: 'claude-sonnet-4-6', + project: 'JSONL Project', + eventCount: 1, + inputTokens: 5_000, + outputTokens: 200, + reasoningOutputTokens: 100, + })); + }); + + it('保留非 Claude 模型名中的小数点', async () => { + const workspaceStorageDir = join(tmpDir, 'workspaceStorage'); + await writeWorkspaceJsonl({ + rootDir: workspaceStorageDir, + workspaceId: 'ws-gpt', + folderUri: 'file:///Users/test/GPT%20Project', + sessionId: 'session-gpt', + timestamp: Date.parse('2026-02-12T13:00:00.000Z'), + resolvedModel: 'gpt-4.1', + }); + + const [row] = (await scanCopilotVscodeDates( + ['2026-02-12'], + tmpDir, + undefined, + workspaceStorageDir, + )).get('2026-02-12')!; + + expect(row.model).toBe('gpt-4.1'); + }); }); diff --git a/packages/cli/src/scanners/__tests__/copilot.test.ts b/packages/cli/src/scanners/__tests__/copilot.test.ts index 100d1c6..6b4cacd 100644 --- a/packages/cli/src/scanners/__tests__/copilot.test.ts +++ b/packages/cli/src/scanners/__tests__/copilot.test.ts @@ -282,3 +282,68 @@ describe('multi-session aggregation', () => { expect(items.map((i) => i.model).sort()).toEqual(['claude-sonnet-4', 'gpt-4o']); }); }); + +describe('OpenTelemetry usage', () => { + it('解析 chat span 的缓存写入与推理 token，并从 input 中仅扣除 cache read', async () => { + const day = '2026-07-18'; + await writeSession(tmpDir, 'copilot-otel.jsonl', [{ + type: 'span', traceId: 'trace-1', spanId: 'chat-1', name: 'chat claude-sonnet-4-6', + startTime: [1784361600, 0], + attributes: { + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.model': 'claude-sonnet-4-6', + 'gen_ai.conversation.id': 'conv-1', + 'gen_ai.usage.input_tokens': 1000, + 'gen_ai.usage.output_tokens': 50, + 'gen_ai.usage.cache_read_input_tokens': 200, + 'gen_ai.usage.cache_creation_input_tokens': 75, + 'gen_ai.usage.reasoning_tokens': 12, + }, + }]); + + const rows = (await scanCopilotDates([day], tmpDir)).get(day)!; + expect(rows).toEqual([ + expect.objectContaining({ + provider: 'anthropic', model: 'claude-sonnet-4-6', eventCount: 1, inputTokens: 800, + cachedInputTokens: 200, cacheWriteTokens: 75, outputTokens: 50, + reasoningOutputTokens: 12, + }), + ]); + }); + + it('识别纳秒时间戳，并按 response ID 抑制跨 trace 汇总', async () => { + const day = '2026-07-18'; + const timeUnixNano = (BigInt(Date.parse('2026-07-18T10:00:00Z')) * 1_000_000n).toString(); + const attrs = { + 'gen_ai.response.id': 'response-1', + 'gen_ai.request.model': 'gpt-5.6', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 2, + }; + await writeSession(tmpDir, 'copilot-nanos.jsonl', [ + { type: 'log', traceId: 'trace-detail', timeUnixNano, attributes: { ...attrs, 'event.name': 'gen_ai.client.inference.operation.details' } }, + { type: 'span', traceId: 'trace-summary', spanId: 'summary', name: 'invoke_agent', timeUnixNano, attributes: { ...attrs, 'gen_ai.operation.name': 'invoke_agent' } }, + ]); + + const rows = (await scanCopilotDates([day], tmpDir)).get(day)!; + expect(rows[0]).toEqual(expect.objectContaining({ + provider: 'openai', eventCount: 1, inputTokens: 10, outputTokens: 2, + })); + }); + + it('同一 trace 有 chat span 时抑制 aggregate invoke_agent，避免双计', async () => { + const day = '2026-07-18'; + const attrs = { + 'gen_ai.request.model': 'gpt-5.6', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 2, + }; + await writeSession(tmpDir, 'copilot-dedup.jsonl', [ + { type: 'span', traceId: 'trace-2', spanId: 'summary', name: 'invoke_agent', startTime: [1784361600, 0], attributes: { ...attrs, 'gen_ai.operation.name': 'invoke_agent' } }, + { type: 'span', traceId: 'trace-2', spanId: 'chat', name: 'chat gpt-5.6', startTime: [1784361601, 0], attributes: { ...attrs, 'gen_ai.operation.name': 'chat' } }, + ]); + + const rows = (await scanCopilotDates([day], tmpDir)).get(day)!; + expect(rows[0]).toEqual(expect.objectContaining({ eventCount: 1, inputTokens: 10, outputTokens: 2 })); + }); +}); diff --git a/packages/cli/src/scanners/__tests__/droid.test.ts b/packages/cli/src/scanners/__tests__/droid.test.ts new file mode 100644 index 0000000..44d6911 --- /dev/null +++ b/packages/cli/src/scanners/__tests__/droid.test.ts @@ -0,0 +1,68 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { normalizeDroidModel, scanDroidDates } from '../droid.js'; + +let tmpDir: string; + +beforeEach(async () => { + tmpDir = join(tmpdir(), `aiusage-droid-${Date.now()}`); + await mkdir(tmpDir, { recursive: true }); +}); + +afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }); +}); + +describe('scanDroidDates', () => { + it('无需 transcript 即可读取 settings，保留五类 token 并归一化模型', async () => { + const path = join(tmpDir, 'Users-test-aiusage', 'session.settings.json'); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, JSON.stringify({ + model: 'custom:Claude-Opus-4.5-Thinking-[Anthropic]-0', + providerLock: 'anthropic', + providerLockTimestamp: '2026-07-18T10:00:00Z', + tokenUsage: { + inputTokens: 100, + outputTokens: 20, + cacheCreationTokens: 12, + cacheReadTokens: 30, + thinkingTokens: 5, + }, + })); + + const rows = (await scanDroidDates(['2026-07-18'], tmpDir)).get('2026-07-18')!; + expect(rows).toEqual([ + expect.objectContaining({ + provider: 'anthropic', + product: 'droid', + model: 'claude-opus-4-5-thinking-0', + inputTokens: 100, + cachedInputTokens: 30, + cacheWriteTokens: 12, + outputTokens: 20, + reasoningOutputTokens: 5, + }), + ]); + }); + + it('模型缺失时可从 transcript 的 Model 提示恢复', async () => { + const base = join(tmpDir, 'repo', 'session'); + await mkdir(dirname(base), { recursive: true }); + await writeFile(`${base}.jsonl`, '{"text":"Model: Gemini 2.5 Pro [Google]"}\n'); + await writeFile(`${base}.settings.json`, JSON.stringify({ + providerLockTimestamp: '2026-07-18T10:00:00Z', + tokenUsage: { inputTokens: 1 }, + })); + + const rows = (await scanDroidDates(['2026-07-18'], tmpDir)).get('2026-07-18')!; + expect(rows[0]).toEqual(expect.objectContaining({ model: 'gemini-2-5-pro', provider: 'google' })); + }); +}); + +describe('normalizeDroidModel', () => { + it('清理 custom 前缀、供应商括号、点号和重复连接符', () => { + expect(normalizeDroidModel('custom:Claude-Sonnet-4.6-[Anthropic]')).toBe('claude-sonnet-4-6'); + }); +}); diff --git a/packages/cli/src/scanners/__tests__/gemini.test.ts b/packages/cli/src/scanners/__tests__/gemini.test.ts index 9f4b012..1f024cd 100644 --- a/packages/cli/src/scanners/__tests__/gemini.test.ts +++ b/packages/cli/src/scanners/__tests__/gemini.test.ts @@ -75,9 +75,62 @@ describe('scanGeminiDates', () => { eventCount: 1, inputTokens: 80, cachedInputTokens: 20, - outputTokens: 45, + outputTokens: 50, reasoningOutputTokens: 5, }), ]); }); + + it('解析当前 headless JSONL，按消息 ID 采用最后一条并跳过损坏行', async () => { + const baseDir = join(tmpDir, '.gemini', 'tmp'); + const path = join(baseDir, 'project-a', 'chats', 'session-current.jsonl'); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, [ + JSON.stringify({ type: 'init', model: 'gemini-2.5-pro', session_id: 's1' }), + '{broken', + JSON.stringify({ type: 'gemini', id: 'm1', timestamp: '2026-07-18T10:00:00Z', tokens: { prompt: 100, cached: 30, candidates: 20, thoughts: 5 } }), + JSON.stringify({ type: 'gemini', id: 'm1', timestamp: '2026-07-18T10:00:01Z', tokens: { prompt: 120, cached: 40, candidates: 25, thoughts: 6 } }), + ].join('\n')); + + const rows = (await scanGeminiDates(['2026-07-18'], baseDir)).get('2026-07-18')!; + expect(rows).toEqual([ + expect.objectContaining({ + projectDisplay: 'project-a', + eventCount: 1, + inputTokens: 80, + cachedInputTokens: 40, + outputTokens: 25, + reasoningOutputTokens: 6, + }), + ]); + }); + + it('解析 a2a 的 $set.messages 追加记录', async () => { + const baseDir = join(tmpDir, '.gemini', 'tmp'); + const path = join(baseDir, 'project-b', 'chats', 'session-a2a.jsonl'); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, [ + JSON.stringify({ kind: 'session', sessionId: 's2', startTime: '2026-07-18T09:00:00Z' }), + JSON.stringify({ $set: { messages: [{ + id: 'm2', type: 'gemini', model: 'gemini-2.5-flash', timestamp: '2026-07-18T09:01:00Z', + usageMetadata: { promptTokenCount: 20, cachedContentTokenCount: 5, candidatesTokenCount: 4 }, + }] } }), + ].join('\n')); + + const rows = (await scanGeminiDates(['2026-07-18'], baseDir)).get('2026-07-18')!; + expect(rows[0]).toEqual(expect.objectContaining({ inputTokens: 15, cachedInputTokens: 5, outputTokens: 4 })); + }); + + it('解析按模型分组的 headless stats', async () => { + const baseDir = join(tmpDir, '.gemini', 'tmp'); + await writeJson(join(baseDir, 'project-c', 'session-headless.json'), { + timestamp: '2026-07-18T08:00:00Z', + stats: { models: { 'gemini-2.5-pro': { tokens: { prompt: 12, cached: 5, candidates: 3, thoughts: 2 } } } }, + }); + + const rows = (await scanGeminiDates(['2026-07-18'], baseDir)).get('2026-07-18')!; + expect(rows[0]).toEqual(expect.objectContaining({ + inputTokens: 7, cachedInputTokens: 5, outputTokens: 3, reasoningOutputTokens: 2, + })); + }); }); diff --git a/packages/cli/src/scanners/__tests__/opencode.test.ts b/packages/cli/src/scanners/__tests__/opencode.test.ts new file mode 100644 index 0000000..f28f5cb --- /dev/null +++ b/packages/cli/src/scanners/__tests__/opencode.test.ts @@ -0,0 +1,69 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { parseOpenCodeSqliteRows, scanOpencodeDates } from '../opencode.js'; + +let tmpDir: string; + +beforeEach(async () => { + tmpDir = join(tmpdir(), `aiusage-opencode-${Date.now()}`); + await mkdir(tmpDir, { recursive: true }); +}); + +afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }); +}); + +describe('scanOpencodeDates', () => { + it('旧 JSON 只统计 assistant，补齐 provider 与 cache write', async () => { + const sessionDir = join(tmpDir, 'ses_1'); + await mkdir(sessionDir, { recursive: true }); + const base = { + time: { created: Date.parse('2026-07-18T08:00:00Z') }, + modelID: 'claude-sonnet-4-6', + providerID: 'anthropic', + path: { root: '/Users/test/repo' }, + tokens: { input: 100, output: 20, reasoning: 5, cache: { read: 30, write: 10 } }, + }; + await writeFile(join(sessionDir, 'assistant.json'), JSON.stringify({ ...base, id: 'm1', role: 'assistant' })); + await writeFile(join(sessionDir, 'user.json'), JSON.stringify({ ...base, id: 'm2', role: 'user' })); + + const rows = (await scanOpencodeDates(['2026-07-18'], tmpDir)).get('2026-07-18')!; + expect(rows).toEqual([ + expect.objectContaining({ + provider: 'anthropic', + product: 'opencode', + eventCount: 1, + inputTokens: 100, + cachedInputTokens: 30, + cacheWriteTokens: 10, + outputTokens: 20, + reasoningOutputTokens: 5, + }), + ]); + }); +}); + +describe('parseOpenCodeSqliteRows', () => { + it('解析 OpenCode 1.2+ message.data，并以 session directory 补齐项目', () => { + const records = parseOpenCodeSqliteRows([{ + id: 'row-1', + session_id: 'ses-1', + workspace_root: '/Users/test/sqlite-project', + data: JSON.stringify({ + id: 'msg-1', role: 'assistant', modelID: 'gpt-5.6', providerID: 'openai', + time: { created: Date.parse('2026-07-18T08:00:00Z') }, + tokens: { input: 7, output: 3, reasoning: 2, cache: { read: 5, write: 1 } }, + }), + }]); + + expect(records).toEqual([ + expect.objectContaining({ + id: 'msg-1', provider: 'openai', model: 'gpt-5.6', + projectRoot: '/Users/test/sqlite-project', + tokens: { input: 7, cached: 5, cacheWrite: 1, output: 3, reasoning: 2 }, + }), + ]); + }); +}); diff --git a/packages/cli/src/scanners/__tests__/pi.test.ts b/packages/cli/src/scanners/__tests__/pi.test.ts new file mode 100644 index 0000000..d9f134c --- /dev/null +++ b/packages/cli/src/scanners/__tests__/pi.test.ts @@ -0,0 +1,72 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { scanPiDates } from '../pi.js'; + +let tmpDir: string; + +beforeEach(async () => { + tmpDir = join(tmpdir(), `aiusage-pi-${Date.now()}`); + await mkdir(tmpDir, { recursive: true }); +}); + +afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }); +}); + +async function writeJsonl(path: string, rows: unknown[]): Promise { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, rows.map(row => JSON.stringify(row)).join('\n')); +} + +describe('scanPiDates', () => { + it('兼容 title 前置记录，保留 cache write，并优先使用消息 provider', async () => { + await writeJsonl(join(tmpDir, '--Users-test-repo--', 'session.jsonl'), [ + { type: 'title', title: 'generated' }, + { type: 'session', id: 's1', cwd: '/Users/test/repo' }, + { + type: 'message', + id: 'm1', + timestamp: '2026-07-18T09:00:00Z', + message: { + role: 'assistant', + model: 'claude-sonnet-4-6', + provider: 'anthropic', + usage: { input: 100, output: 20, cacheRead: 30, cacheWrite: 10 }, + }, + }, + ]); + + const rows = (await scanPiDates(['2026-07-18'], tmpDir)).get('2026-07-18')!; + expect(rows).toEqual([ + expect.objectContaining({ + provider: 'anthropic', + product: 'pi', + project: '/Users/test/repo', + eventCount: 1, + inputTokens: 100, + cachedInputTokens: 30, + cacheWriteTokens: 10, + outputTokens: 20, + }), + ]); + }); + + it('provider 缺失时根据模型推断，并将去重范围限制在各会话内', async () => { + for (const session of ['a', 'b']) { + await writeJsonl(join(tmpDir, session, `${session}.jsonl`), [ + { type: 'session', id: session, cwd: `/tmp/${session}` }, + { + type: 'message', id: 'same-id', timestamp: '2026-07-18T10:00:00Z', + message: { role: 'assistant', model: 'gpt-5.6', usage: { input: 3, output: 2 } }, + }, + ]); + } + + const rows = (await scanPiDates(['2026-07-18'], tmpDir)).get('2026-07-18')!; + expect(rows).toHaveLength(2); + expect(rows.every(row => row.provider === 'openai')).toBe(true); + expect(rows.reduce((sum, row) => sum + row.eventCount, 0)).toBe(2); + }); +}); diff --git a/packages/cli/src/scanners/__tests__/qwen.test.ts b/packages/cli/src/scanners/__tests__/qwen.test.ts new file mode 100644 index 0000000..2a1b378 --- /dev/null +++ b/packages/cli/src/scanners/__tests__/qwen.test.ts @@ -0,0 +1,67 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { scanQwenDates } from '../qwen.js'; + +let tmpDir: string; + +beforeEach(async () => { + tmpDir = join(tmpdir(), `aiusage-qwen-${Date.now()}`); + await mkdir(tmpDir, { recursive: true }); +}); + +afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }); +}); + +async function writeJsonl(path: string, rows: unknown[]): Promise { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, rows.map(row => JSON.stringify(row)).join('\n')); +} + +describe('scanQwenDates', () => { + it('读取当前 projects 目录格式，拆分缓存与思考 token 并按消息去重', async () => { + const file = join(tmpDir, 'my-project', 'chats', 'session.jsonl'); + const usage = { + promptTokenCount: 100, + candidatesTokenCount: 20, + cachedContentTokenCount: 30, + thoughtsTokenCount: 5, + }; + await writeJsonl(file, [ + { type: 'assistant', uuid: 'm1', sessionId: 's1', timestamp: '2026-07-18T08:00:00Z', model: 'qwen3-coder-plus', usageMetadata: usage }, + { type: 'assistant', uuid: 'm1', sessionId: 's1', timestamp: '2026-07-18T08:00:01Z', model: 'qwen3-coder-plus', usageMetadata: usage }, + ]); + + const rows = (await scanQwenDates(['2026-07-18'], tmpDir)).get('2026-07-18')!; + expect(rows).toEqual([ + expect.objectContaining({ + provider: 'alibaba', + product: 'qwen-code', + model: 'qwen3-coder-plus', + projectDisplay: 'my-project', + eventCount: 1, + inputTokens: 70, + cachedInputTokens: 30, + outputTokens: 20, + reasoningOutputTokens: 5, + }), + ]); + }); + + it('缺少时间戳时使用文件 mtime，不丢弃真实 token', async () => { + const file = join(tmpDir, 'project', 'chats', 'session.jsonl'); + await writeJsonl(file, [{ + type: 'assistant', + model: 'qwen-plus', + usageMetadata: { promptTokenCount: 4, candidatesTokenCount: 2 }, + }]); + const today = new Date(); + const day = `${today.getFullYear()}-${String(today.getMonth() + 1).padStart(2, '0')}-${String(today.getDate()).padStart(2, '0')}`; + + const rows = (await scanQwenDates([day], tmpDir)).get(day)!; + expect(rows).toHaveLength(1); + expect(rows[0]).toEqual(expect.objectContaining({ inputTokens: 4, outputTokens: 2 })); + }); +}); diff --git a/packages/cli/src/scanners/__tests__/utils.test.ts b/packages/cli/src/scanners/__tests__/utils.test.ts index 8bf73e6..3917e52 100644 --- a/packages/cli/src/scanners/__tests__/utils.test.ts +++ b/packages/cli/src/scanners/__tests__/utils.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { parseTs, dateKey } from '../utils.js'; +import { parseTs, dateKey, inferProviderFromModel, projectFromPath, resolveProjectFields } from '../utils.js'; describe('parseTs', () => { it('treats 10-digit numeric values as Unix seconds (not milliseconds)', () => { @@ -48,3 +48,19 @@ describe('dateKey', () => { expect(dateKey(d)).toBe('2026-06-02'); }); }); + +describe('inferProviderFromModel', () => { + it('识别常见跨工具模型供应商，并保留未知模型的 fallback', () => { + expect(inferProviderFromModel('claude-sonnet-4-6', 'pi')).toBe('anthropic'); + expect(inferProviderFromModel('gpt-5.6', 'pi')).toBe('openai'); + expect(inferProviderFromModel('gemini-3.1-pro-preview', 'pi')).toBe('google'); + expect(inferProviderFromModel('custom-model', 'pi')).toBe('pi'); + }); +}); + +describe('project path parsing', () => { + it('兼容 Windows 与 POSIX 路径分隔符', () => { + expect(projectFromPath('C:\\Users\\test\\project')).toBe('project'); + expect(resolveProjectFields('/Users/test/project').projectDisplay).toBe('project'); + }); +}); diff --git a/packages/cli/src/scanners/amp.ts b/packages/cli/src/scanners/amp.ts index 7e8ff8e..a593a59 100644 --- a/packages/cli/src/scanners/amp.ts +++ b/packages/cli/src/scanners/amp.ts @@ -1,11 +1,13 @@ import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { basename, join } from 'node:path'; import { homedir } from 'node:os'; import type { IngestBreakdown } from '@aiusage/shared'; import { parseTs, dateKey, walkFiles, + fileModifiedTs, + inferProviderFromModel, initDateMap, accumulate, finalize, @@ -13,23 +15,28 @@ import { } from './utils.js'; /** - * Amp (by Sourcegraph) scanner. + * Amp (Sourcegraph) scanner. * - * 数据目录: ~/.local/share/amp/threads/ (亦检查 $AMP_DATA_DIR / $XDG_DATA_HOME) - * 文件格式: T-*.json (完整 JSON，非 JSONL) - * - * 优先从 thread.usageLedger.events 提取用量; - * 无 ledger 时回退到 thread.messages[].usage。 + * 同时解析 usageLedger 和 assistant message usage。两者会按 messageId、 + * 再按模型与 token 指纹对账，既保留部分 ledger 之外的消息，也避免完整 ledger 双计。 */ +type AmpId = string | number; + +interface AmpMessageUsage { + model?: string; + inputTokens?: number; + outputTokens?: number; + cacheReadInputTokens?: number; + cacheCreationInputTokens?: number; +} + interface AmpMessage { + role?: string; + messageId?: AmpId; timestamp?: string | number; model?: string; - usage?: { - inputTokens?: number; - outputTokens?: number; - cacheReadInputTokens?: number; - }; + usage?: AmpMessageUsage; } interface AmpLedgerEvent { @@ -38,16 +45,34 @@ interface AmpLedgerEvent { tokens?: { input?: number; output?: number; + cacheReadInputTokens?: number; + cacheCreationInputTokens?: number; }; - toMessageId?: string; + toMessageId?: AmpId; } interface AmpThread { + id?: string; created?: string | number; messages?: Record | AmpMessage[]; - usageLedger?: { - events?: AmpLedgerEvent[]; - }; + usageLedger?: { events?: AmpLedgerEvent[] }; +} + +interface AmpTokens { + input: number; + cached: number; + cacheWrite: number; + output: number; + reasoning: number; +} + +interface AmpRecord { + model: string; + timestamp: Date; + hasExplicitTimestamp: boolean; + messageId?: string; + ledgerToMessageId?: string; + tokens: AmpTokens; } function resolveAmpDir(baseDir?: string): string { @@ -65,143 +90,167 @@ export async function scanAmpDates( _projectAliases?: Record, ): Promise> { const dates = new Set(targetDates); - const dir = resolveAmpDir(baseDir); - - const files = await walkFiles(dir, '.json'); - const threadFiles = files.filter((f) => { - const name = f.split('/').pop() ?? ''; - return name.startsWith('T-') && name.endsWith('.json'); - }); - if (threadFiles.length === 0) return emptyResult(dates); + const files = (await walkFiles(resolveAmpDir(baseDir), '.json')).filter(file => + basename(file).startsWith('T-'), + ); + if (files.length === 0) return emptyResult(dates); const grouped = initDateMap(dates); - for (const filePath of threadFiles) { - let raw: string; - try { - raw = await readFile(filePath, 'utf-8'); - } catch { - continue; - } - + for (const filePath of files) { let thread: AmpThread; try { - thread = JSON.parse(raw); + thread = JSON.parse(await readFile(filePath, 'utf-8')) as AmpThread; } catch { continue; } - const ledgerEvents = thread.usageLedger?.events; - const useLedger = Array.isArray(ledgerEvents) && ledgerEvents.length > 0; + const fileTs = await fileModifiedTs(filePath); + const threadTs = parseTs(thread.created) ?? fileTs; + if (!threadTs) continue; + + const ledger = buildLedgerRecords(thread.usageLedger?.events ?? [], threadTs); + const messages = buildMessageRecords(thread.messages, threadTs); + const records = reconcileAmpRecords(ledger, messages); + + for (const record of records) { + const dayMap = grouped.get(dateKey(record.timestamp)); + if (!dayMap) continue; + const { input, cached, cacheWrite, output, reasoning } = record.tokens; + if (input + cached + cacheWrite + output + reasoning === 0) continue; + const provider = inferProviderFromModel(record.model, 'anthropic'); - if (useLedger) { - processLedger(thread, ledgerEvents!, dates, grouped); - } else { - processMessages(thread, dates, grouped); + accumulate( + dayMap, + `${provider}|${record.model}|unknown`, + { + provider, + product: 'amp', + channel: 'cli', + model: record.model, + project: 'unknown', + projectDisplay: 'unknown', + inputTokens: 0, + cachedInputTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningOutputTokens: 0, + }, + record.tokens, + ); } } return finalize(grouped); } -function getMessageByKey( +function buildLedgerRecords(events: AmpLedgerEvent[], fallbackTs: Date): AmpRecord[] { + return events.map(event => { + const explicitTs = parseTs(event.timestamp); + return { + model: event.model ?? 'unknown', + timestamp: explicitTs ?? fallbackTs, + hasExplicitTimestamp: Boolean(explicitTs), + ledgerToMessageId: normalizeId(event.toMessageId), + tokens: { + input: clamp(event.tokens?.input), + output: clamp(event.tokens?.output), + cached: clamp(event.tokens?.cacheReadInputTokens), + cacheWrite: clamp(event.tokens?.cacheCreationInputTokens), + reasoning: 0, + }, + }; + }); +} + +function buildMessageRecords( messages: Record | AmpMessage[] | undefined, - key: string, -): AmpMessage | undefined { - if (!messages) return undefined; - if (Array.isArray(messages)) { - const idx = parseInt(key, 10); - return isNaN(idx) ? undefined : messages[idx]; - } - return messages[key]; + fallbackTs: Date, +): AmpRecord[] { + if (!messages) return []; + const entries = Array.isArray(messages) + ? messages.map((message, index) => [String(index), message] as const) + : Object.entries(messages); + + return entries.flatMap(([key, message], index) => { + if (message.role && message.role !== 'assistant') return []; + const usage = message.usage; + if (!usage) return []; + const numericId = typeof message.messageId === 'number' ? message.messageId : undefined; + const timestamp = parseTs(message.timestamp) + ?? new Date(fallbackTs.getTime() + Math.max(numericId ?? index, 0) * 1_000); + return [{ + model: usage.model ?? message.model ?? 'unknown', + timestamp, + hasExplicitTimestamp: Boolean(parseTs(message.timestamp)), + messageId: normalizeId(message.messageId ?? key), + tokens: { + input: clamp(usage.inputTokens), + output: clamp(usage.outputTokens), + cached: clamp(usage.cacheReadInputTokens), + cacheWrite: clamp(usage.cacheCreationInputTokens), + reasoning: 0, + }, + }]; + }); } -function processLedger( - thread: AmpThread, - events: AmpLedgerEvent[], - dates: Set, - grouped: ReturnType, -): void { - for (const event of events) { - const ts = parseTs(event.timestamp); - if (!ts) continue; - const dk = dateKey(ts); - const dayMap = grouped.get(dk); - if (!dayMap) continue; - - const model = event.model ?? 'unknown'; - const input = event.tokens?.input ?? 0; - const output = event.tokens?.output ?? 0; - - let cached = 0; - if (event.toMessageId) { - const msg = getMessageByKey(thread.messages, event.toMessageId); - cached = msg?.usage?.cacheReadInputTokens ?? 0; - } +function reconcileAmpRecords(ledger: AmpRecord[], messages: AmpRecord[]): AmpRecord[] { + if (ledger.length === 0) return messages.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); - accumulate( - dayMap, - `${model}|unknown`, - { - provider: 'sourcegraph', - product: 'amp', - channel: 'cli', - model, - project: 'unknown', - projectDisplay: 'unknown', - inputTokens: 0, - cachedInputTokens: 0, - cacheWriteTokens: 0, - outputTokens: 0, - reasoningOutputTokens: 0, + const consumed = new Set(); + const merged = [...ledger]; + const unmatched: AmpRecord[] = []; + + for (const message of messages) { + const index = findLedgerMatch(merged, consumed, message); + if (index < 0) { + unmatched.push(message); + continue; + } + consumed.add(index); + const item = merged[index]!; + merged[index] = { + ...item, + model: item.model === 'unknown' ? message.model : item.model, + timestamp: item.hasExplicitTimestamp ? item.timestamp : message.timestamp, + messageId: message.messageId, + tokens: { + input: item.tokens.input || message.tokens.input, + output: item.tokens.output || message.tokens.output, + cached: item.tokens.cached || message.tokens.cached, + cacheWrite: item.tokens.cacheWrite || message.tokens.cacheWrite, + reasoning: 0, }, - { input, cached, cacheWrite: 0, output, reasoning: 0 }, - ); + }; } + + return [...merged, ...unmatched].sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); } -function processMessages( - thread: AmpThread, - dates: Set, - grouped: ReturnType, -): void { - const msgs = thread.messages; - if (!msgs) return; - - const entries: AmpMessage[] = Array.isArray(msgs) ? msgs : Object.values(msgs); - - for (const msg of entries) { - const usage = msg.usage; - if (!usage) continue; - - const ts = parseTs(msg.timestamp ?? thread.created); - if (!ts) continue; - const dk = dateKey(ts); - const dayMap = grouped.get(dk); - if (!dayMap) continue; - - const model = msg.model ?? 'unknown'; - const input = usage.inputTokens ?? 0; - const output = usage.outputTokens ?? 0; - const cached = usage.cacheReadInputTokens ?? 0; - - accumulate( - dayMap, - `${model}|unknown`, - { - provider: 'sourcegraph', - product: 'amp', - channel: 'cli', - model, - project: 'unknown', - projectDisplay: 'unknown', - inputTokens: 0, - cachedInputTokens: 0, - cacheWriteTokens: 0, - outputTokens: 0, - reasoningOutputTokens: 0, - }, - { input, cached, cacheWrite: 0, output, reasoning: 0 }, +function findLedgerMatch(ledger: AmpRecord[], consumed: Set, message: AmpRecord): number { + if (message.messageId) { + const byId = ledger.findIndex((record, index) => + !consumed.has(index) && record.ledgerToMessageId === message.messageId, ); + if (byId >= 0) return byId; } + return ledger.findIndex((record, index) => + !consumed.has(index) + && record.model === message.model + && tokenFingerprint(record.tokens) === tokenFingerprint(message.tokens), + ); +} + +function tokenFingerprint(tokens: AmpTokens): string { + return [tokens.input, tokens.cached, tokens.cacheWrite, tokens.output, tokens.reasoning].join(':'); +} + +function normalizeId(value: AmpId | undefined): string | undefined { + if (value == null || value === '') return undefined; + return String(value); +} + +function clamp(value: number | undefined): number { + return Math.max(Number.isFinite(value) ? value! : 0, 0); } diff --git a/packages/cli/src/scanners/claude.ts b/packages/cli/src/scanners/claude.ts index d492779..ec7692a 100644 --- a/packages/cli/src/scanners/claude.ts +++ b/packages/cli/src/scanners/claude.ts @@ -1,26 +1,14 @@ -import { readdir, open, readFile } from 'node:fs/promises'; +import { createReadStream } from 'node:fs'; +import { readdir } from 'node:fs/promises'; import { join } from 'node:path'; import { homedir } from 'node:os'; import { createInterface } from 'node:readline'; import type { IngestBreakdown } from '@aiusage/shared'; -import { normalizeModelName, runWithConcurrency, type ProjectFields } from './utils.js'; +import { inferProviderFromModel, normalizeModelName, runWithConcurrency, type ProjectFields } from './utils.js'; const FILE_CONCURRENCY = 16; const MAX_LINE_BYTES = 64 * 1024 * 1024; // 64 MB -// Stats-cache fallback: Claude Code stores rolling daily token totals in -// ~/.claude/stats-cache.json. When JSONL session files have been rotated away, -// this is the only remaining source for historical token counts. -// The dailyModelTokens field tracks (input + output) tokens per model per day -// (cache tokens are NOT included). We distribute using the model's all-time -// input/output ratio from modelUsage, and fall back to 70/30 if unavailable. -const STATS_CACHE_DEFAULT_INPUT_RATIO = 0.7; - -interface StatsCache { - dailyModelTokens?: Array<{ date: string; tokensByModel: Record }>; - modelUsage?: Record; -} - interface ClaudeRecord { timestamp?: string; requestId?: string; @@ -29,9 +17,15 @@ interface ClaudeRecord { type?: string; cwd?: string; costUSD?: number; + providerId?: string; + provider_id?: string; + provider?: string; message?: { id?: string; model?: string; + providerId?: string; + provider_id?: string; + provider?: string; usage?: { input_tokens?: number; output_tokens?: number; @@ -46,6 +40,21 @@ interface ClaudeRecord { }; } +interface ClaudeUsageSnapshot { + input: number; + cached: number; + cacheWrite: number; + cache5m: number; + cache1h: number; + output: number; + costUSD: number; +} + +interface ClaudeSeenUsage { + breakdown: IngestBreakdown; + snapshot: ClaudeUsageSnapshot; +} + function getClaudeProjectDirs(claudeDir?: string): string[] { if (claudeDir) return [claudeDir]; @@ -81,12 +90,9 @@ export async function scanClaudeDates( const baseDirs = getClaudeProjectDirs(claudeDir); - // Global dedup set: compound key messageId:requestId (slopmeter approach). - // Only deduplicates when both IDs are present; entries missing either ID are - // counted without dedup. This mirrors how Claude Code session files are - // re-written on restart — the same completed request appears in multiple - // JSONL files with identical token counts. - const processedHashes = new Set(); + // Claude Code 某些代理/兼容模型没有 requestId，但仍会把同一 messageId + // 复制到父会话和 sidechain 文件；此时退化为 message.id 去重。 + const processedHashes = new Map(); // Track distinct sessions per "date|model|project" group const sessionSets = new Map>(); @@ -118,29 +124,13 @@ export async function scanClaudeDates( } } } + fileJobs.sort((a, b) => a.filePath.localeCompare(b.filePath)); // 并发流式处理文件，直接聚合到 groupedByDate await runWithConcurrency(fileJobs, FILE_CONCURRENCY, async (job) => { await processJsonlFile(job.filePath, job.projectFields, targetDateSet, projectAliases, groupedByDate, processedHashes, sessionSets); }); - - // For dates that have no JSONL data, fall back to stats-cache.json. - // This covers the period before Claude Code's JSONL rotation window. - const missingDates = [...targetDateSet].filter(d => groupedByDate.get(d)?.size === 0); - if (missingDates.length > 0) { - const remaining = new Set(missingDates); - for (const baseDir of baseDirs) { - if (remaining.size === 0) break; - const statsCachePath = join(baseDir, '..', 'stats-cache.json'); - await fillFromStatsCache(statsCachePath, remaining, groupedByDate); - // Remove dates that were just filled - for (const d of remaining) { - if (groupedByDate.get(d)!.size > 0) remaining.delete(d); - } - } - } - // Assign session counts from collected sessionSets for (const [usageDate, grouped] of groupedByDate.entries()) { for (const [key, breakdown] of grouped.entries()) { @@ -161,21 +151,15 @@ async function processJsonlFile( targetDateSet: Set, projectAliases: Record | undefined, groupedByDate: Map>, - processedHashes: Set, + processedHashes: Map, sessionSets: Map>, ): Promise { // Derive fallback sessionId from filename (e.g. "abc-123.jsonl" → "abc-123") const fallbackSessionId = filePath.replace(/^.*[\\/]/, '').replace(/\.jsonl$/, ''); - let fh; - try { - fh = await open(filePath, 'r'); - } catch { - return; - } - + const input = createReadStream(filePath, { encoding: 'utf-8' }); try { const rl = createInterface({ - input: fh.createReadStream({ encoding: 'utf-8' }), + input, crlfDelay: Infinity, }); @@ -204,21 +188,17 @@ async function processJsonlFile( const rawModel = message.model ?? 'unknown'; if (rawModel === '') continue; - // Compound dedup key (slopmeter approach): only skip when BOTH the - // Anthropic message ID and the Claude Code request ID are present. - // Session files are rewritten on restart, so the same completed request - // appears in multiple files with identical token counts. + // 标准记录用 messageId:requestId；兼容模型缺 requestId 时用 message.id。 const messageId = message.id; const requestId = record.requestId; - if (messageId && requestId) { - const hash = `${messageId}:${requestId}`; - if (processedHashes.has(hash)) continue; - processedHashes.add(hash); - } + const hash = messageId + ? (requestId ? `${messageId}:${requestId}` : `message:${messageId}`) + : undefined; const usage = message.usage; let model = normalizeModelName(rawModel); if (usage.speed === 'fast') model = `${model}-fast`; + const provider = resolveClaudeProvider(record, rawModel); const recordFields = record.cwd ? resolveProject(record.cwd, projectAliases) : fallbackFields; const sessionId = record.sessionId ?? fallbackSessionId; const costUSD = record.costUSD ?? 0; @@ -234,8 +214,23 @@ async function processJsonlFile( const grouped = groupedByDate.get(usageDate); if (!grouped) continue; - const cacheWriteTokens = cache5m + cache1h; - const key = `${model}|${recordFields.project}`; + const snapshot: ClaudeUsageSnapshot = { + input: clampToken(usage.input_tokens), + cached: clampToken(usage.cache_read_input_tokens), + cacheWrite: clampToken(usage.cache_creation_input_tokens) + || clampToken(cache5m) + clampToken(cache1h), + cache5m: clampToken(cache5m), + cache1h: clampToken(cache1h), + output: clampToken(usage.output_tokens), + costUSD: clampToken(costUSD), + }; + const cacheWriteTokens = snapshot.cacheWrite; + const seen = hash ? processedHashes.get(hash) : undefined; + if (seen) { + mergeClaudeDuplicate(seen, snapshot); + continue; + } + const key = `${provider}|${model}|${recordFields.project}`; // Track distinct sessions per group const sessionSetKey = `${usageDate}|${key}`; @@ -245,16 +240,17 @@ async function processJsonlFile( const existing = grouped.get(key); if (existing) { existing.eventCount += 1; - existing.inputTokens += usage.input_tokens ?? 0; - existing.cachedInputTokens += usage.cache_read_input_tokens ?? 0; + existing.inputTokens += snapshot.input; + existing.cachedInputTokens += snapshot.cached; existing.cacheWriteTokens += cacheWriteTokens; - existing.cacheWrite5mTokens = (existing.cacheWrite5mTokens ?? 0) + cache5m; - existing.cacheWrite1hTokens = (existing.cacheWrite1hTokens ?? 0) + cache1h; - existing.outputTokens += usage.output_tokens ?? 0; - existing.costUSD = (existing.costUSD ?? 0) + costUSD; + existing.cacheWrite5mTokens = (existing.cacheWrite5mTokens ?? 0) + snapshot.cache5m; + existing.cacheWrite1hTokens = (existing.cacheWrite1hTokens ?? 0) + snapshot.cache1h; + existing.outputTokens += snapshot.output; + existing.costUSD = (existing.costUSD ?? 0) + snapshot.costUSD; + if (hash) processedHashes.set(hash, { breakdown: existing, snapshot }); } else { - grouped.set(key, { - provider: 'anthropic', + const breakdown: IngestBreakdown = { + provider, product: 'claude-code', channel: 'cli', model, @@ -262,90 +258,69 @@ async function processJsonlFile( projectDisplay: recordFields.projectDisplay, projectAlias: recordFields.projectAlias, eventCount: 1, - inputTokens: usage.input_tokens ?? 0, - cachedInputTokens: usage.cache_read_input_tokens ?? 0, + inputTokens: snapshot.input, + cachedInputTokens: snapshot.cached, cacheWriteTokens, - cacheWrite5mTokens: cache5m, - cacheWrite1hTokens: cache1h, - outputTokens: usage.output_tokens ?? 0, + cacheWrite5mTokens: snapshot.cache5m, + cacheWrite1hTokens: snapshot.cache1h, + outputTokens: snapshot.output, reasoningOutputTokens: 0, - costUSD, - }); + costUSD: snapshot.costUSD, + }; + grouped.set(key, breakdown); + if (hash) processedHashes.set(hash, { breakdown, snapshot }); } } + } catch { + // 文件在扫描期间被移动、归档或损坏时跳过该文件。 } finally { - await fh.close(); + input.destroy(); } } -async function fillFromStatsCache( - statsCachePath: string, - missingDates: Set, - groupedByDate: Map>, -): Promise { - let raw: string; - try { - raw = await readFile(statsCachePath, 'utf-8'); - } catch { - return; - } - - let cache: StatsCache; - try { - cache = JSON.parse(raw); - } catch { - return; - } - - // Build input ratio per model from all-time modelUsage aggregates. - // dailyModelTokens stores (input + output) only — no cache tokens. - const inputRatios: Record = {}; - for (const [model, usage] of Object.entries(cache.modelUsage ?? {})) { - const inp = usage.inputTokens ?? 0; - const out = usage.outputTokens ?? 0; - const total = inp + out; - if (total > 0) inputRatios[model] = inp / total; - } - - for (const entry of cache.dailyModelTokens ?? []) { - const { date, tokensByModel } = entry; - if (!missingDates.has(date)) continue; - - const grouped = groupedByDate.get(date); - if (!grouped) continue; - - for (const [rawModel, totalTokens] of Object.entries(tokensByModel)) { - if (!totalTokens) continue; - const model = normalizeModelName(rawModel); - const ratio = inputRatios[rawModel] ?? inputRatios[model] ?? STATS_CACHE_DEFAULT_INPUT_RATIO; +function mergeClaudeDuplicate(seen: ClaudeSeenUsage, incoming: ClaudeUsageSnapshot): void { + const incomingHasNewerCacheWrite = incoming.cacheWrite > seen.snapshot.cacheWrite; + const merged: ClaudeUsageSnapshot = { + input: Math.max(seen.snapshot.input, incoming.input), + cached: Math.max(seen.snapshot.cached, incoming.cached), + cacheWrite: Math.max(seen.snapshot.cacheWrite, incoming.cacheWrite), + cache5m: incomingHasNewerCacheWrite ? incoming.cache5m : seen.snapshot.cache5m, + cache1h: incomingHasNewerCacheWrite ? incoming.cache1h : seen.snapshot.cache1h, + output: Math.max(seen.snapshot.output, incoming.output), + costUSD: Math.max(seen.snapshot.costUSD, incoming.costUSD), + }; + const breakdown = seen.breakdown; + breakdown.inputTokens += merged.input - seen.snapshot.input; + breakdown.cachedInputTokens += merged.cached - seen.snapshot.cached; + breakdown.cacheWrite5mTokens = (breakdown.cacheWrite5mTokens ?? 0) + + merged.cache5m - seen.snapshot.cache5m; + breakdown.cacheWrite1hTokens = (breakdown.cacheWrite1hTokens ?? 0) + + merged.cache1h - seen.snapshot.cache1h; + breakdown.cacheWriteTokens += merged.cacheWrite - seen.snapshot.cacheWrite; + breakdown.outputTokens += merged.output - seen.snapshot.output; + breakdown.costUSD = (breakdown.costUSD ?? 0) + merged.costUSD - seen.snapshot.costUSD; + seen.snapshot = merged; +} - const inputTokens = Math.round(totalTokens * ratio); - const outputTokens = totalTokens - inputTokens; +function clampToken(value: number | undefined): number { + return Math.max(Number.isFinite(value) ? value! : 0, 0); +} - // stats-cache has no per-project breakdown - const key = `${model}|unknown`; - const existing = grouped.get(key); - if (existing) { - existing.eventCount += 1; - existing.inputTokens += inputTokens; - existing.outputTokens += outputTokens; - } else { - grouped.set(key, { - provider: 'anthropic', - product: 'claude-code', - channel: 'cli', - model, - project: 'unknown', - eventCount: 1, - inputTokens, - cachedInputTokens: 0, - cacheWriteTokens: 0, - outputTokens, - reasoningOutputTokens: 0, - }); - } - } - } +function resolveClaudeProvider(record: ClaudeRecord, model: string): string { + const explicit = ( + record.message?.providerId + ?? record.message?.provider_id + ?? record.message?.provider + ?? record.providerId + ?? record.provider_id + ?? record.provider + )?.trim().toLowerCase(); + const prefix = model.trim().toLowerCase().match(/^([a-z0-9_-]+)\//)?.[1]; + const inferred = prefix ?? inferProviderFromModel(model, 'unknown'); + // 一些兼容层会固定写 anthropic，但模型名已明确指向其他供应商。 + if (explicit && explicit !== 'anthropic') return explicit; + if (inferred !== 'unknown') return inferred; + return explicit ?? 'unknown'; } function parseTimestamp(value?: string): Date | null { diff --git a/packages/cli/src/scanners/codex.ts b/packages/cli/src/scanners/codex.ts index ec95f59..def3076 100644 --- a/packages/cli/src/scanners/codex.ts +++ b/packages/cli/src/scanners/codex.ts @@ -1,9 +1,10 @@ -import { readdir, open, readFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { createReadStream } from 'node:fs'; +import { readdir, readFile } from 'node:fs/promises'; +import { basename, join } from 'node:path'; import { homedir } from 'node:os'; import { createInterface } from 'node:readline'; import { calculateCost, type IngestBreakdown } from '@aiusage/shared'; -import { normalizeModelName, runWithConcurrency, resolveProjectFields, type ProjectFields } from './utils.js'; +import { fileModifiedTs, normalizeModelName, runWithConcurrency, resolveProjectFields, type ProjectFields } from './utils.js'; const FILE_CONCURRENCY = 16; const MAX_LINE_BYTES = 64 * 1024 * 1024; // 64 MB @@ -12,25 +13,74 @@ type CodexServiceTier = 'fast' | 'priority' | null; interface CodexRecord { type?: string; timestamp?: string; - payload?: { - type?: string; + payload?: CodexPayload; +} + +interface CodexPayload { + type?: string; + model?: string; + model_name?: string; + model_info?: { slug?: string }; + cwd?: string; + id?: string; + forked_from_id?: string; + source?: unknown; + thread_source?: string; + turn_id?: string; + started_at?: number; + info?: { model?: string; - cwd?: string; - info?: { - last_token_usage?: TokenUsage; - total_token_usage?: TokenUsage; - }; + model_name?: string; + last_token_usage?: TokenUsage; + total_token_usage?: TokenUsage; }; } interface TokenUsage { input_tokens?: number; cached_input_tokens?: number; + cache_read_input_tokens?: number; output_tokens?: number; reasoning_output_tokens?: number; total_tokens?: number; } +interface CodexTotals { + input: number; + cached: number; + output: number; + reasoning: number; +} + +interface CodexFileState { + currentModel: string; + projectFields: ProjectFields; + hasSessionWorkspace: boolean; + previousTotals?: CodexTotals; + sessionIdFromMeta?: string; + sessionForkedFromId?: string; + forkedChildSessionId?: string; + forkedChildReplaySessionId?: string; + waitingForChildTurn: boolean; + inheritedBaseline?: CodexTotals; + inheritedReportedTotal?: number; + taskStartedTurnIds: Set; + childIsUserFork: boolean; +} + +interface CodexUsageEvent { + usageDate: string; + signature: string; + model: string; + projectFields: ProjectFields; + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; + reasoningOutputTokens: number; + costUSD?: number; + pricingVersion?: string; +} + export async function scanCodex( targetDate: string, codexDir?: string, @@ -57,14 +107,24 @@ export async function scanCodexDates( return new Map([...targetDateSet].map((targetDate) => [targetDate, []])); } - // 跨文件全局签名去重 - const globalSeenSigs = new Set(); - - // 并发流式处理文件 - await runWithConcurrency(sessionFiles, FILE_CONCURRENCY, async (filePath) => { - await processCodexFile(filePath, targetDateSet, projectAliases, groupedByDate, globalSeenSigs, serviceTier); + // 文件并发解析、按稳定路径顺序合并：既保留扫描速度，也让 fork 重放的 + // first-wins 归属不受异步完成顺序影响。 + const eventsByFile: CodexUsageEvent[][] = Array.from({ length: sessionFiles.length }, () => []); + await runWithConcurrency(sessionFiles, FILE_CONCURRENCY, async (filePath, index) => { + eventsByFile[index] = await processCodexFile(filePath, projectAliases, serviceTier); }); + // 跨文件去重，但 key 必须带 session/fork scope；纯 token 数值全局去重会误杀独立会话。 + const globalSeenSigs = new Set(); + for (const events of eventsByFile) { + for (const event of events) { + if (globalSeenSigs.has(event.signature)) continue; + globalSeenSigs.add(event.signature); + if (!targetDateSet.has(event.usageDate)) continue; + mergeCodexEvent(groupedByDate, event); + } + } + return new Map( [...groupedByDate.entries()].map(([usageDate, grouped]) => [usageDate, [...grouped.values()]]), ); @@ -73,28 +133,27 @@ export async function scanCodexDates( /** 流式逐行读取单个 Codex JSONL 文件 */ async function processCodexFile( filePath: string, - targetDateSet: Set, projectAliases: Record | undefined, - groupedByDate: Map>, - globalSeenSigs: Set, serviceTier: CodexServiceTier, -): Promise { - let fh; - try { - fh = await open(filePath, 'r'); - } catch { - return; - } - +): Promise { + const events: CodexUsageEvent[] = []; + const input = createReadStream(filePath, { encoding: 'utf-8' }); try { const rl = createInterface({ - input: fh.createReadStream({ encoding: 'utf-8' }), + input, crlfDelay: Infinity, }); - let currentModel = 'unknown'; - let currentProjectFields: ProjectFields = { project: 'unknown', projectDisplay: 'unknown' }; - let previousTotal: TokenUsage = {}; + const fileSessionId = basename(filePath, '.jsonl'); + const fallbackTs = await fileModifiedTs(filePath); + const state: CodexFileState = { + currentModel: 'unknown', + projectFields: { project: 'unknown', projectDisplay: 'unknown' }, + hasSessionWorkspace: false, + waitingForChildTurn: false, + taskStartedTurnIds: new Set(), + childIsUserFork: false, + }; for await (const line of rl) { if (!line) continue; @@ -109,127 +168,359 @@ async function processCodexFile( continue; } - if (record.type === 'turn_context') { - const rawModel = record.payload?.model ?? currentModel; - // 过滤合成消息 - if (rawModel !== '') { - currentModel = applyCodexServiceTier(normalizeModelName(rawModel), serviceTier); + const payload = record.payload; + if (!payload) continue; + const payloadModel = extractCodexModel(payload); + const isTokenCount = record.type === 'event_msg' && payload.type === 'token_count'; + + // 子会话文件会先复制父会话历史。保持 child 的 workspace/session 状态， + // 直到能确定自己的 turn_context，再开始计数。 + if (state.waitingForChildTurn) { + if (record.type === 'turn_context' && childTurnStartsOwnSession(state, payload.turn_id)) { + state.waitingForChildTurn = false; + state.forkedChildReplaySessionId = undefined; + state.taskStartedTurnIds.clear(); + state.childIsUserFork = false; + state.sessionIdFromMeta = state.forkedChildSessionId ?? state.sessionIdFromMeta; + } else { + if (record.type === 'event_msg' && payload.type === 'task_started' + && childTaskStartsOwnSession(state, payload.turn_id, payload.started_at)) { + if (payload.turn_id) state.taskStartedTurnIds.add(payload.turn_id); + } + if (record.type === 'session_meta' && payload.id + && payload.id !== state.forkedChildSessionId) { + state.forkedChildReplaySessionId = payload.id; + } + if (isTokenCount) rememberInheritedBaseline(state, payload.info); + continue; } - if (record.payload?.cwd) { - currentProjectFields = resolveProjectFields(record.payload.cwd, projectAliases); + } + + if (record.type === 'session_meta') { + const sessionId = payload.id?.trim(); + if (sessionId) state.sessionIdFromMeta = sessionId; + const forkParent = payload.forked_from_id?.trim() ?? forkParentFromSource(payload.source); + if (forkParent) { + const repeatedActiveChild = !state.waitingForChildTurn + && Boolean(sessionId) + && state.forkedChildSessionId === sessionId; + state.sessionForkedFromId = forkParent; + state.forkedChildSessionId = sessionId; + if (!repeatedActiveChild) { + state.waitingForChildTurn = true; + state.forkedChildReplaySessionId = undefined; + state.inheritedBaseline = undefined; + state.inheritedReportedTotal = undefined; + state.taskStartedTurnIds.clear(); + state.childIsUserFork = payload.thread_source === 'user'; + } + } + if (payload.cwd) { + state.projectFields = resolveProjectFields(payload.cwd, projectAliases); + state.hasSessionWorkspace = true; } continue; } - if (record.type !== 'event_msg') continue; - if (record.payload?.type !== 'token_count') continue; + if (record.type === 'turn_context') { + if (payloadModel && payloadModel !== '') { + state.currentModel = applyCodexServiceTier(normalizeModelName(payloadModel), serviceTier); + } + if (payload.cwd && !state.hasSessionWorkspace) { + state.projectFields = resolveProjectFields(payload.cwd, projectAliases); + } + continue; + } - const info = record.payload?.info; - if (!info?.total_token_usage) continue; + if (!isTokenCount) continue; + const info = payload.info; + if (!info?.total_token_usage && !info?.last_token_usage) continue; - const ts = parseTimestamp(record.timestamp); + if (payloadModel && payloadModel !== '') { + state.currentModel = applyCodexServiceTier(normalizeModelName(payloadModel), serviceTier); + } + const ts = parseTimestamp(record.timestamp) ?? fallbackTs; if (!ts) continue; const usageDate = toDateKey(ts); - if (!targetDateSet.has(usageDate)) continue; - - // 跨文件全局去重：total_token_usage 在会话内单调累加，相同的非零累计值 - // 只可能来自 fork/resume 复制行或重复 emit，可安全去重。 - const total = info.total_token_usage; - const totalSum = - (total.input_tokens ?? 0) + - (total.cached_input_tokens ?? 0) + - (total.output_tokens ?? 0) + - (total.reasoning_output_tokens ?? 0) + - (total.total_tokens ?? 0); - // 全零累计是每个会话开头都有的噪声，跨会话签名相同会被误杀， - // 故全零既不参与去重也不计入用量（last 也必为 0，无影响）。 - if (totalSum === 0) continue; - const signature = `${total.input_tokens ?? 0}|${total.cached_input_tokens ?? 0}|${total.output_tokens ?? 0}|${total.reasoning_output_tokens ?? 0}|${total.total_tokens ?? 0}`; - if (globalSeenSigs.has(signature)) continue; - globalSeenSigs.add(signature); - - // Use last_token_usage when available; otherwise compute delta from total_token_usage - const last: TokenUsage = info.last_token_usage ?? { - input_tokens: Math.max(0, (total.input_tokens ?? 0) - (previousTotal.input_tokens ?? 0)), - cached_input_tokens: Math.max(0, (total.cached_input_tokens ?? 0) - (previousTotal.cached_input_tokens ?? 0)), - output_tokens: Math.max(0, (total.output_tokens ?? 0) - (previousTotal.output_tokens ?? 0)), - reasoning_output_tokens: Math.max(0, (total.reasoning_output_tokens ?? 0) - (previousTotal.reasoning_output_tokens ?? 0)), - }; - previousTotal = total; - - // In Codex JSONL, input_tokens includes cached_input_tokens. - // Subtract to get the non-cached portion so cost formula works uniformly. - const nonCachedInput = Math.max(0, (last.input_tokens ?? 0) - (last.cached_input_tokens ?? 0)); - const cachedInput = last.cached_input_tokens ?? 0; - const output = last.output_tokens ?? 0; - const eventCost = calculateCost('openai', 'codex', currentModel, { + + const total = info.total_token_usage ? totalsFromUsage(info.total_token_usage) : undefined; + const last = info.last_token_usage ? totalsFromUsage(info.last_token_usage) : undefined; + if (shouldSkipInheritedSnapshot(state, info.total_token_usage, total)) continue; + state.inheritedBaseline = undefined; + state.inheritedReportedTotal = undefined; + + const parsed = resolveCodexIncrement(total, last, state.previousTotals); + if (!parsed) continue; + if (!parsed.tokens) { + state.previousTotals = parsed.nextTotals; + continue; + } + const { tokens, nextTotals } = parsed; + const cachedInput = Math.min(tokens.cached, tokens.input); + const nonCachedInput = Math.max(tokens.input - cachedInput, 0); + const output = tokens.output; + const reasoning = tokens.reasoning; + if (nonCachedInput + cachedInput + output + reasoning === 0) continue; + state.previousTotals = nextTotals; + + const dedupScope = state.sessionForkedFromId + ?? state.sessionIdFromMeta + ?? fileSessionId; + const signature = total + ? `codex|${dedupScope}|${state.currentModel}|${total.input}|${total.cached}|${total.output}|${total.reasoning}` + : `codex|${dedupScope}|${state.currentModel}|${ts.getTime()}|${tokens.input}|${tokens.cached}|${tokens.output}|${tokens.reasoning}`; + + const eventCost = calculateCost('openai', 'codex', state.currentModel, { inputTokens: nonCachedInput, cachedInputTokens: cachedInput, cacheWriteTokens: 0, outputTokens: output, }); const exactEventCost = eventCost.costStatus === 'exact' ? eventCost.estimatedCostUsd : undefined; - - const grouped = groupedByDate.get(usageDate); - if (!grouped) continue; - const key = `${currentModel}|${currentProjectFields.project}`; - - const existing = grouped.get(key); - if (existing) { - existing.eventCount += 1; - existing.inputTokens += nonCachedInput; - existing.cachedInputTokens += cachedInput; - existing.outputTokens += output; - existing.reasoningOutputTokens += last.reasoning_output_tokens ?? 0; - if (exactEventCost !== undefined) { - existing.costUSD = (existing.costUSD ?? 0) + exactEventCost; - existing.pricingVersion = eventCost.pricingVersion; - } - } else { - const breakdown: IngestBreakdown = { - provider: 'openai', - product: 'codex', - channel: 'cli', - model: currentModel, - project: currentProjectFields.project, - projectDisplay: currentProjectFields.projectDisplay, - projectAlias: currentProjectFields.projectAlias, - eventCount: 1, - inputTokens: nonCachedInput, - cachedInputTokens: cachedInput, - cacheWriteTokens: 0, - outputTokens: output, - reasoningOutputTokens: last.reasoning_output_tokens ?? 0, - }; - if (exactEventCost !== undefined) { - breakdown.costUSD = exactEventCost; - breakdown.pricingVersion = eventCost.pricingVersion; - } - grouped.set(key, breakdown); - } + events.push({ + usageDate, + signature, + model: state.currentModel, + projectFields: { ...state.projectFields }, + inputTokens: nonCachedInput, + cachedInputTokens: cachedInput, + outputTokens: output, + reasoningOutputTokens: reasoning, + costUSD: exactEventCost, + pricingVersion: exactEventCost !== undefined ? eventCost.pricingVersion : undefined, + }); } + } catch { + // 文件在扫描期间被移动、归档或损坏时跳过该文件。 } finally { - await fh.close(); + input.destroy(); } + return events; } -async function collectSessionFiles(baseDir: string): Promise { - const paths: string[] = []; - - // archived_sessions/*.jsonl - const archivedDir = join(baseDir, 'archived_sessions'); - try { - const files = await readdir(archivedDir); - for (const f of files) { - if (f.endsWith('.jsonl')) paths.push(join(archivedDir, f)); +function mergeCodexEvent( + groupedByDate: Map>, + event: CodexUsageEvent, +): void { + const grouped = groupedByDate.get(event.usageDate); + if (!grouped) return; + const key = `${event.model}|${event.projectFields.project}`; + const existing = grouped.get(key); + if (existing) { + existing.eventCount += 1; + existing.inputTokens += event.inputTokens; + existing.cachedInputTokens += event.cachedInputTokens; + existing.outputTokens += event.outputTokens; + existing.reasoningOutputTokens += event.reasoningOutputTokens; + if (event.costUSD !== undefined) { + existing.costUSD = (existing.costUSD ?? 0) + event.costUSD; + existing.pricingVersion = event.pricingVersion; } - } catch { /* ignore */ } + return; + } + + const breakdown: IngestBreakdown = { + provider: 'openai', + product: 'codex', + channel: 'cli', + model: event.model, + project: event.projectFields.project, + projectDisplay: event.projectFields.projectDisplay, + projectAlias: event.projectFields.projectAlias, + eventCount: 1, + inputTokens: event.inputTokens, + cachedInputTokens: event.cachedInputTokens, + cacheWriteTokens: 0, + outputTokens: event.outputTokens, + reasoningOutputTokens: event.reasoningOutputTokens, + }; + if (event.costUSD !== undefined) { + breakdown.costUSD = event.costUSD; + breakdown.pricingVersion = event.pricingVersion; + } + grouped.set(key, breakdown); +} + +function forkParentFromSource(source: unknown): string | undefined { + if (!source || typeof source !== 'object' || Array.isArray(source)) return undefined; + const subagent = (source as Record).subagent; + if (!subagent || typeof subagent !== 'object' || Array.isArray(subagent)) return undefined; + const spawn = (subagent as Record).thread_spawn; + if (!spawn || typeof spawn !== 'object' || Array.isArray(spawn)) return undefined; + const parent = (spawn as Record).parent_thread_id; + return typeof parent === 'string' && parent.trim() ? parent.trim() : undefined; +} + +function clampToken(value: number | undefined): number { + return Math.max(Number.isFinite(value) ? value! : 0, 0); +} - // sessions/**/*.jsonl (递归) +function extractCodexModel(payload: CodexPayload): string | undefined { + return payload.model + ?? payload.model_name + ?? payload.model_info?.slug + ?? payload.info?.model + ?? payload.info?.model_name; +} + +function totalsFromUsage(usage: TokenUsage): CodexTotals { + return { + input: clampToken(usage.input_tokens), + cached: Math.max( + clampToken(usage.cached_input_tokens), + clampToken(usage.cache_read_input_tokens), + ), + output: clampToken(usage.output_tokens), + reasoning: clampToken(usage.reasoning_output_tokens), + }; +} + +function totalsEqual(a: CodexTotals, b: CodexTotals): boolean { + return a.input === b.input + && a.cached === b.cached + && a.output === b.output + && a.reasoning === b.reasoning; +} + +function totalsDelta(current: CodexTotals, previous: CodexTotals): CodexTotals | undefined { + if (current.input < previous.input || current.cached < previous.cached + || current.output < previous.output || current.reasoning < previous.reasoning) return undefined; + return { + input: current.input - previous.input, + cached: current.cached - previous.cached, + output: current.output - previous.output, + reasoning: current.reasoning - previous.reasoning, + }; +} + +function totalsAdd(a: CodexTotals, b: CodexTotals): CodexTotals { + return { + input: a.input + b.input, + cached: a.cached + b.cached, + output: a.output + b.output, + reasoning: a.reasoning + b.reasoning, + }; +} + +function totalsSum(value: CodexTotals): number { + return value.input + value.cached + value.output + value.reasoning; +} + +function looksLikeStaleRegression(current: CodexTotals, previous: CodexTotals, last: CodexTotals): boolean { + const previousSum = totalsSum(previous); + const currentSum = totalsSum(current); + const lastSum = totalsSum(last); + if (previousSum <= 0 || currentSum <= 0 || lastSum <= 0) return false; + return currentSum * 100 >= previousSum * 98 || currentSum + lastSum * 2 >= previousSum; +} + +function resolveCodexIncrement( + total: CodexTotals | undefined, + last: CodexTotals | undefined, + previous: CodexTotals | undefined, +): { tokens?: CodexTotals; nextTotals?: CodexTotals } | undefined { + if (total && last && previous) { + if (totalsEqual(total, previous)) return undefined; + if (!totalsDelta(total, previous) && looksLikeStaleRegression(total, previous, last)) return undefined; + return { tokens: last, nextTotals: total }; + } + if (total && last) return { tokens: last, nextTotals: total }; + if (total && previous) { + if (totalsEqual(total, previous)) return undefined; + const delta = totalsDelta(total, previous); + return delta ? { tokens: delta, nextTotals: total } : { nextTotals: total }; + } + if (total) return { tokens: total, nextTotals: total }; + if (last && previous) return { tokens: last, nextTotals: totalsAdd(previous, last) }; + if (last) return { tokens: last }; + return undefined; +} + +function rememberInheritedBaseline( + state: CodexFileState, + info: CodexPayload['info'], +): void { + const usage = info?.total_token_usage; + if (!usage) return; + const totals = totalsFromUsage(usage); + state.previousTotals = totals; + state.inheritedBaseline = totals; + state.inheritedReportedTotal = reportedTotalTokens(usage); +} + +function shouldSkipInheritedSnapshot( + state: CodexFileState, + usage: TokenUsage | undefined, + totals: CodexTotals | undefined, +): boolean { + const reported = usage ? reportedTotalTokens(usage) : undefined; + if (reported != null && state.inheritedReportedTotal != null + && reported <= state.inheritedReportedTotal) return true; + const baseline = state.inheritedBaseline; + return Boolean(totals && baseline + && totals.input <= baseline.input + && totals.cached <= baseline.cached + && totals.output <= baseline.output + && totals.reasoning <= baseline.reasoning); +} + +function reportedTotalTokens(usage: TokenUsage): number | undefined { + return typeof usage.total_tokens === 'number' && usage.total_tokens >= 0 + ? usage.total_tokens + : undefined; +} + +function childTurnStartsOwnSession(state: CodexFileState, turnId: string | undefined): boolean { + if (!state.forkedChildReplaySessionId) return true; + const childId = state.forkedChildSessionId; + if (!childId) return true; + const childKey = uuidV7OrderKey(childId); + if (!turnId || !childKey) return true; + const turnKey = uuidV7OrderKey(turnId); + if (!turnKey) return state.childIsUserFork || state.taskStartedTurnIds.has(turnId); + const turnMs = turnKey.slice(0, 12); + const childMs = childKey.slice(0, 12); + if (turnMs > childMs) return true; + if (turnMs < childMs) return false; + return state.childIsUserFork || state.taskStartedTurnIds.has(turnId); +} + +function childTaskStartsOwnSession( + state: CodexFileState, + turnId: string | undefined, + startedAt: number | undefined, +): boolean { + const childId = state.forkedChildSessionId; + if (!turnId || !childId) return false; + const childKey = uuidV7OrderKey(childId); + if (!childKey) return true; + const turnKey = uuidV7OrderKey(turnId); + if (turnKey) return turnKey.slice(0, 12) >= childKey.slice(0, 12); + const childStartedMs = Number.parseInt(childKey.slice(0, 12), 16); + return Number.isFinite(startedAt) && startedAt! >= Math.floor(childStartedMs / 1_000); +} + +function uuidV7OrderKey(id: string): string | undefined { + const parts = id.split('-'); + if (parts.length !== 5 || parts[0]?.length !== 8 || parts[1]?.length !== 4 + || parts[2]?.length !== 4 || parts[3]?.length !== 4 || parts[4]?.length !== 12 + || !parts[2]?.startsWith('7') || !parts.every(part => /^[0-9a-f]+$/i.test(part))) return undefined; + return parts.join('').toLowerCase(); +} + +async function collectSessionFiles(baseDir: string): Promise { + // 与 Codex 本身的生命周期一致：活动会话优先，归档副本随后；各目录 + // 内按路径排序，确保重复事件的 workspace 归属稳定。 + const activePaths: string[] = []; const sessionsDir = join(baseDir, 'sessions'); - await walkDir(sessionsDir, paths); + await walkDir(sessionsDir, activePaths); + activePaths.sort(); + + const archivedPaths: string[] = []; + await walkDir(join(baseDir, 'archived_sessions'), archivedPaths); + archivedPaths.sort(); - return paths; + return [...activePaths, ...archivedPaths]; } async function detectCodexServiceTier(baseDir: string): Promise { diff --git a/packages/cli/src/scanners/copilot-vscode.ts b/packages/cli/src/scanners/copilot-vscode.ts index 01e30f7..eff44eb 100644 --- a/packages/cli/src/scanners/copilot-vscode.ts +++ b/packages/cli/src/scanners/copilot-vscode.ts @@ -4,12 +4,15 @@ import { homedir } from 'node:os'; import type { IngestBreakdown } from '@aiusage/shared'; import { dateKey, + inferProviderFromModel, + parseTs, projectFromPath, walkFiles, initDateMap, accumulate, finalize, emptyResult, + normalizeModelName, } from './utils.js'; export async function scanCopilotVscodeDates( @@ -21,9 +24,17 @@ export async function scanCopilotVscodeDates( const dates = new Set(targetDates); const dir = baseDir ?? join(homedir(), 'Library', 'Application Support', 'Code', 'logs'); const workspaceDir = workspaceStorageDir ?? join(homedir(), 'Library', 'Application Support', 'Code', 'User', 'workspaceStorage'); - const logFiles = (await walkFiles(dir, '.log')).filter((filePath) => basename(filePath) === 'GitHub Copilot Chat.log'); - const sessionFiles = (await walkFiles(workspaceDir, '.json')).filter((filePath) => filePath.includes('/chatSessions/')); - if (logFiles.length === 0 && sessionFiles.length === 0) return emptyResult(dates); + const [allLogs, allJson, allJsonl] = await Promise.all([ + walkFiles(dir, '.log'), + walkFiles(workspaceDir, '.json'), + walkFiles(workspaceDir, '.jsonl'), + ]); + const logFiles = allLogs.filter((filePath) => basename(filePath) === 'GitHub Copilot Chat.log'); + const sessionFiles = allJson.filter(isChatSessionFile); + const crdtSessionFiles = allJsonl.filter(isChatSessionFile); + if (logFiles.length === 0 && sessionFiles.length === 0 && crdtSessionFiles.length === 0) { + return emptyResult(dates); + } const grouped = initDateMap(dates); const seenEvents = new Set(); @@ -79,6 +90,9 @@ export async function scanCopilotVscodeDates( for (const filePath of sessionFiles) { await collectWorkspaceSessionEvents(filePath, grouped, seenEvents, projectAliases); } + for (const filePath of crdtSessionFiles) { + await collectWorkspaceJsonlEvents(filePath, grouped, seenEvents, projectAliases); + } return finalize(grouped); } @@ -119,22 +133,31 @@ function extractFileUriPath(line: string): string | null { } } -interface CopilotWorkspaceSession { - requests?: Array<{ - requestId?: string; - timestamp?: number | string; - modelId?: string; - response?: unknown[]; - result?: { - errorDetails?: { - responseIsIncomplete?: boolean; - }; +interface CopilotWorkspaceRequest { + requestId?: string; + timestamp?: number | string; + modelId?: string; + promptTokens?: number; + completionTokens?: number; + response?: unknown[]; + result?: { + errorDetails?: { responseIsIncomplete?: boolean }; + metadata?: { + promptTokens?: number; + outputTokens?: number; + resolvedModel?: string; + toolCallRounds?: Array<{ thinking?: { tokens?: number } }>; }; - }>; + }; +} + +interface CopilotWorkspaceSession { + requests?: CopilotWorkspaceRequest[]; } interface WorkspaceDescriptor { folder?: string; + workspace?: string; } async function collectWorkspaceSessionEvents( @@ -198,6 +221,104 @@ async function collectWorkspaceSessionEvents( } } +async function collectWorkspaceJsonlEvents( + filePath: string, + grouped: Map>, + seenEvents: Set, + projectAliases?: Record, +): Promise { + let content: string; + try { + content = await readFile(filePath, 'utf-8'); + } catch { + return; + } + + const workspacePath = await readWorkspaceFolderPath(filePath); + const project = workspacePath ? projectFromPath(workspacePath, projectAliases) : 'unknown'; + const sessionId = basename(filePath, '.jsonl'); + + for (const line of content.split('\n')) { + if (!line.trim()) continue; + let record: { kind?: number; k?: unknown[]; v?: unknown }; + try { + record = JSON.parse(line) as { kind?: number; k?: unknown[]; v?: unknown }; + } catch { + continue; + } + + let requests: CopilotWorkspaceRequest[] = []; + if (record.kind === 0 && isRecord(record.v)) { + const value = record.v.requests; + if (Array.isArray(value)) requests = value.filter(isRecord) as CopilotWorkspaceRequest[]; + } else if (record.kind === 2 && record.k?.[0] === 'requests' && Array.isArray(record.v)) { + requests = record.v.filter(isRecord) as CopilotWorkspaceRequest[]; + } + + for (const request of requests) { + addTokenizedWorkspaceRequest(request, sessionId, project, grouped, seenEvents); + } + } +} + +function addTokenizedWorkspaceRequest( + request: CopilotWorkspaceRequest, + sessionId: string, + project: string, + grouped: Map>, + seenEvents: Set, +): void { + const metadata = request.result?.metadata; + const resolvedModel = metadata?.resolvedModel?.trim(); + const rawModelId = request.modelId?.trim(); + if (!resolvedModel && !rawModelId?.startsWith('copilot/')) return; + + const input = clampToken(request.promptTokens ?? metadata?.promptTokens); + const output = clampToken(request.completionTokens ?? metadata?.outputTokens); + if (input === 0 && output === 0) return; + + const timestamp = parseTs(request.timestamp); + if (!timestamp) return; + const dayMap = grouped.get(dateKey(timestamp)); + if (!dayMap) return; + + const model = normalizeTokenizedWorkspaceModel(resolvedModel || rawModelId) || 'auto'; + const reasoning = (metadata?.toolCallRounds ?? []).reduce( + (sum, round) => sum + clampToken(round.thinking?.tokens), + 0, + ); + const dedupeKey = `copilot-vscode|${sessionId}|${timestamp.getTime()}`; + if (seenEvents.has(dedupeKey)) return; + seenEvents.add(dedupeKey); + + const provider = inferProviderFromModel(model, 'github-copilot'); + accumulate( + dayMap, + `${provider}|${model}|${project}`, + { + provider, + product: 'copilot-vscode', + channel: 'ide', + model, + project, + inputTokens: 0, + cachedInputTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningOutputTokens: 0, + }, + { input, cached: 0, cacheWrite: 0, output, reasoning }, + ); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function clampToken(value: number | undefined): number { + return Math.max(Number.isFinite(value) ? value! : 0, 0); +} + async function readWorkspaceFolderPath(sessionFilePath: string): Promise { const workspaceJsonPath = join(dirname(sessionFilePath), '..', 'workspace.json'); let content: string; @@ -209,8 +330,8 @@ async function readWorkspaceFolderPath(sessionFilePath: string): Promise; @@ -49,12 +54,16 @@ export async function scanCopilotDates( projectAliases?: Record, ): Promise> { const dates = new Set(targetDates); - const dir = baseDir ?? join(homedir(), '.copilot', 'session-state'); - - const files = await walkFiles(dir, '.jsonl'); + const dirs = baseDir + ? [baseDir] + : [join(homedir(), '.copilot', 'session-state'), join(homedir(), '.copilot', 'otel')]; + const files = [...new Set((await Promise.all(dirs.map(dir => walkFiles(dir, '.jsonl')))).flat())]; + const explicitOtel = baseDir ? undefined : process.env.COPILOT_OTEL_FILE_EXPORTER_PATH?.trim(); + if (explicitOtel) files.push(explicitOtel); if (files.length === 0) return emptyResult(dates); const grouped = initDateMap(dates); + const seenOtel = new Set(); for (const filePath of files) { let content: string; @@ -65,16 +74,19 @@ export async function scanCopilotDates( } let sessionProjectFields: ProjectFields = { project: 'unknown', projectDisplay: 'unknown' }; + const records: Record[] = []; - const lines = content.split('\n'); - for (const line of lines) { + for (const line of content.split('\n')) { if (!line.trim()) continue; - let obj: CopilotEvent; + let parsed: unknown; try { - obj = JSON.parse(line); + parsed = JSON.parse(line) as unknown; } catch { continue; } + if (!isObject(parsed)) continue; + records.push(parsed); + const obj = parsed as CopilotEvent; // 从 session.start / session.resume 获取 project if (obj.type === 'session.start' || obj.type === 'session.resume') { @@ -99,10 +111,12 @@ export async function scanCopilotDates( const usage = entry.usage; if (!usage) continue; - const rawInput = usage.inputTokens ?? 0; - const cacheRead = usage.cacheReadTokens ?? 0; - const output = usage.outputTokens ?? 0; - const input = Math.max(rawInput - cacheRead, 0); + const rawInput = clamp(usage.inputTokens); + const cacheRead = clamp(usage.cacheReadTokens); + const cacheWrite = clamp(usage.cacheWriteTokens); + const output = clamp(usage.outputTokens); + const reasoning = clamp(usage.reasoningTokens); + const input = Math.max(rawInput - Math.min(cacheRead, rawInput), 0); accumulate( dayMap, @@ -121,11 +135,226 @@ export async function scanCopilotDates( outputTokens: 0, reasoningOutputTokens: 0, }, - { input, cached: cacheRead, cacheWrite: 0, output, reasoning: 0 }, + { input, cached: cacheRead, cacheWrite, output, reasoning }, ); } } + + const fallbackTs = await fileModifiedTs(filePath); + if (!fallbackTs) continue; + for (const candidate of selectOtelCandidates(records, fallbackTs)) { + if (seenOtel.has(candidate.dedupKey)) continue; + seenOtel.add(candidate.dedupKey); + const dayMap = grouped.get(dateKey(candidate.timestamp)); + if (!dayMap) continue; + accumulate( + dayMap, + `${candidate.provider}|${candidate.model}|unknown`, + { + provider: candidate.provider, + product: 'copilot-cli', + channel: 'cli', + model: candidate.model, + project: 'unknown', + projectDisplay: 'unknown', + inputTokens: 0, + cachedInputTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningOutputTokens: 0, + }, + candidate.tokens, + ); + } } return finalize(grouped); } + +type OtelSource = 'chat' | 'inference' | 'agent-turn' | 'summary'; + +interface OtelCandidate { + source: OtelSource; + traceId?: string; + responseId?: string; + model: string; + provider: string; + timestamp: Date; + dedupKey: string; + tokens: { input: number; cached: number; cacheWrite: number; output: number; reasoning: number }; +} + +function selectOtelCandidates(records: Record[], fallbackTs: Date): OtelCandidate[] { + const candidates = records.flatMap((record, index) => { + const candidate = otelCandidate(record, index, fallbackTs); + return candidate ? [candidate] : []; + }); + const traces = (source: OtelSource) => new Set(candidates.filter(c => c.source === source).flatMap(c => c.traceId ? [c.traceId] : [])); + const responses = (source: OtelSource) => new Set(candidates.filter(c => c.source === source).flatMap(c => c.responseId ? [c.responseId] : [])); + const chatTraces = traces('chat'); + const inferenceTraces = traces('inference'); + const agentTraces = traces('agent-turn'); + const chatResponses = responses('chat'); + const inferenceResponses = responses('inference'); + const agentResponses = responses('agent-turn'); + + return candidates.filter(candidate => { + if (candidate.source === 'chat') return true; + if (candidate.source === 'inference') { + return !(candidate.traceId && chatTraces.has(candidate.traceId)) + && !(candidate.responseId && chatResponses.has(candidate.responseId)); + } + if (candidate.source === 'agent-turn') { + return !(candidate.traceId && (chatTraces.has(candidate.traceId) || inferenceTraces.has(candidate.traceId))) + && !(candidate.responseId && (chatResponses.has(candidate.responseId) || inferenceResponses.has(candidate.responseId))); + } + return !(candidate.traceId && ( + chatTraces.has(candidate.traceId) + || inferenceTraces.has(candidate.traceId) + || agentTraces.has(candidate.traceId) + )) && !(candidate.responseId && ( + chatResponses.has(candidate.responseId) + || inferenceResponses.has(candidate.responseId) + || agentResponses.has(candidate.responseId) + )); + }); +} + +function otelCandidate( + record: Record, + index: number, + fallbackTs: Date, +): OtelCandidate | undefined { + const attributes = isObject(record.attributes) ? record.attributes : undefined; + if (!attributes) return undefined; + const source = otelSource(record, attributes); + if (!source) return undefined; + + const rawInput = attrNumber(attributes, ['gen_ai.usage.input_tokens']); + const cached = attrNumber(attributes, [ + 'gen_ai.usage.cache_read.input_tokens', 'gen_ai.usage.cache_read_input_tokens', + ]); + const cacheWrite = attrNumber(attributes, [ + 'gen_ai.usage.cache_write.input_tokens', 'gen_ai.usage.cache_creation.input_tokens', + 'gen_ai.usage.cache_write_input_tokens', 'gen_ai.usage.cache_creation_input_tokens', + ]); + const output = attrNumber(attributes, ['gen_ai.usage.output_tokens']); + const reasoning = attrNumber(attributes, [ + 'gen_ai.usage.reasoning.output_tokens', 'gen_ai.usage.reasoning_tokens', + ]); + const input = Math.max(rawInput - Math.min(cached, rawInput), 0); + if (input + cached + cacheWrite + output + reasoning === 0) return undefined; + + const traceId = stringAttr(record, ['traceId']) + ?? (isObject(record.spanContext) ? stringAttr(record.spanContext, ['traceId']) : undefined); + const spanId = stringAttr(record, ['spanId']) + ?? (isObject(record.spanContext) ? stringAttr(record.spanContext, ['spanId']) : undefined); + const responseId = stringAttr(attributes, ['gen_ai.response.id']); + const sessionId = stringAttr(attributes, [ + 'gen_ai.conversation.id', 'session.id', 'github.copilot.session_id', + 'github.copilot.interaction_id', 'copilot_chat.session.id', + ]) ?? traceId ?? 'unknown-session'; + const model = stringAttr(attributes, [ + 'gen_ai.response.model', 'gen_ai.request.model', 'gen_ai.system', 'copilot_chat.model', + ]) ?? 'unknown'; + const provider = inferProviderFromModel(model, 'github-copilot'); + const timestamp = otelRecordTimestamp(record) ?? fallbackTs; + const dedupKey = responseId + ? `copilot:${source}:response:${responseId}` + : traceId && spanId + ? `copilot:${source}:${traceId}:${spanId}` + : `copilot:${source}:${sessionId}:${timestamp.getTime()}:${index}`; + return { + source, traceId, responseId, model, provider, timestamp, dedupKey, + tokens: { input, cached, cacheWrite, output, reasoning }, + }; +} + +function otelSource( + record: Record, + attributes: Record, +): OtelSource | undefined { + const name = typeof record.name === 'string' ? record.name : ''; + const body = typeof record.body === 'string' ? record.body : typeof record._body === 'string' ? record._body : ''; + const eventName = typeof attributes['event.name'] === 'string' ? attributes['event.name'] : ''; + const operation = typeof attributes['gen_ai.operation.name'] === 'string' ? attributes['gen_ai.operation.name'] : ''; + const span = record.type === 'span' || (record.type == null && Boolean(name) && ( + record.spanId != null || record.traceId != null || record.startTime != null || record.endTime != null + )); + if (span && (operation === 'chat' || name.startsWith('chat '))) return 'chat'; + if (!span && (eventName === 'gen_ai.client.inference.operation.details' || body.startsWith('GenAI inference:'))) return 'inference'; + if (!span && (eventName === 'copilot_chat.agent.turn' || body.startsWith('copilot_chat.agent.turn'))) return 'agent-turn'; + if (span && (operation === 'invoke_agent' || name.startsWith('invoke_agent'))) return 'summary'; + return undefined; +} + +function otelTimestamp(value: unknown): Date | null { + if (Array.isArray(value) && value.length >= 1) { + const seconds = Number(value[0]); + const nanos = Number(value[1] ?? 0); + if (Number.isFinite(seconds) && Number.isFinite(nanos)) return parseTs(seconds * 1_000 + nanos / 1_000_000); + } + if (typeof value === 'number' || typeof value === 'string') return otelScalarTimestamp(value); + return null; +} + +function otelRecordTimestamp(record: Record): Date | null { + const start = otelTimestamp(record.startTime); + if (start) return start; + const end = otelTimestamp(record.endTime); + if (end) { + const duration = otelDurationMs(record.duration); + return duration ? new Date(end.getTime() - duration) : end; + } + return otelTimestamp(record.hrTime) + ?? otelTimestamp(record._hrTime) + ?? otelTimestamp(record.time) + ?? otelTimestamp(record.timestamp) + ?? otelTimestamp(record.observedTimestamp) + ?? otelTimestamp(record.timeUnixNano); +} + +function otelScalarTimestamp(value: string | number): Date | null { + if (typeof value === 'string' && !/^\d+(?:\.\d+)?$/.test(value)) return parseTs(value); + const raw = Number(value); + if (!Number.isFinite(raw) || raw <= 0) return null; + const abs = Math.abs(raw); + const millis = abs >= 1e17 ? raw / 1e6 : abs >= 1e14 ? raw / 1e3 : raw; + return parseTs(millis); +} + +function otelDurationMs(value: unknown): number | null { + if (Array.isArray(value) && value.length > 0) { + const seconds = Number(value[0]); + const nanos = Number(value[1] ?? 0); + if (Number.isFinite(seconds) && Number.isFinite(nanos)) return seconds * 1_000 + nanos / 1_000_000; + } + const raw = typeof value === 'number' ? value : typeof value === 'string' ? Number(value) : NaN; + if (!Number.isFinite(raw) || raw <= 0) return null; + return raw >= 1e6 ? raw / 1e6 : raw; +} + +function attrNumber(attributes: Record, keys: string[]): number { + for (const key of keys) { + const raw = attributes[key]; + const value = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN; + if (Number.isFinite(value) && value > 0) return value; + } + return 0; +} + +function stringAttr(value: Record, keys: string[]): string | undefined { + for (const key of keys) { + const raw = value[key]; + if (typeof raw === 'string' && raw.trim()) return raw.trim(); + } + return undefined; +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function clamp(value: number | undefined): number { + return Math.max(Number.isFinite(value) ? value! : 0, 0); +} diff --git a/packages/cli/src/scanners/droid.ts b/packages/cli/src/scanners/droid.ts index 5706682..f234f21 100644 --- a/packages/cli/src/scanners/droid.ts +++ b/packages/cli/src/scanners/droid.ts @@ -6,6 +6,8 @@ import { parseTs, dateKey, walkFiles, + fileModifiedTs, + inferProviderFromModel, initDateMap, accumulate, finalize, @@ -16,26 +18,23 @@ import { * Droid scanner. * * 数据目录: ~/.factory/sessions/ - * 文件格式: *.jsonl + 同目录 {sessionId}.settings.json - * - * JSONL 仅取首行时间戳确定日期； - * Token 用量从 settings.json → tokenUsage 提取。 + * 文件格式: {sessionId}.settings.json（JSONL transcript 仅用于缺失模型时兜底） + * 直接扫描 settings，避免 transcript 缺失或损坏时丢掉已落盘的 token 汇总。 */ interface DroidSettings { model?: string; + providerLock?: string; + providerLockTimestamp?: string | number; tokenUsage?: { inputTokens?: number; outputTokens?: number; + cacheCreationTokens?: number; cacheReadTokens?: number; thinkingTokens?: number; }; } -interface DroidLine { - timestamp?: string | number; -} - function extractProjectFromSlug(dirPath: string): string { const slug = basename(dirPath); const parts = slug.split('-').filter(Boolean); @@ -50,41 +49,12 @@ export async function scanDroidDates( const dates = new Set(targetDates); const dir = baseDir ?? join(homedir(), '.factory', 'sessions'); - const files = await walkFiles(dir, '.jsonl'); + const files = await walkFiles(dir, '.settings.json'); if (files.length === 0) return emptyResult(dates); const grouped = initDateMap(dates); - for (const jsonlPath of files) { - // 读 JSONL 首行取日期 - let content: string; - try { - content = await readFile(jsonlPath, 'utf-8'); - } catch { - continue; - } - - const firstLine = content.split('\n').find((l) => l.trim()); - if (!firstLine) continue; - - let firstObj: DroidLine; - try { - firstObj = JSON.parse(firstLine); - } catch { - continue; - } - - const ts = parseTs(firstObj.timestamp); - if (!ts) continue; - const dk = dateKey(ts); - const dayMap = grouped.get(dk); - if (!dayMap) continue; - - // 构造同目录 settings.json 路径 - const sessionDir = dirname(jsonlPath); - const jsonlName = basename(jsonlPath, '.jsonl'); - const settingsPath = join(sessionDir, `${jsonlName}.settings.json`); - + for (const settingsPath of files) { let settingsRaw: string; try { settingsRaw = await readFile(settingsPath, 'utf-8'); @@ -102,15 +72,25 @@ export async function scanDroidDates( const usage = settings.tokenUsage; if (!usage) continue; - const rawInput = usage.inputTokens ?? 0; - const cacheRead = usage.cacheReadTokens ?? 0; - const rawOutput = usage.outputTokens ?? 0; - const thinking = usage.thinkingTokens ?? 0; - - const input = Math.max(rawInput - cacheRead, 0); - const output = Math.max(rawOutput - thinking, 0); + const input = Math.max(usage.inputTokens ?? 0, 0); + const cacheRead = Math.max(usage.cacheReadTokens ?? 0, 0); + const cacheWrite = Math.max(usage.cacheCreationTokens ?? 0, 0); + const output = Math.max(usage.outputTokens ?? 0, 0); + const thinking = Math.max(usage.thinkingTokens ?? 0, 0); + if (input + cacheRead + cacheWrite + output + thinking === 0) continue; + + const sessionDir = dirname(settingsPath); + const transcriptPath = settingsPath.replace(/\.settings\.json$/, '.jsonl'); + const transcriptModel = settings.model ? undefined : await extractModelFromTranscript(transcriptPath); + const model = normalizeDroidModel(settings.model ?? transcriptModel ?? `${settings.providerLock ?? 'droid'}-unknown`); + const provider = settings.providerLock?.trim().toLowerCase() + || inferProviderFromModel(model, 'droid'); + const ts = parseTs(settings.providerLockTimestamp) ?? await fileModifiedTs(settingsPath); + if (!ts) continue; + const dk = dateKey(ts); + const dayMap = grouped.get(dk); + if (!dayMap) continue; - const model = settings.model ?? 'unknown'; const rawProject = extractProjectFromSlug(sessionDir); const alias = projectAliases?.[rawProject]; const project = alias ?? rawProject; @@ -119,7 +99,7 @@ export async function scanDroidDates( dayMap, `${model}|${project}`, { - provider: 'droid', + provider, product: 'droid', channel: 'cli', model, @@ -132,9 +112,31 @@ export async function scanDroidDates( outputTokens: 0, reasoningOutputTokens: 0, }, - { input, cached: cacheRead, cacheWrite: 0, output, reasoning: thinking }, + { input, cached: cacheRead, cacheWrite, output, reasoning: thinking }, ); } return finalize(grouped); } + +export function normalizeDroidModel(model: string): string { + return model + .replace(/^custom:/i, '') + .replace(/\[[^\]]*\]/g, '') + .replace(/\./g, '-') + .replace(/[\s_]+/g, '-') + .replace(/-+/g, '-') + .replace(/-+$/, '') + .toLowerCase(); +} + +async function extractModelFromTranscript(filePath: string): Promise { + let content: string; + try { + content = await readFile(filePath, 'utf-8'); + } catch { + return undefined; + } + const match = content.match(/Model:\s*([^\\\["\n]+)/i); + return match?.[1]?.trim() || undefined; +} diff --git a/packages/cli/src/scanners/gemini.ts b/packages/cli/src/scanners/gemini.ts index 088c89d..3e39fac 100644 --- a/packages/cli/src/scanners/gemini.ts +++ b/packages/cli/src/scanners/gemini.ts @@ -1,5 +1,5 @@ -import { readFile, readdir } from 'node:fs/promises'; -import { basename, join } from 'node:path'; +import { readFile } from 'node:fs/promises'; +import { basename, extname, join, relative, sep } from 'node:path'; import { homedir } from 'node:os'; import type { IngestBreakdown } from '@aiusage/shared'; import { @@ -10,57 +10,36 @@ import { finalize, emptyResult, walkFiles, + fileModifiedTs, normalizeModelName, + resolveProjectFields, } from './utils.js'; /** * Gemini CLI scanner. * - * 日志目录: ~/.gemini/tmp/{projectId}/chats/session-*.json - * 每个 JSON 文件包含 data.messages[] 或 data.history[] - * 两种 token 格式: msg.tokens / msg.usage(usageMetadata) + * 支持 legacy/modern JSON、当前 session-*.jsonl、headless stream JSONL， + * 以及 a2a 追加式 `$set.messages` 记录。损坏行会被跳过，缺失时间戳用 mtime 兜底。 */ -interface GeminiTokens { - input?: number; - output?: number; - cached?: number; - thoughts?: number; -} - -interface GeminiUsageMetadata { - promptTokenCount?: number; - candidatesTokenCount?: number; - cachedContentTokenCount?: number; - thoughtsTokenCount?: number; +interface GeminiLogEntry { + type?: string; + timestamp?: string | number; } -interface GeminiMessage { - model?: string; - timestamp?: string | number; - createTime?: string | number; - tokens?: GeminiTokens; - usage?: GeminiUsageMetadata; - usageMetadata?: GeminiUsageMetadata; -} - -interface GeminiSession { - model?: string; - createTime?: string | number; - startTime?: string | number; - messages?: GeminiMessage[]; - history?: GeminiMessage[]; - data?: { - model?: string; - createTime?: string | number; - messages?: GeminiMessage[]; - history?: GeminiMessage[]; - }; +interface TokenValues { + input: number; + cached: number; + cacheWrite: number; + output: number; + reasoning: number; } -interface GeminiLogEntry { - type?: string; - timestamp?: string | number; +interface GeminiUsageEvent { + model: string; + timestamp: Date; + tokens: TokenValues; + dedupKey?: string; } export async function scanGeminiDates( @@ -70,151 +49,346 @@ export async function scanGeminiDates( ): Promise> { const dates = new Set(targetDates); const dir = baseDir ?? join(homedir(), '.gemini', 'tmp'); - - const files = await walkFiles(dir, '.json'); + const [jsonFiles, jsonlFiles] = await Promise.all([ + walkFiles(dir, '.json'), + walkFiles(dir, '.jsonl'), + ]); + const files = [...new Set([...jsonFiles, ...jsonlFiles])]; if (files.length === 0) return emptyResult(dates); const grouped = initDateMap(dates); const tokenBackedDates = new Set(); + const seen = new Set(); for (const filePath of files) { if (basename(filePath) === 'logs.json') continue; - let content: string; try { content = await readFile(filePath, 'utf-8'); } catch { continue; } + const fallbackTs = await fileModifiedTs(filePath); + if (!fallbackTs) continue; + const events = extname(filePath) === '.jsonl' + ? parseGeminiJsonl(content, basename(filePath, '.jsonl'), fallbackTs) + : parseGeminiJson(content, basename(filePath, '.json'), fallbackTs); + const fields = projectFieldsFromFile(filePath, dir, projectAliases); - let session: GeminiSession; - try { - session = JSON.parse(content); - } catch { - continue; - } - - const messages = session.messages ?? session.history ?? session.data?.messages ?? session.data?.history ?? []; - const fallbackModel = session.model ?? session.data?.model ?? 'unknown'; - const fallbackTs = session.createTime ?? session.startTime ?? session.data?.createTime; - - for (const msg of messages) { - const tokens = extractTokens(msg); - if (!tokens) continue; - - const model = normalizeModelName(msg.model ?? fallbackModel); - const ts = parseTs(msg.timestamp) ?? parseTs(msg.createTime) ?? parseTs(fallbackTs); - if (!ts) continue; - - const dk = dateKey(ts); + for (const event of events) { + if (event.dedupKey) { + if (seen.has(event.dedupKey)) continue; + seen.add(event.dedupKey); + } + const dk = dateKey(event.timestamp); const dayMap = grouped.get(dk); if (!dayMap) continue; tokenBackedDates.add(dk); - + const model = normalizeModelName(event.model || 'unknown'); accumulate( dayMap, - `${model}|unknown`, + `${model}|${fields.project}`, { provider: 'google', product: 'gemini-cli', channel: 'cli', model, - project: 'unknown', - projectDisplay: 'unknown', + project: fields.project, + projectDisplay: fields.projectDisplay, + projectAlias: fields.projectAlias, inputTokens: 0, cachedInputTokens: 0, cacheWriteTokens: 0, outputTokens: 0, reasoningOutputTokens: 0, }, - tokens, + event.tokens, ); } } - const logFiles = files.filter((filePath) => basename(filePath) === 'logs.json'); - for (const filePath of logFiles) { + for (const filePath of jsonFiles.filter(file => basename(file) === 'logs.json')) { await collectGeminiLogEvents(filePath, grouped, tokenBackedDates); } - return finalize(grouped); } +function parseGeminiJson(content: string, fallbackId: string, fallbackTs: Date): GeminiUsageEvent[] { + let value: unknown; + try { + value = JSON.parse(content); + } catch { + return []; + } + if (!isObject(value)) return []; + return parseGeminiValue(value, fallbackId, fallbackTs); +} + +function parseGeminiJsonl(content: string, fallbackId: string, fallbackTs: Date): GeminiUsageEvent[] { + let sessionId = fallbackId; + let currentModel: string | undefined; + const keyed = new Map(); + const anonymous: GeminiUsageEvent[] = []; + + const add = (event: GeminiUsageEvent): void => { + if (event.dedupKey) keyed.set(event.dedupKey, event); + else anonymous.push(event); + }; + + for (const line of content.split('\n')) { + if (!line.trim()) continue; + let value: Record; + try { + const parsed = JSON.parse(line) as unknown; + if (!isObject(parsed)) continue; + value = parsed; + } catch { + continue; + } + + sessionId = stringValue(value.session_id) ?? stringValue(value.sessionId) ?? sessionId; + currentModel = stringValue(value.model) ?? currentModel; + if (value.type === 'init') continue; + + const set = isObject(value.$set) ? value.$set : undefined; + if (set) { + currentModel = stringValue(set.model) ?? currentModel; + for (const message of arrayValue(set.messages)) { + const event = parseGeminiMessage(message, currentModel, sessionId, fallbackTs, true); + if (event) add(event); + } + } + + for (const message of arrayValue(value.messages)) { + const event = parseGeminiMessage(message, currentModel, sessionId, fallbackTs, true); + if (event) add(event); + } + + const direct = parseDirectGeminiEvent(value, currentModel, sessionId, fallbackTs); + if (direct) { + currentModel = direct.model; + add(direct); + continue; + } + + const stats = isObject(value.stats) + ? value.stats + : isObject(value.result) && isObject(value.result.stats) ? value.result.stats : undefined; + if (stats) { + for (const event of parseGeminiStats(stats, currentModel, sessionId, timestampFrom(value) ?? fallbackTs)) add(event); + } + } + return [...keyed.values(), ...anonymous]; +} + +function parseGeminiValue( + value: Record, + fallbackId: string, + fallbackTs: Date, +): GeminiUsageEvent[] { + const data = isObject(value.data) ? value.data : undefined; + const sessionId = stringValue(value.sessionId) + ?? stringValue(value.session_id) + ?? fallbackId; + const fallbackModel = stringValue(value.model) ?? stringValue(data?.model); + const sessionTs = timestampFrom(value) + ?? timestampFrom(data) + ?? parseTs(value.startTime as string | number | undefined) + ?? fallbackTs; + const messages = [ + ...arrayValue(value.messages), + ...arrayValue(value.history), + ...arrayValue(data?.messages), + ...arrayValue(data?.history), + ]; + const keyed = new Map(); + const anonymous: GeminiUsageEvent[] = []; + for (const message of messages) { + const event = parseGeminiMessage(message, fallbackModel, sessionId, sessionTs, false); + if (!event) continue; + if (event.dedupKey) keyed.set(event.dedupKey, event); + else anonymous.push(event); + } + if (keyed.size + anonymous.length > 0) return [...keyed.values(), ...anonymous]; + + const direct = parseDirectGeminiEvent(value, fallbackModel, sessionId, sessionTs); + if (direct) return [direct]; + const stats = isObject(value.stats) + ? value.stats + : isObject(value.result) && isObject(value.result.stats) ? value.result.stats : undefined; + return stats ? parseGeminiStats(stats, fallbackModel, sessionId, sessionTs) : []; +} + +function parseGeminiMessage( + message: Record, + fallbackModel: string | undefined, + sessionId: string, + fallbackTs: Date, + usageMetadataIsInclusive: boolean, +): GeminiUsageEvent | undefined { + const nested = isObject(message.message) ? message.message : undefined; + const source = nested ?? message; + const usageMetadata = isObject(source.usageMetadata) + ? source.usageMetadata + : isObject(source.usage) ? source.usage : undefined; + const tokens = isObject(source.tokens) ? source.tokens : undefined; + const tokenValues = usageMetadata + ? extractTokenValues(usageMetadata, usageMetadataIsInclusive || hasUsageMetadataKeys(usageMetadata)) + : tokens ? extractTokenValues(tokens, false) : undefined; + if (!tokenValues || tokenTotal(tokenValues) === 0) return undefined; + + const model = stringValue(source.model) ?? stringValue(message.model) ?? fallbackModel ?? 'unknown'; + const timestamp = timestampFrom(message) ?? timestampFrom(source) ?? fallbackTs; + const id = stringValue(message.id) ?? stringValue(source.id) ?? stringValue(message.messageId); + return { + model, + timestamp, + tokens: tokenValues, + dedupKey: id ? `gemini:${sessionId}:${id}` : undefined, + }; +} + +function parseDirectGeminiEvent( + value: Record, + modelHint: string | undefined, + sessionId: string, + fallbackTs: Date, +): GeminiUsageEvent | undefined { + const tokens = isObject(value.tokens) ? value.tokens : undefined; + if (!tokens || (value.type !== 'gemini' && !value.model && !modelHint)) return undefined; + const tokenValues = extractTokenValues( + tokens, + hasAny(tokens, ['prompt', 'prompt_tokens', 'input_tokens', 'promptTokenCount']), + ); + if (tokenTotal(tokenValues) === 0) return undefined; + const id = stringValue(value.id); + return { + model: stringValue(value.model) ?? modelHint ?? 'unknown', + timestamp: timestampFrom(value) ?? fallbackTs, + tokens: tokenValues, + dedupKey: id ? `gemini:${sessionId}:${id}` : undefined, + }; +} + +function parseGeminiStats( + stats: Record, + modelHint: string | undefined, + sessionId: string, + timestamp: Date, +): GeminiUsageEvent[] { + const models = isObject(stats.models) ? stats.models : undefined; + const entries = models ? Object.entries(models) : [[modelHint ?? 'unknown', stats] as const]; + return entries.flatMap(([model, raw], index) => { + if (!isObject(raw)) return []; + const wrapped = isObject(raw.tokens); + const source = wrapped ? raw.tokens as Record : raw; + const inclusive = hasAny(source, ['prompt', 'prompt_tokens', 'input_tokens']) || wrapped; + const tokens = extractTokenValues(source, inclusive); + if (tokenTotal(tokens) === 0) return []; + return [{ model, timestamp, tokens, dedupKey: `gemini:${sessionId}:stats:${model}:${index}:${timestamp.getTime()}` }]; + }); +} + +function extractTokenValues(value: Record, inputIncludesCache: boolean): TokenValues { + const rawInput = firstNumber(value, ['input', 'prompt', 'input_tokens', 'prompt_tokens', 'promptTokenCount']); + const cached = firstNumber(value, ['cached', 'cached_tokens', 'cachedContentTokenCount']); + const output = firstNumber(value, ['output', 'candidates', 'output_tokens', 'completion_tokens', 'candidates_tokens', 'candidatesTokenCount']); + const reasoning = firstNumber(value, ['thoughts', 'reasoning', 'thoughts_tokens', 'reasoning_tokens', 'thoughtsTokenCount']); + const tool = firstNumber(value, ['tool', 'tool_tokens']); + const total = optionalNumber(value, ['total', 'total_tokens', 'totalTokenCount']); + + let input = rawInput; + if (inputIncludesCache) { + input = Math.max(rawInput - Math.min(rawInput, cached), 0); + } else if (total != null && cached > 0) { + const inclusiveTotal = rawInput + output + reasoning + tool; + if (total === inclusiveTotal) input = Math.max(rawInput - Math.min(rawInput, cached), 0); + } + return { input: input + tool, cached, cacheWrite: 0, output, reasoning }; +} + async function collectGeminiLogEvents( filePath: string, grouped: Map>, tokenBackedDates: Set, ): Promise { - let content: string; - try { - content = await readFile(filePath, 'utf-8'); - } catch { - return; - } - let rows: GeminiLogEntry[]; try { - rows = JSON.parse(content) as GeminiLogEntry[]; + rows = JSON.parse(await readFile(filePath, 'utf-8')) as GeminiLogEntry[]; } catch { return; } - if (!Array.isArray(rows)) return; - for (const row of rows) { if (row.type && row.type !== 'user') continue; - const ts = parseTs(row.timestamp); if (!ts) continue; - const dk = dateKey(ts); if (tokenBackedDates.has(dk)) continue; - const dayMap = grouped.get(dk); if (!dayMap) continue; - accumulate( dayMap, 'unknown|unknown', { - provider: 'google', - product: 'gemini-cli', - channel: 'cli', - model: 'unknown', - project: 'unknown', - inputTokens: 0, - cachedInputTokens: 0, - cacheWriteTokens: 0, - outputTokens: 0, - reasoningOutputTokens: 0, + provider: 'google', product: 'gemini-cli', channel: 'cli', model: 'unknown', + project: 'unknown', projectDisplay: 'unknown', inputTokens: 0, cachedInputTokens: 0, + cacheWriteTokens: 0, outputTokens: 0, reasoningOutputTokens: 0, }, { input: 0, cached: 0, cacheWrite: 0, output: 0, reasoning: 0 }, ); } } -function extractTokens( - msg: GeminiMessage, -): { input: number; cached: number; cacheWrite: number; output: number; reasoning: number } | null { - // 格式 1: msg.tokens - if (msg.tokens) { - const t = msg.tokens; - const cached = t.cached ?? 0; - const input = Math.max((t.input ?? 0) - cached, 0); - const thoughts = t.thoughts ?? 0; - const output = Math.max((t.output ?? 0) - thoughts, 0); - return { input, cached, cacheWrite: 0, output, reasoning: thoughts }; - } +function projectFieldsFromFile( + filePath: string, + baseDir: string, + aliases?: Record, +) { + const first = relative(baseDir, filePath).split(sep).filter(Boolean)[0]; + return first && first !== basename(filePath) + ? resolveProjectFields(first, aliases) + : { project: 'unknown', projectDisplay: 'unknown' }; +} + +function timestampFrom(value: Record | undefined): Date | null { + if (!value) return null; + return parseTs((value.timestamp ?? value.createTime ?? value.created_at ?? value.lastUpdated) as string | number | undefined); +} + +function firstNumber(value: Record, keys: string[]): number { + return Math.max(optionalNumber(value, keys) ?? 0, 0); +} - // 格式 2: msg.usage 或 msg.usageMetadata - const u = msg.usage ?? msg.usageMetadata; - if (u) { - const cached = u.cachedContentTokenCount ?? 0; - const input = Math.max((u.promptTokenCount ?? 0) - cached, 0); - const thoughts = u.thoughtsTokenCount ?? 0; - const output = Math.max((u.candidatesTokenCount ?? 0) - thoughts, 0); - return { input, cached, cacheWrite: 0, output, reasoning: thoughts }; +function optionalNumber(value: Record, keys: string[]): number | undefined { + for (const key of keys) { + const raw = value[key]; + const parsed = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN; + if (Number.isFinite(parsed)) return parsed; } + return undefined; +} + +function hasUsageMetadataKeys(value: Record): boolean { + return hasAny(value, ['promptTokenCount', 'candidatesTokenCount', 'cachedContentTokenCount']); +} + +function hasAny(value: Record, keys: string[]): boolean { + return keys.some(key => value[key] != null); +} + +function tokenTotal(tokens: TokenValues): number { + return tokens.input + tokens.cached + tokens.cacheWrite + tokens.output + tokens.reasoning; +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function arrayValue(value: unknown): Record[] { + return Array.isArray(value) ? value.filter(isObject) : []; +} - return null; +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; } diff --git a/packages/cli/src/scanners/opencode.ts b/packages/cli/src/scanners/opencode.ts index fe9de1f..9c1793d 100644 --- a/packages/cli/src/scanners/opencode.ts +++ b/packages/cli/src/scanners/opencode.ts @@ -1,5 +1,6 @@ +import { execFile } from 'node:child_process'; import { readFile } from 'node:fs/promises'; -import { basename } from 'node:path'; +import { basename, join } from 'node:path'; import { homedir } from 'node:os'; import type { IngestBreakdown } from '@aiusage/shared'; import { @@ -11,32 +12,47 @@ import { accumulate, finalize, emptyResult, + inferProviderFromModel, } from './utils.js'; /** - * OpenCode scanner (JSON mode only, skip SQLite). + * OpenCode scanner. * - * 数据目录: ~/.local/share/opencode/storage/message/ses_* /*.json - * 每个 JSON 文件一条消息记录。 + * OpenCode 1.2+ 读取 ~/.local/share/opencode/opencode.db；同时保留旧版 + * storage/message/*.json。SQLite 通过系统 sqlite3 只读访问，不存在时安全回退旧 JSON。 */ interface OpenCodeMessage { + id?: string; + sessionID?: string; role?: string; modelID?: string; - time?: { - created?: string | number; - }; + providerID?: string; + time?: { created?: string | number; completed?: string | number }; tokens?: { input?: number; output?: number; - cache?: { - read?: number; - }; + cache?: { read?: number; write?: number }; reasoning?: number; }; - path?: { - root?: string; - }; + path?: { root?: string } | string; +} + +interface OpenCodeSqliteRow { + id?: string; + session_id?: string; + data?: string; + workspace_root?: string | null; +} + +interface ParsedOpenCodeRecord { + id?: string; + fingerprint: string; + timestamp: Date; + model: string; + provider: string; + projectRoot?: string; + tokens: { input: number; cached: number; cacheWrite: number; output: number; reasoning: number }; } export async function scanOpencodeDates( @@ -45,70 +61,188 @@ export async function scanOpencodeDates( projectAliases?: Record, ): Promise> { const dates = new Set(targetDates); - const dir = baseDir ?? `${homedir()}/.local/share/opencode/storage/message`; - - const files = await walkFiles(dir, '.json'); - if (files.length === 0) return emptyResult(dates); - + const dataDir = baseDir ?? join(homedir(), '.local', 'share', 'opencode'); + const legacyDir = baseDir ?? join(dataDir, 'storage', 'message'); + const dbPath = join(dataDir, 'opencode.db'); const grouped = initDateMap(dates); + const seenIds = new Set(); + const seenFingerprints = new Set(); + const sqliteRows = await queryOpenCodeSqlite(dbPath); + const sqliteRecords = parseOpenCodeSqliteRows(sqliteRows); + for (const record of sqliteRecords) { + addOpenCodeRecord(record, grouped, seenIds, seenFingerprints, projectAliases); + } + + const files = await walkFiles(legacyDir, '.json'); for (const filePath of files) { - let raw: string; + let message: OpenCodeMessage; try { - raw = await readFile(filePath, 'utf-8'); + message = JSON.parse(await readFile(filePath, 'utf-8')) as OpenCodeMessage; } catch { continue; } + const record = parseOpenCodeMessage(message, basename(filePath, '.json')); + if (record) addOpenCodeRecord(record, grouped, seenIds, seenFingerprints, projectAliases); + } - let msg: OpenCodeMessage; + if (sqliteRows.length === 0 && files.length === 0) return emptyResult(dates); + return finalize(grouped); +} + +/** 供 `report --range all` 在扫描前发现 SQLite 与旧 JSON 中的实际日期。 */ +export async function discoverOpenCodeUsageDates(baseDir?: string): Promise> { + const dataDir = baseDir ?? join(homedir(), '.local', 'share', 'opencode'); + const legacyDir = baseDir ?? join(dataDir, 'storage', 'message'); + const result = new Set(); + for (const record of parseOpenCodeSqliteRows(await queryOpenCodeSqlite(join(dataDir, 'opencode.db')))) { + result.add(dateKey(record.timestamp)); + } + for (const filePath of await walkFiles(legacyDir, '.json')) { try { - msg = JSON.parse(raw); + const record = parseOpenCodeMessage( + JSON.parse(await readFile(filePath, 'utf-8')) as OpenCodeMessage, + basename(filePath, '.json'), + ); + if (record) result.add(dateKey(record.timestamp)); } catch { - continue; + // 单个损坏文件不影响其他会话日期发现。 } + } + return result; +} - const tokens = msg.tokens; - if (!tokens) continue; - - const input = tokens.input ?? 0; - const output = tokens.output ?? 0; - if (input === 0 && output === 0) continue; - - const ts = parseTs(msg.time?.created); - if (!ts) continue; - const dk = dateKey(ts); - const dayMap = grouped.get(dk); - if (!dayMap) continue; - - const model = msg.modelID ?? 'unknown'; - const cached = tokens.cache?.read ?? 0; - const reasoning = tokens.reasoning ?? 0; - - const rootPath = msg.path?.root; - const fields = rootPath - ? resolveProjectFields(rootPath, projectAliases) - : { project: 'unknown', projectDisplay: 'unknown' }; - - accumulate( - dayMap, - `${model}|${fields.project}`, - { - provider: 'opencode', - product: 'opencode', - channel: 'cli', - model, - project: fields.project, - projectDisplay: fields.projectDisplay, - projectAlias: fields.projectAlias, - inputTokens: 0, - cachedInputTokens: 0, - cacheWriteTokens: 0, - outputTokens: 0, - reasoningOutputTokens: 0, - }, - { input, cached, cacheWrite: 0, output, reasoning }, - ); +export function parseOpenCodeSqliteRows(rows: OpenCodeSqliteRow[]): ParsedOpenCodeRecord[] { + return rows.flatMap(row => { + if (!row.data) return []; + let message: OpenCodeMessage; + try { + message = JSON.parse(row.data) as OpenCodeMessage; + } catch { + return []; + } + message.sessionID ??= row.session_id; + const record = parseOpenCodeMessage(message, row.id); + if (!record) return []; + record.projectRoot = row.workspace_root || record.projectRoot; + return [record]; + }); +} + +async function queryOpenCodeSqlite(dbPath: string): Promise { + const modern = ` + SELECT m.id, m.session_id, m.data, NULLIF(s.directory, '') AS workspace_root + FROM message m + LEFT JOIN session s ON s.id = m.session_id + WHERE json_extract(m.data, '$.role') = 'assistant' + AND json_extract(m.data, '$.tokens') IS NOT NULL + ORDER BY m.id, m.session_id`; + const legacy = ` + SELECT m.id, m.session_id, m.data, NULL AS workspace_root + FROM message m + WHERE json_extract(m.data, '$.role') = 'assistant' + AND json_extract(m.data, '$.tokens') IS NOT NULL + ORDER BY m.id, m.session_id`; + + for (const query of [modern, legacy]) { + try { + const stdout = await runSqlite(['-readonly', '-json', dbPath, query]); + const rows = JSON.parse(stdout || '[]') as unknown; + if (Array.isArray(rows)) return rows.filter(isSqliteRow); + } catch { + // 数据库、session 表或 sqlite3 不存在时尝试下一种格式，最终回退旧 JSON。 + } } + return []; +} - return finalize(grouped); +function runSqlite(args: string[]): Promise { + return new Promise((resolve, reject) => { + execFile( + 'sqlite3', + args, + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + (error, stdout) => error ? reject(error) : resolve(stdout), + ); + }); +} + +function parseOpenCodeMessage( + message: OpenCodeMessage, + fallbackId?: string, +): ParsedOpenCodeRecord | undefined { + if (message.role !== 'assistant' || !message.tokens) return undefined; + const timestamp = parseTs(message.time?.created); + if (!timestamp) return undefined; + const model = message.modelID ?? 'unknown'; + const provider = message.providerID?.trim().toLowerCase() + || inferProviderFromModel(model, 'opencode'); + const tokens = { + input: clamp(message.tokens.input), + cached: clamp(message.tokens.cache?.read), + cacheWrite: clamp(message.tokens.cache?.write), + output: clamp(message.tokens.output), + reasoning: clamp(message.tokens.reasoning), + }; + if (Object.values(tokens).every(value => value === 0)) return undefined; + const projectRoot = typeof message.path === 'string' ? message.path : message.path?.root; + const id = message.id ?? fallbackId; + return { + id, + timestamp, + model, + provider, + projectRoot, + tokens, + fingerprint: [ + timestamp.getTime(), model, provider, tokens.input, tokens.cached, + tokens.cacheWrite, tokens.output, tokens.reasoning, + ].join('|'), + }; +} + +function addOpenCodeRecord( + record: ParsedOpenCodeRecord, + grouped: ReturnType, + seenIds: Set, + seenFingerprints: Set, + aliases?: Record, +): void { + if (record.id && seenIds.has(record.id)) return; + if (seenFingerprints.has(record.fingerprint)) return; + if (record.id) seenIds.add(record.id); + seenFingerprints.add(record.fingerprint); + + const dayMap = grouped.get(dateKey(record.timestamp)); + if (!dayMap) return; + const fields = record.projectRoot + ? resolveProjectFields(record.projectRoot, aliases) + : { project: 'unknown', projectDisplay: 'unknown' }; + accumulate( + dayMap, + `${record.provider}|${record.model}|${fields.project}`, + { + provider: record.provider, + product: 'opencode', + channel: 'cli', + model: record.model, + project: fields.project, + projectDisplay: fields.projectDisplay, + projectAlias: fields.projectAlias, + inputTokens: 0, + cachedInputTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningOutputTokens: 0, + }, + record.tokens, + ); +} + +function isSqliteRow(value: unknown): value is OpenCodeSqliteRow { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function clamp(value: number | undefined): number { + return Math.max(Number.isFinite(value) ? value! : 0, 0); } diff --git a/packages/cli/src/scanners/pi.ts b/packages/cli/src/scanners/pi.ts index 5538ece..1c7ae56 100644 --- a/packages/cli/src/scanners/pi.ts +++ b/packages/cli/src/scanners/pi.ts @@ -8,6 +8,8 @@ import { projectFromPath, resolveProjectFields, walkFiles, + fileModifiedTs, + inferProviderFromModel, initDateMap, accumulate, finalize, @@ -19,6 +21,7 @@ import { * Pi Coding Agent scanner. * * 日志目录: ~/.pi/agent/sessions/{encoded-cwd}/{timestamp}_{sessionId}.jsonl + * 同时兼容 Oh My Pi: ~/.omp/agent/sessions/ * 支持环境变量 PI_CODING_AGENT_DIR 覆盖基础目录 * * JSONL 行格式: @@ -34,13 +37,16 @@ interface PiLine { type?: string; timestamp?: string | number; cwd?: string; + name?: string; message?: { role?: string; model?: string; + provider?: string; usage?: { input?: number; output?: number; cacheRead?: number; + cacheWrite?: number; }; }; } @@ -51,9 +57,8 @@ export async function scanPiDates( projectAliases?: Record, ): Promise> { const dates = new Set(targetDates); - const sessionsDir = baseDir ?? getSessionsDir(); - - const files = await walkFiles(sessionsDir, '.jsonl'); + const sessionDirs = baseDir ? [baseDir] : getSessionDirs(); + const files = [...new Set((await Promise.all(sessionDirs.map(dir => walkFiles(dir, '.jsonl')))).flat())]; if (files.length === 0) return emptyResult(dates); const grouped = initDateMap(dates); @@ -71,6 +76,9 @@ export async function scanPiDates( const parentDir = dirname(filePath); const encodedCwd = basename(parentDir); let sessionProjectFields: ProjectFields = { project: extractProjectFromEncoded(encodedCwd), projectDisplay: extractProjectFromEncoded(encodedCwd) }; + let sessionId = basename(filePath, '.jsonl'); + let messageIndex = 0; + const fallbackTs = await fileModifiedTs(filePath); for (const line of content.split('\n')) { if (!line.trim()) continue; @@ -83,9 +91,14 @@ export async function scanPiDates( // session header 提供 cwd if (obj.type === 'session' && obj.cwd) { + sessionId = obj.id ?? sessionId; sessionProjectFields = resolveProjectFields(obj.cwd, projectAliases); continue; } + if (obj.type === 'session') { + sessionId = obj.id ?? sessionId; + continue; + } if (obj.type !== 'message') continue; @@ -97,28 +110,30 @@ export async function scanPiDates( if (!usage) continue; if (usage.input == null && usage.output == null) continue; - const ts = parseTs(obj.timestamp); + const ts = parseTs(obj.timestamp) ?? fallbackTs; if (!ts) continue; const dk = dateKey(ts); const dayMap = grouped.get(dk); if (!dayMap) continue; - // id 去重 - if (obj.id) { - if (seen.has(obj.id)) continue; - seen.add(obj.id); - } + const messageKey = `${sessionId}:${obj.id ?? messageIndex}`; + messageIndex += 1; + if (seen.has(messageKey)) continue; + seen.add(messageKey); const model = msg.model ?? 'unknown'; - const input = usage.input ?? 0; - const cached = usage.cacheRead ?? 0; - const output = usage.output ?? 0; + const provider = msg.provider?.trim() || inferProviderFromModel(model, 'inflection'); + const input = Math.max(usage.input ?? 0, 0); + const cached = Math.max(usage.cacheRead ?? 0, 0); + const cacheWrite = Math.max(usage.cacheWrite ?? 0, 0); + const output = Math.max(usage.output ?? 0, 0); + if (input + cached + cacheWrite + output === 0) continue; accumulate( dayMap, `${model}|${sessionProjectFields.project}`, { - provider: 'inflection', + provider, product: 'pi', channel: 'cli', model, @@ -131,7 +146,7 @@ export async function scanPiDates( outputTokens: 0, reasoningOutputTokens: 0, }, - { input, cached, cacheWrite: 0, output, reasoning: 0 }, + { input, cached, cacheWrite, output, reasoning: 0 }, ); } } @@ -139,10 +154,12 @@ export async function scanPiDates( return finalize(grouped); } -function getSessionsDir(): string { +function getSessionDirs(): string[] { const envDir = process.env.PI_CODING_AGENT_DIR; - if (envDir) return join(envDir, 'sessions'); - return join(homedir(), '.pi', 'agent', 'sessions'); + const piDir = envDir + ? join(envDir, 'sessions') + : join(homedir(), '.pi', 'agent', 'sessions'); + return [piDir, join(homedir(), '.omp', 'agent', 'sessions')]; } function extractProjectFromEncoded(encoded: string): string { diff --git a/packages/cli/src/scanners/qwen.ts b/packages/cli/src/scanners/qwen.ts index 6b6d935..d86d13d 100644 --- a/packages/cli/src/scanners/qwen.ts +++ b/packages/cli/src/scanners/qwen.ts @@ -7,6 +7,7 @@ import { dateKey, resolveProjectFields, walkFiles, + fileModifiedTs, initDateMap, accumulate, finalize, @@ -16,15 +17,16 @@ import { /** * Qwen Code scanner (Gemini CLI fork). * - * 日志目录: ~/.qwen/tmp/{projectId}/chats/*.jsonl - * 行格式: { type, timestamp, uuid, model, cwd, usageMetadata } - * 仅解析 type === 'assistant'，按 uuid 去重 + * 当前目录: ~/.qwen/projects/{projectId}/chats/*.jsonl + * 兼容旧目录: ~/.qwen/tmp/{projectId}/chats/*.jsonl + * 仅解析 assistant usage；优先按 uuid 去重，无 uuid 时使用会话内稳定序号。 */ interface QwenRecord { type?: string; timestamp?: string | number; uuid?: string; + sessionId?: string; model?: string; cwd?: string; usageMetadata?: { @@ -41,19 +43,23 @@ export async function scanQwenDates( projectAliases?: Record, ): Promise> { const dates = new Set(targetDates); - const dir = baseDir ?? join(homedir(), '.qwen', 'tmp'); - - const files = await walkFiles(dir, '.jsonl'); + const dirs = baseDir + ? [baseDir] + : [join(homedir(), '.qwen', 'projects'), join(homedir(), '.qwen', 'tmp')]; + const files = [...new Set((await Promise.all(dirs.map(dir => walkFiles(dir, '.jsonl')))).flat())]; if (files.length === 0) return emptyResult(dates); const grouped = initDateMap(dates); const seen = new Set(); for (const filePath of files) { - // 从路径提取 projectId: .qwen/tmp/{projectId}/chats/xxx.jsonl + // 从 projects/{projectId}/chats 或 tmp/{projectId}/chats 提取 projectId。 const chatsDir = dirname(filePath); const projectDir = dirname(chatsDir); const projectId = basename(projectDir); + const pathSessionId = basename(filePath, '.jsonl'); + const fallbackTs = await fileModifiedTs(filePath); + let assistantIndex = 0; let content: string; try { @@ -73,30 +79,32 @@ export async function scanQwenDates( if (obj.type !== 'assistant') continue; - const ts = parseTs(obj.timestamp); + const u = obj.usageMetadata; + if (!u) continue; + + const messageKey = obj.uuid ?? `${obj.sessionId ?? pathSessionId}:${assistantIndex}`; + assistantIndex += 1; + if (seen.has(messageKey)) continue; + seen.add(messageKey); + + const ts = parseTs(obj.timestamp) ?? fallbackTs; if (!ts) continue; const dk = dateKey(ts); const dayMap = grouped.get(dk); if (!dayMap) continue; - // uuid 去重 - if (obj.uuid) { - if (seen.has(obj.uuid)) continue; - seen.add(obj.uuid); - } - - const u = obj.usageMetadata; - if (!u) continue; - const model = obj.model ?? 'unknown'; const fields = obj.cwd ? resolveProjectFields(obj.cwd, projectAliases) : resolveProjectFields(projectId, projectAliases); - const cached = u.cachedContentTokenCount ?? 0; - const input = Math.max((u.promptTokenCount ?? 0) - cached, 0); - const thoughts = u.thoughtsTokenCount ?? 0; - const output = Math.max((u.candidatesTokenCount ?? 0) - thoughts, 0); + const cached = Math.max(u.cachedContentTokenCount ?? 0, 0); + const prompt = Math.max(u.promptTokenCount ?? 0, 0); + const input = Math.max(prompt - Math.min(cached, prompt), 0); + const thoughts = Math.max(u.thoughtsTokenCount ?? 0, 0); + // candidatesTokenCount 与 thoughtsTokenCount 是并列字段，不互相包含。 + const output = Math.max(u.candidatesTokenCount ?? 0, 0); + if (input + cached + output + thoughts === 0) continue; accumulate( dayMap, diff --git a/packages/cli/src/scanners/utils.ts b/packages/cli/src/scanners/utils.ts index c154061..4dd0bad 100644 --- a/packages/cli/src/scanners/utils.ts +++ b/packages/cli/src/scanners/utils.ts @@ -1,4 +1,4 @@ -import { readdir } from 'node:fs/promises'; +import { readdir, stat } from 'node:fs/promises'; import { join } from 'node:path'; import type { IngestBreakdown } from '@aiusage/shared'; @@ -24,8 +24,31 @@ export function dateKey(date: Date): string { return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`; } +/** 文件内没有可靠时间戳时，以 mtime 兜底，避免有真实 token 的记录被静默丢弃。 */ +export async function fileModifiedTs(filePath: string): Promise { + try { + return parseTs((await stat(filePath)).mtimeMs); + } catch { + return null; + } +} + +/** 根据模型名推断底层供应商；无法识别时保留调用方指定的产品供应商。 */ +export function inferProviderFromModel(model: string, fallback: string): string { + const value = model.trim().toLowerCase(); + if (/^(claude|opus|sonnet|haiku)(?:[-.]|$)/.test(value)) return 'anthropic'; + if (/^(gpt|chatgpt|codex|o[134])(?:[-.]|$)/.test(value)) return 'openai'; + if (/^gemini(?:[-.]|$)/.test(value)) return 'google'; + if (/^qwen(?:[-.]|$)/.test(value)) return 'alibaba'; + if (/^deepseek(?:[-.]|$)/.test(value)) return 'deepseek'; + if (/^(glm|codegeex)(?:[-.]|$)/.test(value)) return 'zhipu'; + if (/^(kimi|moonshot)(?:[-/.]|$)/.test(value)) return 'moonshot'; + if (/^grok(?:[-.]|$)/.test(value)) return 'xai'; + return fallback; +} + export function projectFromPath(raw: string, aliases?: Record): string { - const parts = raw.split('/').filter(Boolean); + const parts = raw.split(/[\\/]+/).filter(Boolean); const name = parts[parts.length - 1] || 'unknown'; return aliases?.[raw] ?? aliases?.[name] ?? name; } @@ -40,7 +63,7 @@ export function resolveProjectFields( rawPath: string, aliases?: Record, ): ProjectFields { - const parts = rawPath.split('/').filter(Boolean); + const parts = rawPath.split(/[\\/]+/).filter(Boolean); const display = parts[parts.length - 1] || 'unknown'; const alias = aliases?.[rawPath] ?? aliases?.[display]; return { @@ -59,6 +82,7 @@ export async function walkFiles(dir: string, ext: string): Promise { async function walk(dir: string, ext: string, out: string[]): Promise { let entries; try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; } + entries.sort((a, b) => a.name.localeCompare(b.name)); for (const e of entries) { const full = join(dir, e.name); if (e.isDirectory()) await walk(full, ext, out);