From 33972c5cdc98fa90bac226d9fbb9c173e258bfc8 Mon Sep 17 00:00:00 2001 From: stv1024 Date: Fri, 23 Oct 2026 13:03:45 +0800 Subject: [PATCH 1/2] feat: add a setting to prefer price estimates over reported costs Runtimes can report a list-price fallback for models they do not know, so local or unbilled turns show a cost that was never charged. When the new host setting is on and a custom or reference price matches, that estimate replaces the reported cost. Turns without a price keep the reported cost, and the ledger file still stores the raw value. Closes #1 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++++ OVERVIEW.md | 3 ++- README.md | 5 +++-- client/settings.tsx | 3 ++- server/preferences.test.ts | 9 ++++++++- server/pricing.ts | 4 +++- shared/preferences.ts | 1 + 7 files changed, 23 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5942a7c..873739e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- Add a **Prefer price estimates** setting (off by default). When a custom or reference price matches, it replaces the runtime-reported cost, which can be a list-price fallback for local or unbilled models. The reported value stays in the ledger file (#1). + ## v0.8.0 (2026-10-23) — Paseo 0.12 background workspaces and accounting accuracy Requires Paseo **0.11.0 or newer**; verified on 0.11.1 and 0.12.0. Existing ledger records are reinterpreted on read, not rewritten; existing custom price files are kept. Reload or reopen the app after updating. See the [release notes](docs/releases/v0.8.0.md). diff --git a/OVERVIEW.md b/OVERVIEW.md index 479cb3c..c6cfccf 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -6,11 +6,12 @@ Token counts come from the provider. Claude terminal aggregates and observed Cod The pill distinguishes known session snapshots (`Session ≈4.12M tok`), unknown-scope snapshots, waiting for usage and no reported usage. Old timeline cards without scope metadata are labelled as legacy; their ledger inclusion is not guessed from the card. -The plugin settings page on each host has four options: +The plugin settings page on each host has five options: - **Timeline summaries** turns the usage summary after each new turn on or off. Existing summaries stay. - **OpenRouter reference prices** lets the plugin download the public model price list from `openrouter.ai` in the background and cache it for 24 hours. The request carries no tokens, prompts, or usage data. Turn it off to stop the download and stop using those prices. - **Built-in fallback prices** controls estimates when no custom or OpenRouter price matches a model. +- **Prefer price estimates** uses a matching custom or reference price instead of the cost the runtime reports. Use it for local or unbilled models. It is off by default. - **Refresh frequency** selects normal or relaxed polling. Live updates still refresh the views immediately. The ledger is a JSON Lines file on the daemon host, `~/.paseo/plugins/token-ledger/` (or under `PASEO_HOME`), retaining 2,000 records. It stores observed usage, timestamps, identity, scope and quality, not prompts, responses, tool arguments or credentials. Older records remain readable and are reinterpreted without rewriting them. `pricing.json` starts empty and preserves your existing prices. Full model IDs and explicit aliases prevent unrelated routes from sharing a guessed price. History does not sync between hosts. diff --git a/README.md b/README.md index 00027a2..a54c05a 100644 --- a/README.md +++ b/README.md @@ -94,9 +94,10 @@ Open **Settings → Plugins → token-ledger → TokenLedger** on the selected h - **Timeline summaries** controls summaries for newly finished turns; existing rows remain intact. - **OpenRouter reference prices** controls both cached reference prices and background catalog refresh. - **Built-in fallback prices** controls estimates when neither custom nor enabled OpenRouter pricing matches. +- **Prefer price estimates** (off by default) uses a matching custom or reference price instead of the runtime-reported cost, for example for local models that are never billed. Turns without a matching price keep the reported cost. The reported value stays in the ledger file, so turning the option off restores it. - **Refresh frequency** selects normal (3 seconds active / 30 seconds idle) or relaxed (10 / 60 seconds) fallback polling. Live updates still trigger refreshes. -Settings apply without a plugin reload and are shared by clients of that host. Reported costs and `pricing.json` overrides always retain priority. These preferences do not alter ledger retention or erase history. +Settings apply without a plugin reload and are shared by clients of that host. By default, reported costs and `pricing.json` overrides retain priority. These preferences do not alter ledger retention or erase history. All three switches default to on, and refresh frequency defaults to normal. @@ -114,7 +115,7 @@ A "turn" is one user message through to `turn_completed`, `turn_failed`, or `tur Counting conventions are selected by verified harness/provider identity, never by the model name. OMP input is already exclusive of cache; Codex input is inclusive. “IN” is uncached input after subtracting cache reads and known writes; if writes are omitted, Codex IN may still include unclassified writes. Unknown provider values stay as reported, without guessed normalization. -For attributable turn costs, the order is runtime-reported cost, local override, OpenRouter reference, then built-in fallback. Claude session-cost deltas use the known baseline; a lower counter alone does not prove a session reset. A runtime's reported cost can itself be an estimate: it is not a verified invoice, and custom reference pricing does not replace it. Residuals do not identify missing tokens or billing categories. +For attributable turn costs, the order is runtime-reported cost, local override, OpenRouter reference, then built-in fallback; **Prefer price estimates** moves the reported cost to the end. Claude session-cost deltas use the known baseline; a lower counter alone does not prove a session reset. A runtime's reported cost can itself be an estimate: it is not a verified invoice, and custom pricing replaces it only when **Prefer price estimates** is on. Residuals do not identify missing tokens or billing categories. Estimated costs are marked `≈`; missing components/rates or uncertain request coverage add `partial`. Summaries expose unknown-cost turns rather than presenting the known subtotal as a complete bill. Codex request observations are priced separately; legacy records without requests use the aggregate as an explicitly partial estimate. A tiered price cannot be selected when the complete prompt size is unknown. Per-request routed model identities are not exposed, so reference pricing uses the recorded selected model, not a claim of the actual billed model. diff --git a/client/settings.tsx b/client/settings.tsx index 0112d30..d6f0738 100644 --- a/client/settings.tsx +++ b/client/settings.tsx @@ -31,10 +31,11 @@ export function TokenLedgerSettings({ theme }: PluginSurfaceProps) { save({ openRouterPricing })} /> save({ builtinPricing })} /> + save({ preferEstimates })} /> - Reported costs take priority, followed by custom prices and enabled reference sources. Reference prices may differ from gateway discounts or Fast Mode charges. Preferences apply to clients connected to this host. + Reported costs take priority unless price estimates are preferred, followed by custom prices and enabled reference sources. Reference prices may differ from gateway discounts or Fast Mode charges. Preferences apply to clients connected to this host. {settings.saveError ? {settings.saveError} : null} ); diff --git a/server/preferences.test.ts b/server/preferences.test.ts index 170392d..81fde62 100644 --- a/server/preferences.test.ts +++ b/server/preferences.test.ts @@ -25,7 +25,7 @@ test('settings changes win over a stale read, update pricing immediately, and pr const pricing = await import('./pricing.ts'); const { TimelinePublisher } = await import('./timeline.ts'); try { - notify({ status: 'ready', revision: 'new', values: { ...DEFAULT_PREFERENCES, timelineSummaries: false, openRouterPricing: false, builtinPricing: false } }); + notify({ status: 'ready', revision: 'new', values: { ...DEFAULT_PREFERENCES, timelineSummaries: false, openRouterPricing: false, builtinPricing: false, preferEstimates: true } }); resolveRead({ status: 'ready', revision: 'old', values: DEFAULT_PREFERENCES }); await settings.ready; assert.equal(getPreferences().openRouterPricing, false); @@ -36,7 +36,13 @@ test('settings changes win over a stale read, update pricing immediately, and pr assert.equal(pricing.estimateUsageCost({ ...usage, model: 'custom' }).effectiveCostUsd, 9); const { finalizeTurn } = await import('../shared/aggregate.ts'); const record = finalizeTurn({ agentId: 'a', turnId: 't', provider: 'claude', model: 'claude-opus-5.5', startedAt: null, endedAt: new Date().toISOString(), status: 'completed', observations: [], finalUsage: { inputTokens: 1_000_000, totalCostUsd: 2 }, prevSessionCostUsd: 0 }); + // Preferring estimates still falls back to the reported cost when no price matches. assert.equal(pricing.enrichTurn(record, 1).effectiveCostUsd, 2); + const local = pricing.enrichTurn({ ...record, model: 'custom' }, 1); + assert.equal(local.effectiveCostUsd, 9); + assert.equal(local.costSource, 'override'); + assert.equal(local.costUsd, null); + assert.equal(local.costBreakdown?.otherUsd, null); const publisher = new TimelinePublisher(); publisher.publish({} as never, record); // Disabled summaries must never call the timeline API. await publisher.flush(); @@ -45,6 +51,7 @@ test('settings changes win over a stale read, update pricing immediately, and pr assert.notEqual(pricing.pricingRevision(), previous); assert.equal(pricing.estimateUsageCost(usage).effectiveCostUsd, 4); assert.equal(pricing.estimateUsageCost({ ...usage, model: 'remote' }).effectiveCostUsd, 1); + assert.equal(pricing.enrichTurn({ ...record, model: 'custom' }, 1).effectiveCostUsd, 2, 'reported costs win by default'); } finally { await settings.dispose(); if (oldHome === undefined) delete process.env.PASEO_HOME; else process.env.PASEO_HOME = oldHome; diff --git a/server/pricing.ts b/server/pricing.ts index c933f1f..bfd3922 100644 --- a/server/pricing.ts +++ b/server/pricing.ts @@ -239,7 +239,9 @@ export function enrichTurn(record: TurnRecord, seq: number): TurnRow { const scope = tokenScope(record); const input = freshInput(usageSemantics(record.provider), record.input, record.cached, record.cacheWrite); const estimated = estimateUsageCost(record); - const reported = scope === 'turn' ? record.costUsd : null; + const runtime = scope === 'turn' ? record.costUsd : null; + // Runtimes can report list-price fallbacks for unbilled local models; the raw value stays on disk. + const reported = getPreferences().preferEstimates && estimated.effectiveCostUsd !== null ? null : runtime; const breakdown = estimated.costBreakdown && reported !== null ? { ...estimated.costBreakdown, otherUsd: reported - estimated.effectiveCostUsd! } : estimated.costBreakdown; diff --git a/shared/preferences.ts b/shared/preferences.ts index 6ab2b0a..93bd5d4 100644 --- a/shared/preferences.ts +++ b/shared/preferences.ts @@ -7,6 +7,7 @@ export const preferences = defineSettings({ timelineSummaries: z.boolean().default(true), openRouterPricing: z.boolean().default(true), builtinPricing: z.boolean().default(true), + preferEstimates: z.boolean().default(false), refreshInterval: z.enum(['normal', 'relaxed']).default('normal'), }), }); From 454e5c54be47f1f1d885e29f2c61e449a4274b8c Mon Sep 17 00:00:00 2001 From: stv1024 Date: Fri, 23 Oct 2026 15:28:58 +0800 Subject: [PATCH 2/2] feat: let custom prices override reported costs; follow provider extends Rename the unreleased "Prefer price estimates" setting to "Custom prices override reported costs" (customPricesOverride). Only a matching pricing.json price replaces a runtime-reported cost; OpenRouter and built-in prices never do, because they miss cache writes that the reported Claude cost includes. Custom providers that declare extends: claude/codex in the daemon config now follow their base harness. The plugin reads provider bases through paseo.config.get() (throttled), and new records store the resolved harness so later config changes do not reinterpret them. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- OVERVIEW.md | 4 ++-- README.md | 8 ++++---- client/settings.tsx | 4 ++-- index.server.ts | 15 ++++++++------ server/preferences.test.ts | 36 ++++++++++++++++++++++++++++++++-- server/pricing.ts | 20 +++++++++++-------- server/providers.ts | 29 +++++++++++++++++++++++++++ shared/aggregate.ts | 4 +++- shared/ledger.ts | 2 ++ shared/preferences.ts | 2 +- shared/semantics.ts | 40 +++++++++++++++++++++++++++++++------- 12 files changed, 133 insertions(+), 34 deletions(-) create mode 100644 server/providers.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 873739e..ba2f529 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## Unreleased -- Add a **Prefer price estimates** setting (off by default). When a custom or reference price matches, it replaces the runtime-reported cost, which can be a list-price fallback for local or unbilled models. The reported value stays in the ledger file (#1). +- Add a **Custom prices override reported costs** setting (off by default). A matching `pricing.json` price replaces the runtime-reported cost, which can be a list-price fallback for local or unbilled models. Reference prices never replace reported costs. The reported value stays in the ledger file (#1). +- Recognize custom providers that declare `extends: "claude"` or `extends: "codex"` in the daemon config. Their turns now use the base harness's usage and cost semantics instead of unknown-scope snapshots. New records store the resolved harness; earlier records of such providers stay unknown-scope snapshots (#1). ## v0.8.0 (2026-10-23) — Paseo 0.12 background workspaces and accounting accuracy diff --git a/OVERVIEW.md b/OVERVIEW.md index c6cfccf..dedc518 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -2,7 +2,7 @@ TokenLedger records the tokens and cost of each agent turn and keeps the history The agent panel shows session totals, the turn in progress with its context window, and one row per finished turn with status, time, duration, input, cache and output tokens, cost, and a data quality mark. The pill shows the session cost and context use, with an activity icon while a turn runs, and opens the panel. The sidebar row opens the overview, which groups agents by workspace and filters to today, the last 7 days, the last 30 days, or all time. Day ranges start at local midnight on the device that shows them. After each finished turn, the plugin can add one short usage summary to the timeline. -Token counts come from the provider. Claude terminal aggregates and observed Codex requests contribute to turn subtotals. OMP current-session/branch totals and unverified-harness observations remain visible as snapshots but are excluded from turn sums and prices. A separate latest-snapshot card retains late updates without inventing turn attribution. Missing counts/rates display as unknown; cache hit rates require a complete prompt denominator. Partial estimates and unknown-cost turns are explicit in the panel, pill and overview. Runtime-reported costs take priority over your reference prices, OpenRouter references and built-in prices, but are not verified bills. +Token counts come from the provider. Claude terminal aggregates and observed Codex requests contribute to turn subtotals. OMP current-session/branch totals and unverified-harness observations remain visible as snapshots but are excluded from turn sums and prices. A separate latest-snapshot card retains late updates without inventing turn attribution. Missing counts/rates display as unknown; cache hit rates require a complete prompt denominator. Partial estimates and unknown-cost turns are explicit in the panel, pill and overview. By default, runtime-reported costs take priority over your reference prices, OpenRouter references and built-in prices, but are not verified bills. The pill distinguishes known session snapshots (`Session ≈4.12M tok`), unknown-scope snapshots, waiting for usage and no reported usage. Old timeline cards without scope metadata are labelled as legacy; their ledger inclusion is not guessed from the card. @@ -11,7 +11,7 @@ The plugin settings page on each host has five options: - **Timeline summaries** turns the usage summary after each new turn on or off. Existing summaries stay. - **OpenRouter reference prices** lets the plugin download the public model price list from `openrouter.ai` in the background and cache it for 24 hours. The request carries no tokens, prompts, or usage data. Turn it off to stop the download and stop using those prices. - **Built-in fallback prices** controls estimates when no custom or OpenRouter price matches a model. -- **Prefer price estimates** uses a matching custom or reference price instead of the cost the runtime reports. Use it for local or unbilled models. It is off by default. +- **Custom prices override reported costs** uses your matching custom price instead of the cost the runtime reports. Use it for local or unbilled models. It is off by default. - **Refresh frequency** selects normal or relaxed polling. Live updates still refresh the views immediately. The ledger is a JSON Lines file on the daemon host, `~/.paseo/plugins/token-ledger/` (or under `PASEO_HOME`), retaining 2,000 records. It stores observed usage, timestamps, identity, scope and quality, not prompts, responses, tool arguments or credentials. Older records remain readable and are reinterpreted without rewriting them. `pricing.json` starts empty and preserves your existing prices. Full model IDs and explicit aliases prevent unrelated routes from sharing a guessed price. History does not sync between hosts. diff --git a/README.md b/README.md index a54c05a..eb9948c 100644 --- a/README.md +++ b/README.md @@ -94,12 +94,12 @@ Open **Settings → Plugins → token-ledger → TokenLedger** on the selected h - **Timeline summaries** controls summaries for newly finished turns; existing rows remain intact. - **OpenRouter reference prices** controls both cached reference prices and background catalog refresh. - **Built-in fallback prices** controls estimates when neither custom nor enabled OpenRouter pricing matches. -- **Prefer price estimates** (off by default) uses a matching custom or reference price instead of the runtime-reported cost, for example for local models that are never billed. Turns without a matching price keep the reported cost. The reported value stays in the ledger file, so turning the option off restores it. +- **Custom prices override reported costs** (off by default) uses a matching `pricing.json` price instead of the runtime-reported cost, for example for local models that a runtime prices at a list-price fallback. Add `providers` to an entry to limit it to one provider. OpenRouter and built-in prices never replace a reported cost. The reported value stays in the ledger file, so turning the option off restores it. - **Refresh frequency** selects normal (3 seconds active / 30 seconds idle) or relaxed (10 / 60 seconds) fallback polling. Live updates still trigger refreshes. Settings apply without a plugin reload and are shared by clients of that host. By default, reported costs and `pricing.json` overrides retain priority. These preferences do not alter ledger retention or erase history. -All three switches default to on, and refresh frequency defaults to normal. +The other three switches default to on, and refresh frequency defaults to normal. ## How turns are measured @@ -113,9 +113,9 @@ A "turn" is one user message through to `turn_completed`, `turn_failed`, or `tur | Unverified harness | latest reported snapshot | unknown; excluded from per-turn totals and prices | | Nothing reported | unknown, not zero | unavailable | -Counting conventions are selected by verified harness/provider identity, never by the model name. OMP input is already exclusive of cache; Codex input is inclusive. “IN” is uncached input after subtracting cache reads and known writes; if writes are omitted, Codex IN may still include unclassified writes. Unknown provider values stay as reported, without guessed normalization. +Counting conventions are selected by verified harness/provider identity, never by the model name. A custom provider that declares `extends: "claude"` or `extends: "codex"` in the daemon config follows its base harness; new records store the resolved harness. OMP input is already exclusive of cache; Codex input is inclusive. “IN” is uncached input after subtracting cache reads and known writes; if writes are omitted, Codex IN may still include unclassified writes. Unknown provider values stay as reported, without guessed normalization. -For attributable turn costs, the order is runtime-reported cost, local override, OpenRouter reference, then built-in fallback; **Prefer price estimates** moves the reported cost to the end. Claude session-cost deltas use the known baseline; a lower counter alone does not prove a session reset. A runtime's reported cost can itself be an estimate: it is not a verified invoice, and custom pricing replaces it only when **Prefer price estimates** is on. Residuals do not identify missing tokens or billing categories. +For attributable turn costs, the order is runtime-reported cost, local override, OpenRouter reference, then built-in fallback; **Custom prices override reported costs** puts a matching local override first. Claude session-cost deltas use the known baseline; a lower counter alone does not prove a session reset. A runtime's reported cost can itself be an estimate: it is not a verified invoice, and custom pricing replaces it only when **Custom prices override reported costs** is on. Residuals do not identify missing tokens or billing categories. Estimated costs are marked `≈`; missing components/rates or uncertain request coverage add `partial`. Summaries expose unknown-cost turns rather than presenting the known subtotal as a complete bill. Codex request observations are priced separately; legacy records without requests use the aggregate as an explicitly partial estimate. A tiered price cannot be selected when the complete prompt size is unknown. Per-request routed model identities are not exposed, so reference pricing uses the recorded selected model, not a claim of the actual billed model. diff --git a/client/settings.tsx b/client/settings.tsx index d6f0738..80d70e1 100644 --- a/client/settings.tsx +++ b/client/settings.tsx @@ -31,11 +31,11 @@ export function TokenLedgerSettings({ theme }: PluginSurfaceProps) { save({ openRouterPricing })} /> save({ builtinPricing })} /> - save({ preferEstimates })} /> + save({ customPricesOverride })} /> - Reported costs take priority unless price estimates are preferred, followed by custom prices and enabled reference sources. Reference prices may differ from gateway discounts or Fast Mode charges. Preferences apply to clients connected to this host. + Reported costs take priority, followed by custom prices and enabled reference sources. Reference prices may differ from gateway discounts or Fast Mode charges. Preferences apply to clients connected to this host. {settings.saveError ? {settings.saveError} : null} ); diff --git a/index.server.ts b/index.server.ts index e9dff93..22c7d14 100644 --- a/index.server.ts +++ b/index.server.ts @@ -2,12 +2,15 @@ import type { PluginServerContext } from "@getpaseo/plugin/server"; import { ledgerEnsure, ledgerOverview, ledgerSync } from "./shared/ledger.ts"; import { ensureTracker, handleEnsure, handleOverview, handleSync, observeStart, observeEnd, prepareAgent, stopTracker } from "./server/tracker.ts"; import { initializePreferences } from './server/preferences.ts'; +import { refreshProviderBases } from './server/providers.ts'; +import type { PaseoApi } from './server/paseo-types.ts'; export default function contribute(server: PluginServerContext) { const preferences = initializePreferences(server); - server.handle(ledgerSync, async (input, context) => { await preferences.ready; return handleSync(input, context); }); - server.handle(ledgerEnsure, async (input, context) => { await preferences.ready; return handleEnsure(input, context); }); - server.handle(ledgerOverview, async (input, context) => { await preferences.ready; return handleOverview(input, context); }); + const ready = async (paseo: PaseoApi) => { await preferences.ready; await refreshProviderBases(paseo); }; + server.handle(ledgerSync, async (input, context) => { await ready(context.paseo); return handleSync(input, context); }); + server.handle(ledgerEnsure, async (input, context) => { await ready(context.paseo); return handleEnsure(input, context); }); + server.handle(ledgerOverview, async (input, context) => { await ready(context.paseo); return handleOverview(input, context); }); // Establish timeline demand before the provider can emit its first turn. // Lifecycle hooks run even with no desktop/mobile client connected. const beforeOpen = server.before("agent.session_open", async ({ request }, { paseo }) => { @@ -15,7 +18,7 @@ export default function contribute(server: PluginServerContext) { // A newly-created agent is not in the daemon catalog until session_open // returns. Establish catalog demand now; its first upsert attaches usage. try { - await preferences.ready; + await ready(paseo); if (request.reason === "create") await ensureTracker(paseo); else await prepareAgent(paseo, request.agentId); } catch (error) { @@ -24,10 +27,10 @@ export default function contribute(server: PluginServerContext) { }); // Covers a plugin reload while an existing provider session stays open. const onStart = server.on("agent.turn_started", async (event, { paseo }) => { - try { await preferences.ready; await observeStart(event, paseo); } + try { await ready(paseo); await observeStart(event, paseo); } catch (error) { console.error("token-ledger: could not attach turn tracking", error); } }); - const onEnd = server.on("agent.turn_ended", async (event, { paseo }) => { await preferences.ready; await observeEnd(event, paseo); }); + const onEnd = server.on("agent.turn_ended", async (event, { paseo }) => { await ready(paseo); await observeEnd(event, paseo); }); return async () => { beforeOpen(); onStart(); diff --git a/server/preferences.test.ts b/server/preferences.test.ts index 81fde62..3ed5c77 100644 --- a/server/preferences.test.ts +++ b/server/preferences.test.ts @@ -25,7 +25,7 @@ test('settings changes win over a stale read, update pricing immediately, and pr const pricing = await import('./pricing.ts'); const { TimelinePublisher } = await import('./timeline.ts'); try { - notify({ status: 'ready', revision: 'new', values: { ...DEFAULT_PREFERENCES, timelineSummaries: false, openRouterPricing: false, builtinPricing: false, preferEstimates: true } }); + notify({ status: 'ready', revision: 'new', values: { ...DEFAULT_PREFERENCES, timelineSummaries: false, openRouterPricing: false, builtinPricing: false, customPricesOverride: true } }); resolveRead({ status: 'ready', revision: 'old', values: DEFAULT_PREFERENCES }); await settings.ready; assert.equal(getPreferences().openRouterPricing, false); @@ -36,7 +36,7 @@ test('settings changes win over a stale read, update pricing immediately, and pr assert.equal(pricing.estimateUsageCost({ ...usage, model: 'custom' }).effectiveCostUsd, 9); const { finalizeTurn } = await import('../shared/aggregate.ts'); const record = finalizeTurn({ agentId: 'a', turnId: 't', provider: 'claude', model: 'claude-opus-5.5', startedAt: null, endedAt: new Date().toISOString(), status: 'completed', observations: [], finalUsage: { inputTokens: 1_000_000, totalCostUsd: 2 }, prevSessionCostUsd: 0 }); - // Preferring estimates still falls back to the reported cost when no price matches. + // Without a custom price the reported cost stays. assert.equal(pricing.enrichTurn(record, 1).effectiveCostUsd, 2); const local = pricing.enrichTurn({ ...record, model: 'custom' }, 1); assert.equal(local.effectiveCostUsd, 9); @@ -52,9 +52,41 @@ test('settings changes win over a stale read, update pricing immediately, and pr assert.equal(pricing.estimateUsageCost(usage).effectiveCostUsd, 4); assert.equal(pricing.estimateUsageCost({ ...usage, model: 'remote' }).effectiveCostUsd, 1); assert.equal(pricing.enrichTurn({ ...record, model: 'custom' }, 1).effectiveCostUsd, 2, 'reported costs win by default'); + notify({ status: 'ready', revision: 'override', values: { ...DEFAULT_PREFERENCES, customPricesOverride: true } }); + const builtin = pricing.enrichTurn(record, 1); + assert.equal(builtin.effectiveCostUsd, 2, 'reference prices never replace reported costs'); + assert.equal(builtin.costSource, 'reported'); + assert.equal(pricing.enrichTurn({ ...record, model: 'custom' }, 1).effectiveCostUsd, 9); } finally { await settings.dispose(); if (oldHome === undefined) delete process.env.PASEO_HOME; else process.env.PASEO_HOME = oldHome; await rm(home, { recursive: true, force: true }); } }); + +test('custom providers that extend a known harness follow its usage semantics', async () => { + const { setProviderBases, providerBasesRevision } = await import('../shared/semantics.ts'); + const { finalizeTurn } = await import('../shared/aggregate.ts'); + const pricing = await import('./pricing.ts'); + const turn = () => finalizeTurn({ agentId: 'a', turnId: 't', provider: 'local-claude', model: 'acme/local-1', startedAt: null, endedAt: new Date().toISOString(), status: 'completed', observations: [], finalUsage: { inputTokens: 10, cachedInputTokens: 0, outputTokens: 5, totalCostUsd: 2 }, prevSessionCostUsd: 0 }); + try { + const unresolved = turn(); + assert.equal(unresolved.tokenScope, 'unknown'); + assert.equal(unresolved.harness, undefined); + const previous = pricing.pricingRevision(); + setProviderBases({ 'local-claude': 'claude' }); + assert.notEqual(pricing.pricingRevision(), previous); + const resolved = turn(); + assert.equal(resolved.tokenScope, 'turn'); + assert.equal(resolved.harness, 'claude'); + assert.equal(resolved.costUsd, 2); + const revision = providerBasesRevision(); + setProviderBases({ 'local-claude': 'claude' }); + assert.equal(providerBasesRevision(), revision, 'an unchanged config keeps cached rows'); + // The recorded harness keeps semantics when the config later changes. + setProviderBases({}); + assert.equal(pricing.enrichTurn(resolved, 1).effectiveCostUsd, 2); + } finally { + setProviderBases({}); + } +}); diff --git a/server/pricing.ts b/server/pricing.ts index bfd3922..9c7e37b 100644 --- a/server/pricing.ts +++ b/server/pricing.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { z } from "zod"; import type { CostSource, TurnRecord, TurnRow } from "../shared/ledger.ts"; import { canonicalModel, costBreakdownWithPricing, knownCost, modelPricing, partialCost, type CostBreakdown, type ModelPricing } from "../shared/pricing.ts"; -import { freshInput, promptTokens, tokenScope, usageSemantics } from "../shared/semantics.ts"; +import { freshInput, promptTokens, providerBasesRevision, tokenScope, usageSemantics } from "../shared/semantics.ts"; import { getPreferences, preferencesRevision } from './preferences.ts'; type PriceTier = ModelPricing & { upToInputTokens?: number }; @@ -64,7 +64,7 @@ let overridePrices: PriceEntry[] = []; let openRouterPrices: OpenRouterPrice[] = []; let loadPromise: Promise | null = null; let revision = 0; -export const pricingRevision = () => `${revision}:${preferencesRevision()}`; +export const pricingRevision = () => `${revision}:${preferencesRevision()}:${providerBasesRevision()}`; async function loadOverrides(): Promise { await mkdir(DATA_DIR, { recursive: true }); @@ -195,6 +195,7 @@ function resolvePricing(provider: string | null, model: string | null, prompt: n } type PriceableUsage = Pick & { + harness?: TurnRecord['harness']; requests?: TurnRecord['requests']; tokenScope?: TurnRecord['tokenScope']; }; @@ -208,15 +209,16 @@ export function estimateUsageCost(usage: PriceableUsage): { pricingWarning: string | null; } { const unavailable = { effectiveCostUsd: null, costSource: null, costBreakdown: null, partialCost: true, pricingWarning: null }; - if (tokenScope(usage) !== 'turn' || !usageSemantics(usage.provider)) return unavailable; - const semantics = usageSemantics(usage.provider); + const harness = usage.harness ?? usage.provider; + const semantics = usageSemantics(harness); + if (tokenScope(usage) !== 'turn' || !semantics) return unavailable; const requests = usage.requests?.length ? usage.requests : [usage]; let breakdown: CostBreakdown | null = null; let source: Exclude | null = null; let incomplete = false; let pricingWarning: string | null = null; for (const request of requests) { - const resolved = resolvePricing(usage.provider, usage.model, promptTokens(usage.provider, request)); + const resolved = resolvePricing(usage.provider, usage.model, promptTokens(harness, request)); if (!resolved) { incomplete = true; continue; } if ('warning' in resolved) { pricingWarning ??= resolved.warning; incomplete = true; continue; } const input = freshInput(semantics, request.input, request.cached, request.cacheWrite); @@ -237,11 +239,13 @@ export function estimateUsageCost(usage: PriceableUsage): { export function enrichTurn(record: TurnRecord, seq: number): TurnRow { const scope = tokenScope(record); - const input = freshInput(usageSemantics(record.provider), record.input, record.cached, record.cacheWrite); + const harness = record.harness ?? record.provider; + const input = freshInput(usageSemantics(harness), record.input, record.cached, record.cacheWrite); const estimated = estimateUsageCost(record); const runtime = scope === 'turn' ? record.costUsd : null; // Runtimes can report list-price fallbacks for unbilled local models; the raw value stays on disk. - const reported = getPreferences().preferEstimates && estimated.effectiveCostUsd !== null ? null : runtime; + const overridden = getPreferences().customPricesOverride && estimated.costSource === 'override' && estimated.effectiveCostUsd !== null; + const reported = overridden ? null : runtime; const breakdown = estimated.costBreakdown && reported !== null ? { ...estimated.costBreakdown, otherUsd: reported - estimated.effectiveCostUsd! } : estimated.costBreakdown; @@ -249,7 +253,7 @@ export function enrichTurn(record: TurnRecord, seq: number): TurnRow { ...record, input, seq, tokenScope: scope, // Historical session/unknown rows counted snapshots, not model calls. modelCalls: scope === 'turn' ? record.modelCalls : null, - promptTokens: promptTokens(record.provider, record), + promptTokens: promptTokens(harness, record), incompleteUsage: scope !== 'turn' || record.quality !== 'exact' || !!record.usageGap || input === null || record.cached === null || record.cacheWrite == null || record.output === null, partialCost: reported === null && (estimated.partialCost || record.quality !== 'exact' || !!record.usageGap), diff --git a/server/providers.ts b/server/providers.ts new file mode 100644 index 0000000..dcc044a --- /dev/null +++ b/server/providers.ts @@ -0,0 +1,29 @@ +import type { PaseoApi } from './paseo-types.ts'; +import { setProviderBases } from '../shared/semantics.ts'; + +const MAX_AGE_MS = 30_000; +let loadedAt = 0; +let pending: Promise | null = null; + +/** Reads `extends` from custom provider entries; only provider names and base names are kept. */ +export function refreshProviderBases(paseo: PaseoApi): Promise { + if (pending) return pending; + if (Date.now() - loadedAt < MAX_AGE_MS) return Promise.resolve(); + pending = (async () => { + try { + const { config } = await paseo.config.get(); + const bases: Record = {}; + for (const [name, entry] of Object.entries(config.providers ?? {})) { + const base = (entry as { extends?: unknown }).extends; + if (typeof base === 'string' && base && base !== name) bases[name] = base; + } + setProviderBases(bases); + } catch (error) { + console.error('token-ledger: could not read provider config; keeping known provider names', error instanceof Error ? error.message : error); + } finally { + loadedAt = Date.now(); + pending = null; + } + })(); + return pending; +} diff --git a/shared/aggregate.ts b/shared/aggregate.ts index c5ef840..e94d97d 100644 --- a/shared/aggregate.ts +++ b/shared/aggregate.ts @@ -1,4 +1,4 @@ -import { providerSemantics, tokenScope } from "./semantics.ts"; +import { harnessOf, providerSemantics, tokenScope } from "./semantics.ts"; import type { TurnRecord } from "./ledger.ts"; /** Reducer input. Paseo 0.8 exposes these fields except cacheWriteInputTokens. */ @@ -126,6 +126,7 @@ export function finalizeTurn(args: FinalizeInput): TurnRecord { args.startedAt !== null ? Math.max(0, Date.parse(args.endedAt) - Date.parse(args.startedAt)) : null; const scope = tokenScope(args); + const harness = harnessOf(args.provider); return { v: 1, // endedAt is part of the id because Paseo turnIds ("foreground-turn-N") @@ -134,6 +135,7 @@ export function finalizeTurn(args: FinalizeInput): TurnRecord { agentId: args.agentId, turnId: args.turnId, provider: args.provider, + ...(harness !== args.provider ? { harness } : {}), tokenScope: scope, model: args.model, startedAt: args.startedAt, diff --git a/shared/ledger.ts b/shared/ledger.ts index 16ec693..948a37f 100644 --- a/shared/ledger.ts +++ b/shared/ledger.ts @@ -8,6 +8,8 @@ export const TurnRecordSchema = z.object({ turnId: z.string().nullable(), provider: z.string().nullable(), model: z.string().nullable(), + /** Base harness of a custom provider declared with `extends`, resolved when recorded. */ + harness: z.string().nullable().optional(), /** Omitted in v1 history; inferred only from verified harness semantics. */ tokenScope: z.enum(['turn', 'session', 'unknown']).optional(), /** Provider session identity, when observed; optional for pre-0.8 records. */ diff --git a/shared/preferences.ts b/shared/preferences.ts index 93bd5d4..b2933ff 100644 --- a/shared/preferences.ts +++ b/shared/preferences.ts @@ -7,7 +7,7 @@ export const preferences = defineSettings({ timelineSummaries: z.boolean().default(true), openRouterPricing: z.boolean().default(true), builtinPricing: z.boolean().default(true), - preferEstimates: z.boolean().default(false), + customPricesOverride: z.boolean().default(false), refreshInterval: z.enum(['normal', 'relaxed']).default('normal'), }), }); diff --git a/shared/semantics.ts b/shared/semantics.ts index 567c207..942750f 100644 --- a/shared/semantics.ts +++ b/shared/semantics.ts @@ -5,28 +5,54 @@ export type TokenScope = 'turn' | 'session' | 'unknown'; const CLAUDE = /^(?:claude|anthropic)(?:$|[-/:])/i; const CODEX = /^codex(?:$|[-/:])/i; const OMP = /^omp(?:$|[-/:])/i; +const known = (provider: string) => CLAUDE.test(provider) || CODEX.test(provider) || OMP.test(provider); + +// Custom providers declared with `extends` in the daemon config run their base harness. +let bases = new Map(); +let basesRevision = 0; + +export function setProviderBases(next: Record) { + const map = new Map(Object.entries(next)); + if (JSON.stringify([...map]) === JSON.stringify([...bases])) return; + bases = map; + basesRevision++; +} +export const providerBasesRevision = () => basesRevision; + +/** The harness whose usage contract a provider follows; unresolved names stay as they are. */ +export function harnessOf(provider: string | null): string | null { + let current = provider; + for (let depth = 0; current && !known(current) && depth < 4; depth++) { + const base = bases.get(current); + if (!base) break; + current = base; + } + return current && known(current) ? current : provider; +} export function providerSemantics(provider: string | null): { tokens: 'turn' | 'request' | 'session' | 'unknown'; cost: 'session' | 'unknown'; } { - if (CLAUDE.test(provider ?? '')) return { tokens: 'turn', cost: 'session' }; - if (CODEX.test(provider ?? '')) return { tokens: 'request', cost: 'unknown' }; + const harness = harnessOf(provider) ?? ''; + if (CLAUDE.test(harness)) return { tokens: 'turn', cost: 'session' }; + if (CODEX.test(harness)) return { tokens: 'request', cost: 'unknown' }; // OMP exposes current-branch session totals, not monotonic billing counters. // Without branch epochs and final-drain ordering, neither tokens nor cost // can safely be differenced into turns. - if (OMP.test(provider ?? '')) return { tokens: 'session', cost: 'unknown' }; + if (OMP.test(harness)) return { tokens: 'session', cost: 'unknown' }; return { tokens: 'unknown', cost: 'unknown' }; } -export function tokenScope(record: { provider: string | null; tokenScope?: TokenScope }): TokenScope { +export function tokenScope(record: { provider: string | null; harness?: string | null; tokenScope?: TokenScope }): TokenScope { if (record.tokenScope) return record.tokenScope; - const scope = providerSemantics(record.provider).tokens; + const scope = providerSemantics(record.harness ?? record.provider).tokens; return scope === 'request' ? 'turn' : scope; } export function usageSemantics(provider: string | null): UsageSemantics | null { - if (CLAUDE.test(provider ?? '') || OMP.test(provider ?? '')) return { inputIncludesCached: false }; - if (CODEX.test(provider ?? '')) return { inputIncludesCached: true }; + const harness = harnessOf(provider) ?? ''; + if (CLAUDE.test(harness) || OMP.test(harness)) return { inputIncludesCached: false }; + if (CODEX.test(harness)) return { inputIncludesCached: true }; return null; }