From 8a1f242523a81dc7e1ac2a7b7a0f33b28fc03734 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 14:30:07 +0200 Subject: [PATCH 01/58] fix(proxy): Leave the cache marks out of the body hash accBodies hashed each message's raw JSON with its cache_control. The client moves its marks: Claude Code puts one on the newest block every turn. The previous message then had new bytes on each request, the library found no common prefix after the system block, and every token it had saved earlier was priced at the write rate although the provider read it. The booked saving grew each turn: 2.0x, 2.9x and 3.7x the real value on turns 2 to 4 in the review of #768 (B1). The hash now leaves out the mark, on the element and on its content blocks. The size keeps it: the library checks the operations' bytes against the bytes that went upstream. Two tests cover the hash. A third drives three turns with a moving mark through the tracker. Without the fix it prices the replay at the write tier (54,185 tokens). With it, at the read tier. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/accountant_savings_body.go | 70 +++++++++++++++++++++------ proxy/accountant_savings_body_test.go | 45 +++++++++++++++++ proxy/accountant_savings_test.go | 42 ++++++++++++++++ 3 files changed, 143 insertions(+), 14 deletions(-) diff --git a/proxy/accountant_savings_body.go b/proxy/accountant_savings_body.go index 83cffc1e..924119c5 100644 --- a/proxy/accountant_savings_body.go +++ b/proxy/accountant_savings_body.go @@ -1,34 +1,76 @@ package proxy import ( + "strconv" + "strings" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" "github.com/rossoctl/accountant" ) // accBodies hashes a request body in the provider's order: tools, system, messages. A string -// system prompt is one block. Raw JSON is hashed as sent, so two hosts hash the same bytes alike. -// Each element of messages is one position, so Position{messages, i} is the wire message index. -// A body without messages (or with invalid JSON) gives an empty Body. +// system prompt is one block. Raw JSON is hashed as sent, except for the cache marks (see below), so two +// hosts hash the same bytes alike. Each element of messages is one position, so Position{messages, i} is +// the wire message index. A body without messages (or with invalid JSON) gives an empty Body. +// +// THE HASH LEAVES OUT cache_control, THE SIZE KEEPS IT. The provider's own cache key ignores the marks, +// and the client moves its marks: Claude Code puts one on the newest block every turn. If the mark were +// in the hash, the previous message would have new bytes on each request, the library would find no +// common prefix after the system block, and every token it saved earlier would be priced at the write rate +// although the provider read it (accountant#21). The size is the wire size, with the marks, because the +// library checks the operations' bytes against the bytes that really went upstream. func accBodies(raw []byte) accountant.Body { if !gjson.GetBytes(raw, "messages").IsArray() { return accountant.NewBody(nil, nil, nil) } var tools, system, msgs [][]byte - gjson.GetBytes(raw, "tools").ForEach(func(_, v gjson.Result) bool { - tools = append(tools, []byte(v.Raw)) - return true - }) + var sizes []int // the wire size of each element, in hash order + add := func(dst *[][]byte, v gjson.Result) { + *dst = append(*dst, withoutCacheMarks(v.Raw)) + sizes = append(sizes, len(v.Raw)) + } + gjson.GetBytes(raw, "tools").ForEach(func(_, v gjson.Result) bool { add(&tools, v); return true }) if s := gjson.GetBytes(raw, "system"); s.IsArray() { - s.ForEach(func(_, v gjson.Result) bool { system = append(system, []byte(v.Raw)); return true }) + s.ForEach(func(_, v gjson.Result) bool { add(&system, v); return true }) } else if s.Exists() { - system = append(system, []byte(s.Raw)) + add(&system, s) + } + gjson.GetBytes(raw, "messages").ForEach(func(_, v gjson.Result) bool { add(&msgs, v); return true }) + b := accountant.NewBody(tools, system, msgs) + for i := range b.Pos { + b.Pos[i].Bytes = int32(sizes[i]) + } + return b +} + +// withoutCacheMarks is one element's JSON without any cache_control: the element's own, and the ones on +// its content blocks. An element with none is returned as it is, byte for byte, so the common case costs +// no allocation. The result is only hashed. It is never sent anywhere. +func withoutCacheMarks(raw string) []byte { + if !strings.Contains(raw, "cache_control") { + return []byte(raw) + } + out := raw + if gjson.Get(out, "cache_control").Exists() { + if next, err := sjson.Delete(out, "cache_control"); err == nil { + out = next + } + } + if content := gjson.Get(out, "content"); content.IsArray() { + i := 0 + content.ForEach(func(_, blk gjson.Result) bool { + if blk.Get("cache_control").Exists() { + if next, err := sjson.Delete(out, "content."+strconv.Itoa(i)+".cache_control"); err == nil { + out = next + } + } + i++ + return true + }) } - gjson.GetBytes(raw, "messages").ForEach(func(_, v gjson.Result) bool { - msgs = append(msgs, []byte(v.Raw)) - return true - }) - return accountant.NewBody(tools, system, msgs) + return []byte(out) } // accBreakpoints lists the cache_control marks, as positions in the same space as accBodies, one per diff --git a/proxy/accountant_savings_body_test.go b/proxy/accountant_savings_body_test.go index 1ce4aa61..e90b4172 100644 --- a/proxy/accountant_savings_body_test.go +++ b/proxy/accountant_savings_body_test.go @@ -95,3 +95,48 @@ func TestCfBreakpointsOnePerPositionLongerTTL(t *testing.T) { } } } + +// The client moves its cache mark: turn 2 has the mark on message 1 and none on message 0, turn 3 moves it to +// message 2. The text of messages 0 and 1 did not change, so their hashes must not change. If the mark were +// in the hash, message 1 would look edited on turn 3 and the library would find no common prefix after the +// system block (accountant#21, context-guru#768 B1). +func TestAccBodies_AMovedCacheMarkDoesNotChangeTheHash(t *testing.T) { + const mark = `,"cache_control":{"type":"ephemeral"}` + turn := func(m0, m1, m2 string) []byte { + return []byte(`{"system":"s","messages":[` + + `{"role":"user","content":[{"type":"text","text":"a"` + m0 + `}]},` + + `{"role":"assistant","content":[{"type":"text","text":"b"` + m1 + `}]},` + + `{"role":"user","content":[{"type":"text","text":"c"` + m2 + `}]}]}`) + } + t2, t3 := accBodies(turn("", mark, "")), accBodies(turn("", "", mark)) + for i := 0; i < 3; i++ { // system, message 0, message 1: the same text on both turns + if t2.Pos[i].Hash != t3.Pos[i].Hash { + t.Errorf("position %d hashes differ only by a moved cache mark", i) + } + } + // The size is the wire size: the mark is bytes the provider received. + if with, without := t2.Pos[2].Bytes, t3.Pos[2].Bytes; with <= without || int(with-without) != len(mark) { + t.Errorf("message 1 is %d bytes with the mark and %d without, want a difference of %d", with, without, len(mark)) + } +} + +// A mark on the element itself (not on a content block) is left out of the hash too. +func TestAccBodies_AnElementMarkIsLeftOutOfTheHash(t *testing.T) { + a := accBodies([]byte(`{"messages":[{"role":"user","content":"x","cache_control":{"type":"ephemeral"}}]}`)) + b := accBodies([]byte(`{"messages":[{"role":"user","content":"x"}]}`)) + if a.Pos[0].Hash != b.Pos[0].Hash { + t.Fatal("an element mark changed the hash") + } + if a.Pos[0].Bytes <= b.Pos[0].Bytes { + t.Fatal("the marked element should be larger on the wire") + } +} + +// A different text still changes the hash: leaving out the mark must not hide a real edit. +func TestAccBodies_ARealEditStillChangesTheHash(t *testing.T) { + a := accBodies([]byte(`{"messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral"}}]}]}`)) + b := accBodies([]byte(`{"messages":[{"role":"user","content":[{"type":"text","text":"b","cache_control":{"type":"ephemeral"}}]}]}`)) + if a.Pos[0].Hash == b.Pos[0].Hash { + t.Fatal("an edited text kept the same hash") + } +} diff --git a/proxy/accountant_savings_test.go b/proxy/accountant_savings_test.go index f666b8f4..95d05cb5 100644 --- a/proxy/accountant_savings_test.go +++ b/proxy/accountant_savings_test.go @@ -218,3 +218,45 @@ func TestSavings_ChangedChangeIDDoubleCountIsGated(t *testing.T) { } var accPrice = modelinfo.Price{Input: 3e-6, CacheWrite: 3.75e-6, CacheRead: 0.3e-6, Output: 15e-6} + +// The client moves its cache mark to the newest message every turn, as Claude Code does. A change made on turn 1 +// and replayed on turns 2 and 3 sits inside what the provider keeps reading, so on a hot turn it is priced at +// the READ tier. With the mark in the hash, the previous message looked edited every turn, the library found +// no common prefix, and the replay was priced at the WRITE tier: the booked saving grew each turn while the +// real one did not (context-guru#768 B1: 2.0x, 2.9x, 3.7x on turns 2 to 4). +func TestSavings_AMovingCacheMarkDoesNotRepriceAReplayAtTheWriteRate(t *testing.T) { + c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} + const id = "cg:mask:mm" + big, nBig := msg("user", strings.Repeat("tool result ", 400)) + small, nSmall := msg("user", "[masked]") + // withMark puts the mark on a message's single text block. + withMark := func(text string) string { + return fmt.Sprintf(`{"role":"assistant","content":[{"type":"text","text":%q,"cache_control":{"type":"ephemeral"}}]}`, text) + } + plain := func(text string) string { + return fmt.Sprintf(`{"role":"assistant","content":[{"type":"text","text":%q}]}`, text) + } + m0 := mustMsg("user", "first") + op := rewriteOp(id, 1, nBig, nSmall) + + // turn 1: the mark is on message 2. Nothing cached before it. + c.do(1, body(m0, big, withMark("a")), body(m0, small, withMark("a")), Usage{CacheWrite: 3000}, op) + // turn 2 and 3: the mark moves to the newest message. The text of the earlier ones did not change. + c.do(2, body(m0, big, plain("a"), withMark("b")), body(m0, small, plain("a"), withMark("b")), + Usage{CacheRead: 3000, CacheWrite: 100}, op) + res := c.do(3, body(m0, big, plain("a"), plain("b"), withMark("c")), body(m0, small, plain("a"), plain("b"), withMark("c")), + Usage{CacheRead: 3100, CacheWrite: 100}, op) + + var got *accountant.PricedOp + for i := range res.Ops { + if res.Ops[i].ChangeID == id { + got = &res.Ops[i] + } + } + if got == nil { + t.Fatalf("the replayed change is not priced: %+v", res.Ops) + } + if got.Saved.CacheWrite5m != 0 || got.Saved.Input != 0 || got.Saved.CacheRead == 0 { + t.Fatalf("turn 3 priced the replay at %+v, want the read tier (the provider read it)", got.Saved) + } +} From 0c5563b8bc731e9821a603071bff5187b8713fa7 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 14:34:28 +0200 Subject: [PATCH 02/58] fix(dash): Do not add the declaration credit to a request the accountant priced The proxy reports each toolfilter removal to the accountant library, so on a request it priced the removal is in baseline_cost_usd and in the net saving. The measured declaration credit added it again: one 1,000-token removal gave a row of $0.0001 and a total of $0.0002 (review of #768, B2). Both declaration-filter queries now leave out a request that has a toolfilter row with priced_by = 'accountant'. A request the library could not price (priced_by = 'none') and a row from before the switch have nothing in the baseline, so they keep the credit. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/declcredit_test.go | 31 +++++++++++++++++++++++++++++++ dash/toolsuggest.go | 13 +++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/dash/declcredit_test.go b/dash/declcredit_test.go index 4779a6f8..74d50934 100644 --- a/dash/declcredit_test.go +++ b/dash/declcredit_test.go @@ -454,3 +454,34 @@ func TestOverlapFiresForASkillInTheConfigForm(t *testing.T) { bare.SelfOverlap) } } + +// The accountant library prices each toolfilter removal, so on a request it priced the removal is already +// in baseline_cost_usd and in the net saving. The measured credit must not add it a second time +// (context-guru#768 B2: one 1000-token removal gave a row of $0.0001 and a total of $0.0002). A request it +// could not price, and one written before the switch, have nothing in the baseline: they keep the credit. +func TestDeclCreditSkipsRequestsTheAccountantPriced(t *testing.T) { + db := openTestDB(t) + mk := func(i int, pricedBy string) *Event { + e := mkEvent(day(0)+int64(i), "s1", "claude", 100, 90) + e.TenantID, e.Tools, e.FilteredDeclTokens = "t1", 4, 1000 + e.Components = append(e.Components, CompRow{Component: "toolfilter", Kind: KindEnvelope, Acted: true, + Mutated: true, PricedBy: pricedBy}) + return e + } + // 1: the library priced it. 2: it could not (no saving booked). 3: a row from before the switch. + if err := db.insertBatch([]*Event{mk(1, PricedByAccountant), mk(2, PricedByNone), mk(3, "")}); err != nil { + t.Fatal(err) + } + c, err := db.DeclCreditFor(Filter{Tenant: "t1"}, flatPrice, nil) + if err != nil { + t.Fatal(err) + } + if c.FilterRequests != 2 || c.FilterReads != 2000 { + t.Fatalf("filter half = %d requests / %d reads, want 2 / 2000 (the priced request is already in the baseline)", + c.FilterRequests, c.FilterReads) + } + // 2,000 cache-read tokens at flatPrice's $0.1/MTok. + if math.Abs(c.FilterUSD-0.0002) > 1e-9 { + t.Errorf("filter USD = %g, want 0.0002", c.FilterUSD) + } +} diff --git a/dash/toolsuggest.go b/dash/toolsuggest.go index c61c2064..f39c9fcf 100644 --- a/dash/toolsuggest.go +++ b/dash/toolsuggest.go @@ -189,6 +189,15 @@ func (d *DB) DeclFilterSavings(f Filter, price func(string) (modelinfo.Price, bo return out, nil } +// declFilterNotPriced keeps the requests whose declaration filtering is not already in the booked baseline. +// The proxy reports each toolfilter removal to the accountant library, so on a request the library priced +// (a toolfilter row with priced_by='accountant') the removal is in baseline_cost_usd and in the net saving. +// Adding the measured credit on top of it counts the same tokens twice (context-guru#768 B2). A request the +// library could not price (priced_by='none') or one written before the switch has nothing in the baseline, so +// the measured credit stays the only place its filter saving appears. +const declFilterNotPriced = `AND NOT EXISTS (SELECT 1 FROM request_components c + WHERE c.request_id = r.id AND c.component = 'toolfilter' AND c.priced_by = 'accountant')` + // declFilterSavingsQuery builds the SQL and args DeclFilterSavings executes — pulled out for // #502's EXPLAIN-plan test, same reason as cachesplitHistoricalQuery (dash/cachehistory.go). // Bounded by `cond` directly: a session's own index for session scope, the ts range for a @@ -197,7 +206,7 @@ func declFilterSavingsQuery(f Filter) (string, []any) { where, args := f.where() return `SELECT r.session_id, r.model, r.filtered_decl_tokens, r.cache_read, r.cache_write, r.cache_write_1h, r.ts FROM requests r - WHERE ` + where + ` AND r.filtered_decl_tokens > 0`, args + WHERE ` + where + ` AND r.filtered_decl_tokens > 0 ` + declFilterNotPriced, args } // accumulateDeclFilterRow folds one filtered_decl_tokens>0 row into a running total — the one @@ -253,7 +262,7 @@ func (d *DB) DeclFilterSavingsByTenant(since int64, price func(string) (modelinf cond, args := (Filter{Since: since, TenantAll: true}).where() rows, err := d.sql.QueryContext(d.readCtx(), `SELECT r.tenant_id, r.session_id, r.model, r.filtered_decl_tokens, r.cache_read, r.cache_write, r.cache_write_1h, r.ts FROM requests r - WHERE `+cond+` AND r.filtered_decl_tokens > 0`, args...) + WHERE `+cond+` AND r.filtered_decl_tokens > 0 `+declFilterNotPriced, args...) if err != nil { return nil, err } From da17fcadb4c0cbdc8f41dd0d0c1a4cee2b542689 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 14:36:25 +0200 Subject: [PATCH 03/58] fix(proxy): Store no ping facts for a manual ping A manual ping avoided a cache write, and that write is booked on its own row as ManualSavedUSD. PingSavings also stored the ping's facts, so the next real request credited the same write again as a library rescue. The total counted it twice: $0.0200 booked, about $0.0096 true (review of #768, B3). A manual ping is still priced, so its cost stays on its row. Only the facts are not stored. This keeps the user's estimate as the one credit for a manual ping. The other fix, to zero ManualSavedUSD and keep the library rescue, would make that credit a measured figure. It needs to know at ping time whether the next request will read the entry, and it cannot. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/accountant_ping.go | 9 ++++++++- proxy/accountant_ping_test.go | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/proxy/accountant_ping.go b/proxy/accountant_ping.go index 393f98cd..0443be37 100644 --- a/proxy/accountant_ping.go +++ b/proxy/accountant_ping.go @@ -89,6 +89,11 @@ func pingFromReport(r components.KeepAliveReport) accountant.Ping { // thread's next real request. ok is false, and nothing is stored, for a ping that is not // converted: no Window (its ChangeID would be shared by every such ping), a status outside // 200-299, or no usage (it billed nothing). Manual and sweep-substitute pings are converted. +// +// A MANUAL ping is priced, so its cost is on its row, but its facts are not stored. The write it avoided +// is already booked on its own row as ManualSavedUSD, from the user's estimate. Storing the facts too +// would let the next real request credit the same write again as a library rescue, and the total would +// count it twice (context-guru#768 B3: $0.0200 booked, about $0.0096 true). func PingSavings(r components.KeepAliveReport, st *PingStore) (accountant.Result, bool) { if r.Window.IsZero() || r.Status < 200 || r.Status > 299 || r.FreshInput+r.CacheRead+r.CacheWrite+r.CacheWrite1h+r.Output == 0 { @@ -96,6 +101,8 @@ func PingSavings(r components.KeepAliveReport, st *PingStore) (accountant.Result } ops, facts := accountant.PingOps(pingFromReport(r), accountant.Billing{Mode: accountant.BillingAPI}) res := accountant.ComputeStandalone(ops) - st.put(accPingKey(r.Session, accThreadID(r.Thread)), facts) + if !r.Manual { + st.put(accPingKey(r.Session, accThreadID(r.Thread)), facts) + } return res, true } diff --git a/proxy/accountant_ping_test.go b/proxy/accountant_ping_test.go index 8cf3e556..376a166c 100644 --- a/proxy/accountant_ping_test.go +++ b/proxy/accountant_ping_test.go @@ -300,3 +300,26 @@ func TestFinishSpendsThePingFactsOnlyWhenItPrices(t *testing.T) { t.Error("a priced request left the ping facts for the next one") } } + +// A manual ping is priced, but its facts are not stored: its avoided write is already on its row as +// ManualSavedUSD, and a stored fact would let the next real request credit the same write again as a +// rescue (context-guru#768 B3). +func TestPingSavingsDoesNotStoreFactsForAManualPing(t *testing.T) { + st := NewPingStore() + r := okReport() + r.Manual = true + res, ok := PingSavings(r, st) + if !ok || len(res.Ops) != 1 || res.Ops[0].Call == nil { + t.Fatalf("a manual ping must still be priced: ok=%v res=%+v", ok, res) + } + if f, ok := st.take(accPingKey("s", "main")); ok { + t.Fatalf("a manual ping stored facts %+v: the next request would credit its write twice", f) + } + // The automatic ping next to it still stores them. + if _, ok := PingSavings(okReport(), st); !ok { + t.Fatal("an automatic ping was not converted") + } + if _, ok := st.take(accPingKey("s", "main")); !ok { + t.Fatal("an automatic ping stored no facts") + } +} From 0845f6d38533a1039313a931449f7737cd66db02 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 14:41:51 +0200 Subject: [PATCH 04/58] fix(dash): Keep the rescue of a ping that re-created the entry The reachability gate asked for cache_read > 0 on the ping. A ping that found the entry expired re-creates it: it writes, it does not read. The accountant library credits the next request a rescue in that case, because that request reads the entry the ping wrote. The gate zeroed the credit and kept the ping's cost: the session read -$0.01146 where the hand value is -$0.00094, 12x too negative (review of #768, B4). For a row the library priced (a request_components row with priced_by = 'accountant'), the test is now cache_read > 0 OR cache_write > 0. The strict test stays for every other row. The legacy write path credited a row on keepalive_pings alone, and a ping that only re-created the entry earned it a credit it did not deserve. The old doc comment explains why, and it is still true of those rows. The credited-row guard stays first. Measured on 104,000 rows: 1.64 s before, 1.92 s after, for three Sessions calls. The comment warns of a 197x slowdown from a wrong order, and this is not that. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/keepalive.go | 22 ++++++++++++++++++---- dash/keepalivetab_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/dash/keepalive.go b/dash/keepalive.go index d0ef81bb..1057ae92 100644 --- a/dash/keepalive.go +++ b/dash/keepalive.go @@ -198,14 +198,28 @@ func kaSavedGate(alias, pingFloor string) string { // The `> 0` FIRST is not style: it short-circuits the probe to the credited rows. Measured on // a 2.1 GB database, a full-table SUM is 314 ms this way and 61.9 SECONDS with the EXISTS // first, for the identical answer. Do not reorder. - return `(CASE WHEN ` + col + ` > 0 AND EXISTS ( + probe := func(touched string) string { + return `EXISTS ( SELECT 1 FROM requests kp - WHERE kp.keepalive = 1 AND kp.cache_read > 0 + WHERE kp.keepalive = 1 AND ` + touched + ` AND kp.tenant_id = ` + alias + `tenant_id AND kp.session_id = ` + alias + `session_id AND kp.ts <= ` + alias + `ts` + pingFloor + ` AND kp.ts + ` + modelinfo.CacheLifetimeSQL("kp.provider", - alias+`cache_ttl = '`+apply.TTLEphemeral1h+`'`, time.Millisecond) + ` >= ` + alias + `ts) - THEN ` + col + ` ELSE 0 END)` + alias+`cache_ttl = '`+apply.TTLEphemeral1h+`'`, time.Millisecond) + ` >= ` + alias + `ts)` + } + // A row the accountant library priced carries a credit the library gave from usage: it credits a rescue + // when the ping RE-CREATED an expired entry and the next request read what the ping wrote (accountant + // case keepalive/ping-miss-restarts-clock/5m). So for such a row a ping that wrote reached the entry as + // well as one that read it, and the test is cache_read > 0 OR cache_write > 0. The strict test above + // stays for every other row: the legacy write path credited a row on keepalive_pings alone, and a ping + // that only re-created the entry earned it a credit it did not deserve (context-guru#768 B4). + // + // The order is still load-bearing: col > 0 first, so a probe runs once per CREDITED row. The library test + // is a second EXISTS inside the same AND, and it runs only for a row the strict probe refused. + libraryPriced := `EXISTS (SELECT 1 FROM request_components c WHERE c.request_id = ` + alias + `id AND c.priced_by = '` + + PricedByAccountant + `')` + return `(CASE WHEN ` + col + ` > 0 AND (` + probe(`kp.cache_read > 0`) + ` OR (` + libraryPriced + ` AND ` + + probe(`kp.cache_write > 0`) + `)) THEN ` + col + ` ELSE 0 END)` } // KeepAliveCoverage is the "is this figure a measurement or an absence?" statement that diff --git a/dash/keepalivetab_test.go b/dash/keepalivetab_test.go index 9cd29f8f..f333eff2 100644 --- a/dash/keepalivetab_test.go +++ b/dash/keepalivetab_test.go @@ -1299,3 +1299,37 @@ func TestMedianPingUSDEmptyModelKeepsTheOldWriteMultiple(t *testing.T) { t.Errorf("median ping with no model = %v, want 0.10", got) } } + +// A ping that found the entry gone re-created it: it WROTE, it did not read. The accountant library +// credits the rescue on the next request, because that request read the entry the ping wrote +// (accountant case keepalive/ping-miss-restarts-clock/5m). The reachability gate asked for +// cache_read > 0 on the ping, so it zeroed that credit and kept the ping's cost: the session read +// -$0.01146 where the hand value is -$0.00094, 12x too negative (context-guru#768 B4). +func TestTheGateKeepsTheRescueOfAPingThatWroteTheEntry(t *testing.T) { + rescued := kaCredit(1_100_000, "s1", 0.01045695) + rescued.Components = []CompRow{{Component: "mask", PricedBy: PricedByAccountant}} + fx := newKAFixture(t, + kaAgent(1_000_000, "s1", 0.10), + kaPing(1_050_000, "s1", 0.01145625, 0, 3051), // the entry had expired: the ping wrote it again + rescued, + ) + rows, _, err := fx.db.Sessions(Filter{TenantAll: true}, 50, 0) + if err != nil { + t.Fatal(err) + } + var got *SessionRow + for _, r := range rows { + if r.SessionID == "s1" { + got = r + } + } + if got == nil { + t.Fatal("session s1 is missing from the list") + } + if math.Abs(got.KeepAliveSavedUSD-0.01045695) > 1e-9 { + t.Errorf("saved = %.6f, want 0.010457: the rescue the library credited was zeroed by the gate", got.KeepAliveSavedUSD) + } + if want := 0.01045695 - 0.01145625; math.Abs(got.KeepAliveNetUSD-want) > 1e-9 { + t.Errorf("net = %.6f, want %.6f", got.KeepAliveNetUSD, want) + } +} From 320b2bf43cfe5b5e480c21e105f5e4993295d237 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 14:45:42 +0200 Subject: [PATCH 05/58] fix(dash): Let the replay cap close a credit, never erase a cost DeltaUSD removed every row past the cap. Since the accountant library prices breaks and TTL premiums, a stored saving can be negative, and a negative row past the cap is money that was spent. Removing it raised the net: rows +1, +1, -2 gave a correction of +1.00 (review of #768, B5). A negative row now counts, whether the cap is open or not. Only positive credit past the cap is removed, which is what the cap is for. The review also asked to check creditTurn and creditOpening in compactepisode.go for the same sign problem. They do not have it: for a SavedUSD of -3 both book the cost in FirstRemovalCreditUSD and keep it in the net. It is labelled "first removal credit", which is a poor name for a cost, but the number is right. No change there. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/replaycap.go | 5 ++++- dash/replaycap_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/dash/replaycap.go b/dash/replaycap.go index 99544dff..d318badd 100644 --- a/dash/replaycap.go +++ b/dash/replaycap.go @@ -278,7 +278,10 @@ func walkCapConversation(conv []capRow, window windowFn, compactionTS []int64, o if assumed { l.WindowAssumedUSD += c.USD } - if state == capOpen { + // The cap closes the credit of a replay. It never erases a cost: a break or a TTL premium is money that + // was spent whether or not the cap is open. A negative row counts, so DeltaUSD cannot raise the net + // (context-guru#768 B5: rows +1, +1, -2 gave a correction of +1.00). + if state == capOpen || c.USD < 0 { l.CountedUSD += c.USD l.FirstRemovalUSD += c.FirstUSD l.ReplayUSD += c.USD - c.FirstUSD diff --git a/dash/replaycap_test.go b/dash/replaycap_test.go index 28c37a15..a597a094 100644 --- a/dash/replaycap_test.go +++ b/dash/replaycap_test.go @@ -669,3 +669,38 @@ func TestReplayLedgerThroughSQLReadsAgentCompaction(t *testing.T) { } nearEq(t, "past real", l.ByComponent["searchfold"].PastClientCompact, 1) } + +// The cap closes the credit of a replay. It must never erase a cost. Since the accountant library prices +// breaks and TTL premiums, a stored saving can be negative, and a negative row after the cap is money that was +// spent. Before, DeltaUSD removed every row past the cap, so a loss was taken out of the net and the "correction" +// raised it (context-guru#768 B5: rows +1, +1, -2 gave a correction of +1.00). +// +// The row that reaches the ceiling is already past the cap: the walker closes the cap before it reads that row. +func TestReplayCapNeverErasesACost(t *testing.T) { + rows := []capRow{ + capTestRow(1, 100_000, 50_000, 400_000, "extract_llm", 1), // counted + capTestRow(2, 110_000, 500_000, 500_000, "extract_llm", 1), // cf 1,000k: closes the cap, so this credit is past it + capTestRow(3, 120_000, 500_000, 500_000, "extract_llm", -2), // after the cap: a cost + } + l := walkReplayCap(rows, fixedWindow(1_000_000)) + c := l.ByComponent["extract_llm"] + nearEq(t, "the credit past the cap", c.PastCapUSD, 1) + nearEq(t, "the cost stays in the net", c.CountedUSD, -1) // +1 counted, -2 kept + if d := l.DeltaUSD(); d > 1e-9 { + t.Fatalf("DeltaUSD = %g: a cap must never raise the net", d) + } + nearEq(t, "delta", l.DeltaUSD(), -1) +} + +// A positive row past the cap is still removed: this is the cap's job, and it must not change. +func TestReplayCapStillRemovesAPositiveCreditPastTheCap(t *testing.T) { + rows := []capRow{ + capTestRow(1, 100_000, 50_000, 400_000, "extract_llm", 1), // counted + capTestRow(2, 110_000, 500_000, 500_000, "extract_llm", 1), // closes the cap: credit, removed + capTestRow(3, 120_000, 500_000, 500_000, "extract_llm", 3), // after the cap: credit, removed + capTestRow(4, 130_000, 500_000, 500_000, "extract_llm", -2), // after the cap: a cost, kept + } + c := walkReplayCap(rows, fixedWindow(1_000_000)).ByComponent["extract_llm"] + nearEq(t, "past cap", c.PastCapUSD, 4) + nearEq(t, "counted", c.CountedUSD, -1) // +1 counted, -2 kept +} From 989aaa0a20b56bd5415c488645e53717df70a4d3 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 14:50:49 +0200 Subject: [PATCH 06/58] fix(dash): Count a component's costs in its first-removal verdict NetUSDFirstRemoval is documented as what the proxy's /stats reports. /stats reports saved_usd, which nets the costs a component caused: added tokens and the breaks it left behind. The verdict was first removal minus the component's own LLM spend, with no costs term, so a component that loses money read as a gain: saved_usd -$0.0302 beside +$0.0113 (review of #768, B6). The same function already added the costs to SavedUSDDecomposed, so the two totals disagreed. Both the general path and the replay-cap override now include them, and the Components tooltip says so. The summarize override is left as it is. It sets the split from its own ledger and has no costs term; I could not show that SavedUSDCosts is the right number there. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accounting_test.go | 30 ++++++++++++++++++++++++++++++ dash/readvalue.go | 5 ++++- dash/replaycap.go | 5 +++-- dash/ui/app.js | 5 +++-- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/dash/accounting_test.go b/dash/accounting_test.go index 435f1baf..a390f33f 100644 --- a/dash/accounting_test.go +++ b/dash/accounting_test.go @@ -1061,3 +1061,33 @@ func TestDecompositionSumsTheAccountantSplitAsStored(t *testing.T) { t.Fatalf("an accountant row was estimated again: %v", c.SavedUSDEstimated) } } + +// NetUSDFirstRemoval is documented as what the proxy's /stats reports, and /stats reports saved_usd, which nets +// the costs a component caused (added tokens, breaks). Leaving them out showed a component that loses money as a +// gain: saved_usd -$0.0302 beside net_usd_first_removal +$0.0113 (context-guru#768 B6). +func TestTheFirstRemovalVerdictIncludesTheCostsAComponentCaused(t *testing.T) { + db := openTestDB(t) + // First removal +0.0100, replay +0.0050, and a break costs -0.0600: saved_usd is -0.0450. + e := &Event{TS: 1000, SessionID: "s1", Model: "m1", TenantID: "t1", + TokensBefore: 20000, TokensAfter: 19000, FreshInput: 5, CacheRead: 40000, OutputTokens: 50, + TokenAccounting: AccountingComplete, + Components: []CompRow{{Component: "extract", Acted: true, Mutated: true, SavedGross: 1000, SavedUnique: 1000, + SavedUSD: -0.0450, SavedFirstUSD: 0.0100, SavedReplayUSD: 0.0050, PricedBy: PricedByAccountant}}} + insertReq(t, db, e) + rows, err := db.Components(Filter{TenantAll: true}) + if err != nil { + t.Fatal(err) + } + if err := db.DecomposeComponentSavedUSD(Filter{TenantAll: true}, handPrice, rows); err != nil { + t.Fatal(err) + } + c := rows[0] + nearEq(t, "costs", c.SavedUSDCosts, -0.0600) + nearEq(t, "first removal", c.SavedUSDFirstRemoval, 0.0100) + if c.NetUSDFirstRemoval >= 0 { + t.Fatalf("net_usd_first_removal = %g for a component whose saved_usd is %g: a loss must not read as a gain", + c.NetUSDFirstRemoval, c.SavedUSD) + } + // 0.0100 first removal - 0.0600 costs, and the component has no LLM spend of its own. + nearEq(t, "verdict", c.NetUSDFirstRemoval, -0.0500) +} diff --git a/dash/readvalue.go b/dash/readvalue.go index 16aed16c..c4e23c03 100644 --- a/dash/readvalue.go +++ b/dash/readvalue.go @@ -289,7 +289,10 @@ func decomposeFromGroups(cgroups map[compGroupKey]*compGroup, p modelinfo.Pricer } for _, c := range out { c.SavedUSDDecomposed = c.SavedUSDFirstRemoval + c.SavedUSDReplay + c.SavedUSDCosts - c.NetUSDFirstRemoval = c.SavedUSDFirstRemoval - c.LLMCostUSD + // The costs the accountant library netted into saved_usd (added tokens, breaks) are part of the verdict. + // /stats reports saved_usd, and this field is documented as what /stats reports: leaving them out showed a + // component that loses money as a gain (context-guru#768 B6: saved_usd -$0.0302, this field +$0.0113). + c.NetUSDFirstRemoval = c.SavedUSDFirstRemoval + c.SavedUSDCosts - c.LLMCostUSD if c.SavedUSDFirstRemoval > 0 { c.ReplayMultiple = (c.SavedUSDFirstRemoval + c.SavedUSDReplay) / c.SavedUSDFirstRemoval } diff --git a/dash/replaycap.go b/dash/replaycap.go index d318badd..a08f907a 100644 --- a/dash/replaycap.go +++ b/dash/replaycap.go @@ -490,9 +490,10 @@ func applyReplayLedgerToRow(c *ComponentRow, l *ReplayLedger) { pastReplay := cl.PastCapUSD - cl.PastCapFirstUSD c.SavedUSDFirstRemoval = math.Max(c.SavedUSDFirstRemoval-cl.PastCapFirstUSD, 0) c.SavedUSDReplay = math.Max(c.SavedUSDReplay-pastReplay, 0) - c.SavedUSDDecomposed = c.SavedUSDFirstRemoval + c.SavedUSDReplay + // The costs term (SavedUSDCosts) stays: the cap removes credit, never a cost (see walkCapConversation). + c.SavedUSDDecomposed = c.SavedUSDFirstRemoval + c.SavedUSDReplay + c.SavedUSDCosts c.SavedUSDDecomposedStored = math.Max(c.SavedUSDDecomposedStored+delta, 0) - c.NetUSDFirstRemoval = c.SavedUSDFirstRemoval - c.LLMCostUSD + c.NetUSDFirstRemoval = c.SavedUSDFirstRemoval + c.SavedUSDCosts - c.LLMCostUSD c.ReplayMultiple = 0 if c.SavedUSDFirstRemoval > 0 { c.ReplayMultiple = c.SavedUSDDecomposed / c.SavedUSDFirstRemoval diff --git a/dash/ui/app.js b/dash/ui/app.js index f7f2724e..792edc4c 100644 --- a/dash/ui/app.js +++ b/dash/ui/app.js @@ -2701,8 +2701,9 @@ async function loadComponents() { class: 'num' + (savedKnown(c) && c.net_usd_first_removal < 0 ? ' warn-text' : ''), title: !savedKnown(c) ? '' : 'PER-TURN verdict: each piece of content credited ONCE at ' + 'the rate it would have entered the prompt at (' - + usd(c.saved_usd_first_removal || 0) + ') − own LLM cost ' - + usd(c.llm_cost_usd || 0) + '. This is what the proxy\'s /stats endpoint reports.', + + usd(c.saved_usd_first_removal || 0) + ')' + + (c.saved_usd_costs ? ' + the costs the component caused (breaks, added tokens) ' + usd(c.saved_usd_costs) : '') + + ' − own LLM cost ' + usd(c.llm_cost_usd || 0) + '. This is what the proxy\'s /stats endpoint reports.', }, c.saved_gross ? usdOrNA(c.net_usd_first_removal, savedKnown(c), 'the saving half of this net is not a number — ' + savedNAReason(c)) From 988bc4d98301bff49ab7ddfdbfbf7cca93f025ca Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 14:57:30 +0200 Subject: [PATCH 07/58] fix(proxy): Say why the Responses wire books no saving The OpenAI Responses API keeps its conversation in input. accBodies hashes messages and apply reports no operations on that wire, so the accountant priced nothing there: the conservation check failed and every component read priced_by = none with acted_without_figure. A reviewer found that all Responses savings dropped to $0 and the PR text did not say so (review of #768, B7). This is a guard, not support. A request on the wire opens no record, the event is marked, and PriceMode books no saving with the reason unsupported_wire. Chat Completions and Messages are unchanged. Support itself means hashing input, instructions and tools, reporting operations from the Responses write path, and an OpenAI profile in the library (accountant#10 and #17). That is a feature, not a fix, and it cannot be checked here against a real stream. docs/dashboard.md now says that every saving on this wire is $0, which the claim "every saving comes from the accountant" had left out. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accountant_event.go | 6 +++++ dash/accountant_price.go | 3 +++ dash/event.go | 2 ++ docs/dashboard.md | 9 +++++-- proxy/accountant_wire_test.go | 46 +++++++++++++++++++++++++++++++++++ proxy/dashcapture.go | 7 +++++- proxy/proxy.go | 8 +++++- 7 files changed, 77 insertions(+), 4 deletions(-) diff --git a/dash/accountant_event.go b/dash/accountant_event.go index 8eab47ea..8ef62556 100644 --- a/dash/accountant_event.go +++ b/dash/accountant_event.go @@ -20,6 +20,12 @@ func (e *Event) AccountantResult() (accountant.Result, bool) { return *e.accRes, true } +// SetAccountantUnsupportedWire marks the request as one on a wire the accountant library does not read yet: +// the OpenAI Responses API keeps its conversation in `input`, and the proxy hashes `messages` and reports no +// operations there. PriceMode then books no saving and says why (fallback unsupported_wire), where it would +// have said no_result, which reads as a request that was merely skipped (context-guru#768 B7). +func (e *Event) SetAccountantUnsupportedWire() { e.accUnsupportedWire = true } + // SetAccountantContext records what qualifies the attached Result: how many of the request's // changes the trace could not place as operations, and whether the run was observe mode. Both // gate the accountant override in PriceMode. diff --git a/dash/accountant_price.go b/dash/accountant_price.go index 6bf17b23..4b64d455 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -140,6 +140,7 @@ const ( FallbackFigureWithoutRow = "figure_without_row" // a library figure for a component with no row FallbackDuplicateName = "duplicate_name" // two rows share a component name FallbackActedWithoutFigure = "acted_without_figure" // a component acted and the library has no figure for it + FallbackUnsupportedWire = "unsupported_wire" // the request is on a wire the accountant does not read yet (OpenAI Responses) ) // gatedChecks maps each library check (accountant internal/engine/compute.go checks() and conservation()) that @@ -229,6 +230,8 @@ func (e *Event) PriceMode(p modelinfo.Price, accountingComplete bool) { // accOverride applies the accountant override, or returns the reason it cannot and changes nothing. func (e *Event) accOverride(p modelinfo.Price) string { switch { + case e.accUnsupportedWire: + return FallbackUnsupportedWire // before no_result: say the real reason, not that nothing was attached case e.accRes == nil: return FallbackNoResult case e.accOpsDropped != 0: diff --git a/dash/event.go b/dash/event.go index c028cc87..64da6617 100644 --- a/dash/event.go +++ b/dash/event.go @@ -125,6 +125,8 @@ type Event struct { // accOpsDropped and accObserve qualify accRes (see PriceMode). Unexported: not serialized. accOpsDropped int accObserve bool + // accUnsupportedWire marks a request on a wire the accountant does not read (see SetAccountantUnsupportedWire). + accUnsupportedWire bool // accFallback is why PriceMode booked no saving (see AccountantFallback). accFallback string diff --git a/docs/dashboard.md b/docs/dashboard.md index 1c965025..f3df7728 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -123,8 +123,13 @@ What the library prices: When the library cannot price an event, the event books **no saving**, and the proxy logs one `acc_fallback` line with the reason: `no_result`, `ops_dropped`, `observe`, `unpriced`, `check_sum`, `check_unique_change`, `check_spans`, `size_unknown`, `implausible_size`, -`figure_without_row`, `duplicate_name` or `acted_without_figure`. A saving nothing measured is not -shown. +`figure_without_row`, `duplicate_name`, `acted_without_figure` or `unsupported_wire`. A saving nothing +measured is not shown. + +`unsupported_wire` is the OpenAI Responses API. It keeps the conversation in `input`, and the proxy hashes +`messages` and reports no operations there, so **every saving on that wire is $0**. Chat Completions and the +Anthropic Messages API are priced. Until Responses is read, a Responses account sees no saving and the +keep-alive tab on it can read negative: the pings still cost, and no rescue is credited. Limits: diff --git a/proxy/accountant_wire_test.go b/proxy/accountant_wire_test.go index 0979e91d..76f39b29 100644 --- a/proxy/accountant_wire_test.go +++ b/proxy/accountant_wire_test.go @@ -236,3 +236,49 @@ func TestNoDashboardBuildsNoTracker(t *testing.T) { t.Error("a handler with a dashboard must build the tracker") } } + +// The OpenAI Responses API keeps its conversation in `input`. accBodies hashes `messages` and apply reports no +// operations on that wire, so the accountant prices nothing there: the library's conservation check failed and +// every component read priced_by=none with a bare no_result or acted_without_figure. A request on it is now +// marked, and says why it has no saving (context-guru#768 B7). Chat Completions is unaffected. +func TestResponsesWireBooksNoSavingAndSaysWhy(t *testing.T) { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"resp_1","object":"response","status":"completed", + "output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}], + "usage":{"input_tokens":2000,"output_tokens":20,"input_tokens_details":{"cached_tokens":1500}}}`) + })) + defer up.Close() + h, rec, tap := accHandler(t, up.URL) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + big := strings.Repeat("line of tool output\n", 400) + for i := 0; i < 2; i++ { + body := fmt.Sprintf(`{"model":"gpt-5.5","instructions":"agent","input":[`+ + `{"role":"user","content":"run it"},`+ + `{"type":"function_call","call_id":"c1","name":"bash","arguments":"{}"},`+ + `{"type":"function_call_output","call_id":"c1","output":%q}]}`, big) + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/openai/v1/responses", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer sk-test") + req.Header.Set("x-context-guru-session", "sess-resp") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + } + waitForRows(t, rec, 2) + for i, e := range tap.all() { + if fb := e.AccountantFallback(); fb != dash.FallbackUnsupportedWire { + t.Errorf("request %d: fallback %q, want %q", i+1, fb, dash.FallbackUnsupportedWire) + } + if _, ok := e.AccountantResult(); ok { + t.Errorf("request %d: a Responses request got a library Result", i+1) + } + if saved := e.BaselineCostUSD - e.CostUSD; saved > 1e-12 { + t.Errorf("request %d: booked a saving of %g on a wire the accountant does not read", i+1, saved) + } + } +} diff --git a/proxy/dashcapture.go b/proxy/dashcapture.go index d9487fbb..caa3b71a 100644 --- a/proxy/dashcapture.go +++ b/proxy/dashcapture.go @@ -128,6 +128,8 @@ type capture struct { // pings holds the keep-alive ping facts the next request consumes (Options.Pings; nil = none). pings *PingStore accReq *accountant.Request + // accUnsupportedWire is set for a request on a wire the accountant does not read (OpenAI Responses). + accUnsupportedWire bool // accClient is the body the agent sent, kept for accEnd's envelope diff (envelopeOps). accClient []byte // accThread is the thread id accEnd gave the library, the second half of the ping store's key. @@ -729,8 +731,11 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content } } e.SetAccountantContext(c.trace.OpsDropped, c.observe) + if c.accUnsupportedWire { + e.SetAccountantUnsupportedWire() + } e.PriceMode(price, usageOK && priced) - if c.accReq != nil && e.AccountantFallback() != "" { + if (c.accReq != nil || c.accUnsupportedWire) && e.AccountantFallback() != "" { logFallback(e) } if cgIn > 0 || cgOut > 0 || cgCacheWrite > 0 || cgCacheRead > 0 { diff --git a/proxy/proxy.go b/proxy/proxy.go index 59ad64c4..d5c2ffe6 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -1423,7 +1423,13 @@ func (h *Handler) chat(provider bschemas.ModelProvider, static upstream, pick fu // the client body, so opening the record late loses nothing. `orig` is what the agent // sent. An empty session means the pipeline did not run (bypassed, observe, no // messages): there are no operations to price, so the request is skipped. - if cp != nil && cp.savings != nil && tr.Session != "" { + // + // Not on the OpenAI Responses wire: it keeps the conversation in `input`, accBodies hashes + // `messages`, and apply reports no operations there. The record would price nothing and fail the + // library's sum check. The event is marked instead, so the dashboard says why it has no saving. + if cp != nil && cp.savings != nil && tr.Session != "" && schema.IsResponses(wire) { + cp.accUnsupportedWire = true + } else if cp != nil && cp.savings != nil && tr.Session != "" { cp.accBegin(accountant.RequestMeta{ RequestID: "r" + strconv.FormatUint(h.accSeq.Add(1), 10), Session: tr.Session, From 16e8f2f9be038d5ac7651cdc8d7ed684f13e4ad2 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 15:04:39 +0200 Subject: [PATCH 08/58] fix(dash): Show a component whose requests all fell back as unknown, not $0 A request the accountant could not price books no saving. Its components carry priced_by = 'none' and a stored saved_usd of 0, and they were counted nowhere: saved_usd_unpriced_rows stayed 0, so the Components tab printed a confident $0 for a component that had removed 5,000 tokens (review of #768, B8). A row of priced_by 'none' that removed content now counts as unpriced, so the column reads n/a. A row that removed nothing does not, and a row the library priced at $0 is still a measured $0. The unpriced text blamed the operator's price list every time. For a fallback that advice is wrong: the model may be priced, and setting MODEL_PRICES changes nothing. The three messages now give both causes and point at acc_fallback in the proxy log. Not done: the review also asked for a fallback_requests count on Overview. It needs another query on both the full and the lean path, which TestLeanStatsMatchFullForTheSameWindow and the 20,000-request budget test pin, and it adds visibility, not correctness. The per-component column and the acc_fallback log line already say it. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accounting_test.go | 56 +++++++++++++++++++++++++++++++++++++++++ dash/query.go | 10 +++++--- dash/readvalue.go | 7 ++++++ dash/ui/app.js | 13 ++++++---- 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/dash/accounting_test.go b/dash/accounting_test.go index a390f33f..cb2f606b 100644 --- a/dash/accounting_test.go +++ b/dash/accounting_test.go @@ -1091,3 +1091,59 @@ func TestTheFirstRemovalVerdictIncludesTheCostsAComponentCaused(t *testing.T) { // 0.0100 first removal - 0.0600 costs, and the component has no LLM spend of its own. nearEq(t, "verdict", c.NetUSDFirstRemoval, -0.0500) } + +// A request the accountant could not price books no saving: its components carry priced_by='none' and a stored +// saved_usd of 0. The component still removed content there, so the saving is unknown, not zero. It was counted +// nowhere: saved_usd_unpriced_rows stayed 0 and the column printed a confident $0 beside "accounting complete" +// (context-guru#768 B8: 5 requests, 5,000 tokens removed, all priced_by none). +func TestAComponentWhoseRequestsAllFellBackIsUnpricedNotZero(t *testing.T) { + db := openTestDB(t) + for i := 0; i < 5; i++ { + e := &Event{TS: int64(1000 + i), SessionID: "s1", Model: "m1", TenantID: "t1", + TokensBefore: 20000, TokensAfter: 19000, FreshInput: 5, CacheRead: 40000, OutputTokens: 50, + TokenAccounting: AccountingComplete, + Components: []CompRow{{Component: "extract", Acted: true, Mutated: true, SavedGross: 1000, SavedUnique: 1000, + PricedBy: PricedByNone}}} + insertReq(t, db, e) + } + rows, err := db.Components(Filter{TenantAll: true}) + if err != nil { + t.Fatal(err) + } + if err := db.EstimateComponentSavedUSD(Filter{TenantAll: true}, handPrice, rows); err != nil { + t.Fatal(err) + } + c := rows[0] + if c.SavedUSD != 0 { + t.Fatalf("saved_usd = %g, want the stored 0", c.SavedUSD) + } + if c.SavedUSDUnpricedRows != 5 { + t.Fatalf("saved_usd_unpriced_rows = %d, want 5: the component removed 5,000 tokens and its saving is unknown, not $0", + c.SavedUSDUnpricedRows) + } +} + +// A row the library priced at $0 is a measured $0, and a row that removed nothing is not unknown either. +func TestOnlyAFallbackRowThatRemovedContentCountsAsUnpriced(t *testing.T) { + db := openTestDB(t) + mk := func(ts int64, gross int, pricedBy string) *Event { + return &Event{TS: ts, SessionID: "s1", Model: "m1", TenantID: "t1", + TokensBefore: 20000, TokensAfter: 19000, FreshInput: 5, CacheRead: 40000, OutputTokens: 50, + TokenAccounting: AccountingComplete, + Components: []CompRow{{Component: "extract", Acted: true, Mutated: true, SavedGross: gross, SavedUnique: gross, + PricedBy: pricedBy}}} + } + insertReq(t, db, mk(1000, 1000, PricedByAccountant)) // a measured $0 + insertReq(t, db, mk(1001, 0, PricedByNone)) // fell back, removed nothing + insertReq(t, db, mk(1002, 1000, PricedByNone)) // fell back, removed content: unknown + rows, err := db.Components(Filter{TenantAll: true}) + if err != nil { + t.Fatal(err) + } + if err := db.EstimateComponentSavedUSD(Filter{TenantAll: true}, handPrice, rows); err != nil { + t.Fatal(err) + } + if n := rows[0].SavedUSDUnpricedRows; n != 1 { + t.Fatalf("saved_usd_unpriced_rows = %d, want 1 (only the fallback row that removed content)", n) + } +} diff --git a/dash/query.go b/dash/query.go index 18987888..1650a159 100644 --- a/dash/query.go +++ b/dash/query.go @@ -1154,6 +1154,10 @@ type compGroupKey struct { // acc is a group of rows the accountant library priced (priced_by): their dollar split is // stored (firstUSD, replayUSD), and no read-time formula re-prices them. acc bool + // fallback is a group of rows of a request the accountant path handled and could not price + // (priced_by 'none'). They are also acc: no read-time formula values them. They book no saving, and the + // reader must be told that, because a stored 0 here is not a measured 0 (context-guru#768 B8). + fallback bool } // compGroup is one cell of that pass. uClamp and rClamp are the decomposition's unique and replay @@ -1204,14 +1208,14 @@ func (d *DB) componentGroups(cond string, args []any) (map[compGroupKey]*compGro CASE WHEN r.cache_read > 0 THEN 'read' WHEN r.cache_write > 0 AND r.cache_write >= r.fresh_input THEN 'write' ELSE 'fresh' END, - c.saved_usd <> 0, c.saved_gross > 0, r.token_accounting = 'complete', c.priced_by <> '', + c.saved_usd <> 0, c.saved_gross > 0, r.token_accounting = 'complete', c.priced_by <> '', c.priced_by = 'none', MAX(c.kind), COUNT(*), SUM(c.acted), SUM(c.mutated), SUM(c.reverted), SUM(c.skipped), SUM(c.saved_gross), SUM(c.saved_unique), SUM(c.saved_usd), SUM(c.duration_ms), SUM(CASE WHEN c.err <> '' THEN 1 ELSE 0 END), SUM(`+uniq+`), SUM(`+gross+` - `+uniq+`), SUM(CASE WHEN c.saved_gross <> c.saved_unique THEN 1 ELSE 0 END), SUM(c.saved_first_usd), SUM(c.saved_replay_usd) FROM request_components c JOIN requests r ON r.id = c.request_id - WHERE `+cond+pred+` GROUP BY 1, 2, 3, 4, 5, 6, 7`, + WHERE `+cond+pred+` GROUP BY 1, 2, 3, 4, 5, 6, 7, 8`, append(append([]any(nil), args...), bounds...)...) if err != nil { return err @@ -1221,7 +1225,7 @@ func (d *DB) componentGroups(cond string, args []any) (map[compGroupKey]*compGro var k compGroupKey var g compGroup var usd sql.NullFloat64 - if err := rows.Scan(&k.comp, &k.model, &k.tier, &k.usd, &k.gross, &k.complete, &k.acc, + if err := rows.Scan(&k.comp, &k.model, &k.tier, &k.usd, &k.gross, &k.complete, &k.acc, &k.fallback, &g.kind, &g.runs, &g.acted, &g.mutated, &g.reverted, &g.skipped, &g.gross, &g.unique, &usd, &g.dur, &g.errs, &g.uClamp, &g.rClamp, &g.nDiff, &g.firstUSD, &g.replayUSD); err != nil { return err diff --git a/dash/readvalue.go b/dash/readvalue.go index c4e23c03..1b5fc942 100644 --- a/dash/readvalue.go +++ b/dash/readvalue.go @@ -375,6 +375,13 @@ func estimateFromGroups(cgroups map[compGroupKey]*compGroup, p modelinfo.Pricer, // cannot say" and "it was worth nothing" are different answers. unpriced := map[string]int64{} for k, g := range cgroups { + // A request the accountant could not price (priced_by 'none') books no saving. The component still + // removed content on it, so the saving is not zero, it is unknown. Counted with the rows that could not + // be valued, so the column reads n/a and not a confident $0 (context-guru#768 B8). + if k.fallback && k.gross { + unpriced[k.comp] += g.runs + continue + } // A row the accountant priced is never estimated: its saved_usd is a figure, also when 0. if k.acc || k.usd || !k.gross { continue diff --git a/dash/ui/app.js b/dash/ui/app.js index 792edc4c..c3fe7aa0 100644 --- a/dash/ui/app.js +++ b/dash/ui/app.js @@ -2498,7 +2498,8 @@ function savedNAReason(c) { + 'recorded per request instead, as the prefix-cache figure on Overview.'; } return num(c.saved_usd_unpriced_rows) + ' of this component\u2019s acted rows ran on a model ' - + 'with no entry in the operator\u2019s price list, so their saving could not be valued. ' + + 'with no entry in the operator\u2019s price list, or on a request the accountant could not price ' + + '(see acc_fallback in the proxy log), so their saving could not be valued. ' + 'It is missing from this figure, not zero.'; } @@ -2517,8 +2518,9 @@ function savedTitle(c) { : usd(c.saved_usd) + ' recorded per request']; if (c.saved_usd_unpriced_rows) { parts.push(num(c.saved_usd_unpriced_rows) + ' row' - + (c.saved_usd_unpriced_rows === 1 ? '' : 's') + ' could not be priced at all — no model ' - + 'price for the model they ran on, so their saving is MISSING from this figure, not zero'); + + (c.saved_usd_unpriced_rows === 1 ? '' : 's') + ' could not be priced at all — no price for the model they ' + + 'ran on, or the request fell back (see acc_fallback in the proxy log) — so their saving is MISSING from ' + + 'this figure, not zero'); } return parts.join(' · '); } @@ -2554,8 +2556,9 @@ function renderUnpricedNote(components) { host.appendChild(el('div', {}, el('strong', {}, num(rows) + ' acted rows across ' + unpriced.length + ' component' + (unpriced.length === 1 ? '' : 's') + ' could not be priced'), - ' — the model they ran on has no entry in the operator’s price list, so their saving is ' - + 'absent from the dollar columns rather than zero. Set MODEL_PRICES to value them.')); + ' — the model they ran on has no entry in the operator’s price list (set MODEL_PRICES to value them), or the ' + + 'accountant could not price the request (the proxy logs acc_fallback with the reason). Either way their ' + + 'saving is absent from the dollar columns rather than zero.')); if (partial.length) { host.appendChild(el('div', {}, 'Partly unpriced, so the dollar figure shown is a ', el('strong', {}, 'lower bound'), From 7cd2ba3869e3024504c7e233aa4cb380ace4c037 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 15:08:33 +0200 Subject: [PATCH 09/58] build: Pin accountant to the fix branch for issues 9, 14, 20 and 21 Takes accountant 38a32264 (PR 23's branch) for four fixes the review of #768 needed in the library: the rescue no longer depends on what the request wrote (#9), the ping extent ignores the ping's fresh input (#14), a split is credited only when the system section changed (#20), and B* follows the usage when the provider read the whole previous extent (#21). THIS COMMIT IS NOT ON accountant main. PR 22 (the cases) is still open, and PR 23 merged into 22's branch, not into main. The pseudo-version resolves because Go reads the commit from the repository, but it names a commit that a squash or a force-push would drop. Re-pin to the merge commit on main before this branch merges. All dash, proxy, apply, keepalive and components tests pass on it, including dash/accountant_play_test.go, which plays the library's shared cases through this repo's pricing. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- go.mod | 2 +- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 2fe77349..a0e12ca2 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.26.4 require github.com/maximhq/bifrost/core v1.7.0 // accountant prices context operations. This module's tests import it and its shared test cases. -require github.com/rossoctl/accountant v0.1.1-0.20261103073505-583b9f916ea2 +require github.com/rossoctl/accountant v0.1.1-0.20261104130101-38a32264c26b require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 diff --git a/go.sum b/go.sum index 62f104aa..dee7dc9b 100644 --- a/go.sum +++ b/go.sum @@ -100,12 +100,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rossoctl/accountant v0.1.1-0.20261103052133-c5f558098d3d h1:uZjLQLg2JAm6JsN1sBhzjPaH6Q0yyNjmkJe2vk7BIzQ= -github.com/rossoctl/accountant v0.1.1-0.20261103052133-c5f558098d3d/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= -github.com/rossoctl/accountant v0.1.1-0.20261103072919-fc9dde70e622 h1:OQrj6zCScMDgYMK7Lza5nQeHXewAwVc+EUOiPPzoXcs= -github.com/rossoctl/accountant v0.1.1-0.20261103072919-fc9dde70e622/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= -github.com/rossoctl/accountant v0.1.1-0.20261103073505-583b9f916ea2 h1:ZM6txJq6SoYNWeJHI4syFTHLPhkxSQebCnZLSm7Lkvg= -github.com/rossoctl/accountant v0.1.1-0.20261103073505-583b9f916ea2/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261104130101-38a32264c26b h1:Y84ZyAxLrNZWUhUVsUHiNmx1R7Vqw3HVMyGyjW0Nat8= +github.com/rossoctl/accountant v0.1.1-0.20261104130101-38a32264c26b/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From d1b857a6dc1c4053d98f149fea2c7ca48db3c73b Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 15:10:36 +0200 Subject: [PATCH 10/58] test(proxy): Cover the cachesplit credit of an unchanged system block The review of #768 (B9) found $350.93 of cachesplit_saved_usd on $29.96 of cost in a replay of four Claude Code sessions, where the system snapshot never changed and the true saving was $0. Two faults made it: the body hash included the moving cache mark (B1), and the library credited a split with no check that the system section changed (accountant#20). Both are fixed on this branch. The test drives the replay's shape through the tracker: a split every turn, an unchanged system block, a mark that moves. It books a split effect in the state #768 merged in (103 tokens on turn 3) and none on this branch. I checked each fault alone: with only the old hash, or only the old library, the test passes. Either fix is enough to keep the credit out of this scenario, so the test guards the combination. The comment says so. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/accountant_savings_test.go | 59 ++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/proxy/accountant_savings_test.go b/proxy/accountant_savings_test.go index 95d05cb5..e873d8c4 100644 --- a/proxy/accountant_savings_test.go +++ b/proxy/accountant_savings_test.go @@ -260,3 +260,62 @@ func TestSavings_AMovingCacheMarkDoesNotRepriceAReplayAtTheWriteRate(t *testing. t.Fatalf("turn 3 priced the replay at %+v, want the read tier (the provider read it)", got.Saved) } } + +// context-guru#768 B9: the cachesplit credit was the whole cached read. A replay of four Claude Code sessions booked +// $350.93 of it on $29.96 of cost, although the system snapshot never changed and the true saving was $0. Two +// faults made that: the hash included the moving cache mark (B1), so the library lost B*, and the library credited a +// split with no check that the system section changed (accountant#20). Both are fixed. This drives the shape of the +// replay through the real tracker: a split every turn, an unchanged system block, a mark that moves. +// +// It fails only with BOTH faults present (the hash with the mark and the library before #20/#21): either fix alone +// is enough to keep the split effect out of this scenario. So it guards the combination, not each cause. The causes +// have their own tests: TestAccBodies_* for the hash, and the accountant cases split/tail-unchanged-* for the library. +func TestSavings_ASplitOfAnUnchangedSystemBlockBooksNothingWhenTheMarkMoves(t *testing.T) { + c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} + const sys = "stable instructions. " + vol := "gitStatus: clean" + clientSys := func() string { + return fmt.Sprintf(`"system":[{"type":"text","text":%q,"cache_control":{"type":"ephemeral"}}]`, strings.Repeat(sys, 80)+vol) + } + fwdSys := func() string { // cachesplit puts the stable half first, with the mark, and the volatile tail after + return fmt.Sprintf(`"system":[{"type":"text","text":%q,"cache_control":{"type":"ephemeral"}},{"type":"text","text":%q}]`, + strings.Repeat(sys, 80), vol) + } + turn := func(n int, withMark func(i int) bool, sysJSON string) []byte { + var msgs []string + for i := 0; i < n; i++ { + text := fmt.Sprintf("message %d %s", i, strings.Repeat("x", 300)) + mark := "" + if withMark(i) { + mark = `,"cache_control":{"type":"ephemeral"}` + } + role := "user" + if i%2 == 1 { + role = "assistant" + } + msgs = append(msgs, fmt.Sprintf(`{"role":%q,"content":[{"type":"text","text":%q%s}]}`, role, text, mark)) + } + return []byte(`{` + sysJSON + `,"messages":[` + strings.Join(msgs, ",") + `]}`) + } + split := accountant.Operation{Kind: accountant.KindRewrite, Operator: "context-guru.cachesplit", ChangeID: "cg:cachesplit:b9", + At: accountant.Position{Section: accountant.SectionSystem, Index: 0}, + SizeBefore: accountant.Size{Bytes: len(clientSys())}, SizeAfter: accountant.Size{Bytes: len(clientSys())}} + var prevTokens int64 + for n := 2; n <= 6; n++ { + last := n - 1 + client := turn(n, func(i int) bool { return i == last }, clientSys()) // the mark is on the newest message + fwd := turn(n, func(i int) bool { return i == last }, fwdSys()) + tokens := int64((len(fwd) + 3) / 4) + u := Usage{CacheRead: prevTokens, CacheWrite: tokens - prevTokens} + if n == 2 { + u = Usage{CacheWrite: tokens} + } + res := c.do(n, client, fwd, u, split) + prevTokens = tokens + for _, e := range res.CacheEffects { + if e.Kind == accountant.EffectSplit { + t.Fatalf("turn %d: a split effect %+v on a system block that did not change", n, e.Saved) + } + } + } +} From 7ac20acc6133f5f2ca3fee5ba620d457b9483460 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 15:12:02 +0200 Subject: [PATCH 11/58] fix(proxy): Recover a panic in the accountant's Compute The library asks the host to wrap Compute, and this repo's rule is to fail open. The call ran in finish, outside the function the pre-forward recover covers, so a panic would have lost the request's cg.request log line and its dashboard row (review of #768, major 1). The reviewer could not make the library panic, so this is a missing guard, not a fix for a seen failure. computeSafe wraps it. On a panic the request is recorded with no result and PriceMode falls back with no_result. The panic is logged at error level. Tests keep calling compute, so a panic still fails them. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/accountant_savings.go | 16 ++++++++++++++++ proxy/accountant_savings_test.go | 21 +++++++++++++++++++++ proxy/dashcapture.go | 4 +++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/proxy/accountant_savings.go b/proxy/accountant_savings.go index 87bfe2d1..2630cfba 100644 --- a/proxy/accountant_savings.go +++ b/proxy/accountant_savings.go @@ -1,6 +1,8 @@ package proxy import ( + "log/slog" + "github.com/rossoctl/accountant" ) @@ -39,6 +41,20 @@ func (s *savingsTracker) end(req *accountant.Request, th accountant.Thread, forw } } +// computeSafe is compute for the request path. The library asks the host to wrap Compute, and this repo's rule is +// to fail open: a panic here would unwind through finish and lose the cg.request log line and the dashboard row +// of a request that was already forwarded. It reports ok=false instead, the request is recorded with no saving +// (PriceMode falls back with no_result), and the panic is logged. Tests call compute, so a panic still fails them. +func (s *savingsTracker) computeSafe(req *accountant.Request, u Usage) (res accountant.Result, ok bool) { + defer func() { + if r := recover(); r != nil { + slog.Error("accountant: Compute panicked; the request is recorded with no saving", "panic", r) + res, ok = accountant.Result{}, false + } + }() + return s.compute(req, u), true +} + // compute prices the request. It stores the new ThreadState (so a shadow run still advances it). func (s *savingsTracker) compute(req *accountant.Request, u Usage) accountant.Result { return req.Compute(accountant.Usage{Input: int(u.FreshInput), CacheRead: int(u.CacheRead), diff --git a/proxy/accountant_savings_test.go b/proxy/accountant_savings_test.go index e873d8c4..2b0c421c 100644 --- a/proxy/accountant_savings_test.go +++ b/proxy/accountant_savings_test.go @@ -319,3 +319,24 @@ func TestSavings_ASplitOfAnUnchangedSystemBlockBooksNothingWhenTheMarkMoves(t *t } } } + +// The library asks the host to wrap Compute, and the repo's rule is to fail open (context-guru#768, major 1). +// A panic in Compute used to unwind through finish and lose the request's log line and its dashboard row. +// computeSafe turns it into "no result": the request is recorded with no saving. A nil request is a panic the +// test can cause on purpose; the real library has none that anyone has triggered. +func TestComputeSafeTurnsAPanicIntoNoResult(t *testing.T) { + s := newSavingsTracker() + if _, ok := s.computeSafe(nil, Usage{CacheRead: 10}); ok { + t.Fatal("a panic in Compute was reported as a result") + } + // A real request still prices, through the same wrapper. + c := accRun{s, accountant.Thread{ID: "main"}} + m0 := mustMsg("user", "hi") + req := c.s.begin(accountant.RequestMeta{RequestID: "r1", Session: "s", At: time.Unix(1000, 0), + Billing: accountant.Billing{Mode: accountant.BillingAPI}}, body(m0)) + c.s.end(req, c.th, body(m0), nil, accFacts) + res, ok := c.s.computeSafe(req, Usage{FreshInput: 10, CacheWrite: 100}) + if !ok || res.Phase == "" { + t.Fatalf("a normal request did not price through computeSafe: ok=%v phase=%q", ok, res.Phase) + } +} diff --git a/proxy/dashcapture.go b/proxy/dashcapture.go index caa3b71a..c0b9eebf 100644 --- a/proxy/dashcapture.go +++ b/proxy/dashcapture.go @@ -725,7 +725,9 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content if c.accReq != nil && usageOK && !c.accDone { c.accDone = true if c.rounds <= 1 { - e.SetAccountant(c.savings.compute(c.accReq, usage)) + if res, ok := c.savings.computeSafe(c.accReq, usage); ok { + e.SetAccountant(res) + } // The ping facts were read for this request; now that it is priced they are spent. c.pings.drop(accPingKey(c.accReq.Meta.Session, c.accThread), c.accPingWindow) } From c4f21a402014bd62cbb9f82efaa1eba4a24f3059 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 17:36:57 +0200 Subject: [PATCH 12/58] build: Pin accountant to main with the merged fixes for issues 9, 14, 20 and 21 Replaces the pin on the unmerged fix branch (38a3226) with main at d3b2135. This is the pin the PR needed before it could merge. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a0e12ca2..8fbaccdd 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.26.4 require github.com/maximhq/bifrost/core v1.7.0 // accountant prices context operations. This module's tests import it and its shared test cases. -require github.com/rossoctl/accountant v0.1.1-0.20261104130101-38a32264c26b +require github.com/rossoctl/accountant v0.1.1-0.20261104152152-d3b21359717a require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 diff --git a/go.sum b/go.sum index dee7dc9b..939cad01 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rossoctl/accountant v0.1.1-0.20261104130101-38a32264c26b h1:Y84ZyAxLrNZWUhUVsUHiNmx1R7Vqw3HVMyGyjW0Nat8= -github.com/rossoctl/accountant v0.1.1-0.20261104130101-38a32264c26b/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261104152152-d3b21359717a h1:EkB2CqzV7Naeqfowf/A4mwZWZMATiuylWPHpJFrE8OI= +github.com/rossoctl/accountant v0.1.1-0.20261104152152-d3b21359717a/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 66605cbb0a3b87b26950ae1835706ebb1f34600e Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 17:36:58 +0200 Subject: [PATCH 13/58] test(proxy): Fit the accountant fixtures to the library's usage checks Accountant main now refuses usage outside 0.05 to 2.0 tokens per byte (ratio_range, accountant#28) and prices a request with no client cache mark at the read tier (accountant#24). Fixtures with a few hundred bytes and thousands of tokens failed, and the absurd-usage cases now fall back as "unpriced" with a failed ratio_range check, not "implausible_size". - Scale the token counts of the savings, envelope and injection fixtures to their body sizes. - Count the read tier when a test asks whether a change was priced. - Re-derive the realistic-usage figure at the read rate (2e-7). - The moving-cache-mark test now asserts that no break is reported. Without the B1 fix the library no longer prices the replay at the write rate, it reports a false break. Checked: fails without the fix. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/accountant_envelope_test.go | 4 ++-- proxy/accountant_inject_test.go | 2 +- proxy/accountant_savings_test.go | 21 ++++++++++++------ proxy/accountant_shadow_test.go | 10 ++++----- proxy/accountant_wire_test.go | 37 ++++++++++++++++--------------- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/proxy/accountant_envelope_test.go b/proxy/accountant_envelope_test.go index 07c89d3a..13ae0102 100644 --- a/proxy/accountant_envelope_test.go +++ b/proxy/accountant_envelope_test.go @@ -179,7 +179,7 @@ func TestHeadTTLRelabelBooksThePremium(t *testing.T) { st := newSavingsTracker() req := st.begin(accountant.RequestMeta{RequestID: "r1", Session: "s", Model: "m", At: time.Unix(1_700_000_000, 0)}, client) st.end(req, accountant.Thread{ID: "main"}, fwd, ops, accountant.EntryFacts{TTL: time.Hour, Lifetime: accountant.LifetimeExact}) - res := st.compute(req, Usage{CacheWrite: 2000, CacheWrite1h: 2000}) + res := st.compute(req, Usage{CacheWrite: 100, CacheWrite1h: 100}) for _, e := range res.CacheEffects { if e.Kind == accountant.EffectTTLPremium { return @@ -217,7 +217,7 @@ func assertPremiumBooked(t *testing.T, client, fwd []byte, ops []accountant.Oper st := newSavingsTracker() req := st.begin(accountant.RequestMeta{RequestID: "r1", Session: "s", Model: "m", At: time.Unix(1_700_000_000, 0)}, client) st.end(req, accountant.Thread{ID: "main"}, fwd, ops, accountant.EntryFacts{TTL: time.Hour, Lifetime: accountant.LifetimeExact}) - res := st.compute(req, Usage{CacheWrite: 2000, CacheWrite1h: 2000}) + res := st.compute(req, Usage{CacheWrite: 100, CacheWrite1h: 100}) if len(res.Checks) != 0 { t.Errorf("checks failed: %+v", res.Checks) } diff --git a/proxy/accountant_inject_test.go b/proxy/accountant_inject_test.go index ca179f97..1cb8be8d 100644 --- a/proxy/accountant_inject_test.go +++ b/proxy/accountant_inject_test.go @@ -29,7 +29,7 @@ func TestSavingsRecordEndsWithTheInjectedForwardedBody(t *testing.T) { fwd.Store(b) w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"id":"m1","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],`+ - `"stop_reason":"end_turn","usage":{"input_tokens":20,"output_tokens":1,"cache_creation_input_tokens":50}}`) + `"stop_reason":"end_turn","usage":{"input_tokens":200,"output_tokens":1,"cache_creation_input_tokens":300}}`) })) defer up.Close() h, rec, tap := accHandlerYAML(t, up.URL, "pipeline: [extract, extract_llm_sweep]\n") diff --git a/proxy/accountant_savings_test.go b/proxy/accountant_savings_test.go index 2b0c421c..800bebe4 100644 --- a/proxy/accountant_savings_test.go +++ b/proxy/accountant_savings_test.go @@ -44,7 +44,7 @@ func countChange(res accountant.Result, id string) (n int, saved int) { for _, o := range res.Ops { if o.ChangeID == id { n++ - saved += o.Saved.Input + o.Saved.CacheWrite5m + o.Saved.CacheWrite1h + saved += o.Saved.Input + o.Saved.CacheWrite5m + o.Saved.CacheWrite1h + o.Saved.CacheRead } } return @@ -57,7 +57,7 @@ func TestSavings_ReplayedChangeIsPricedOnce(t *testing.T) { m0, _ := msg("user", "first") m2, _ := msg("assistant", "answer") client, fwd := body(m0, big, m2), body(m0, small, m2) - usage := Usage{FreshInput: 100, CacheRead: 1000} + usage := Usage{FreshInput: 10, CacheRead: 100} const id = "cg:mask:a1" // (a) first request: reported and priced. @@ -103,7 +103,7 @@ func TestSavings_ReReportsAfterChangeEnded(t *testing.T) { small, nSmall := msg("user", "[masked]") m0, _ := msg("user", "first") m2, _ := msg("assistant", "answer") - usage := Usage{FreshInput: 100, CacheRead: 1000} + usage := Usage{FreshInput: 10, CacheRead: 100} const id = "cg:mask:a1" c.do(1, body(m0, big, m2), body(m0, small, m2), usage, rewriteOp(id, 1, nBig, nSmall)) @@ -182,7 +182,7 @@ func TestSavings_ChangedChangeIDDoubleCountIsGated(t *testing.T) { small, nSmall := msg("user", "[masked]") m0, _ := msg("user", "first") m2, _ := msg("assistant", "answer") - usage := Usage{FreshInput: 100, CacheRead: 1000} + usage := Usage{FreshInput: 10, CacheRead: 100} r1 := c.do(1, body(m0, big, m2), body(m0, small, m2), usage, rewriteOp("cg:mask:1:h1", 1, nBig, nSmall)) if len(r1.Checks) != 0 { t.Fatalf("first request checks = %+v", r1.Checks) @@ -240,12 +240,12 @@ func TestSavings_AMovingCacheMarkDoesNotRepriceAReplayAtTheWriteRate(t *testing. op := rewriteOp(id, 1, nBig, nSmall) // turn 1: the mark is on message 2. Nothing cached before it. - c.do(1, body(m0, big, withMark("a")), body(m0, small, withMark("a")), Usage{CacheWrite: 3000}, op) + c.do(1, body(m0, big, withMark("a")), body(m0, small, withMark("a")), Usage{CacheWrite: 100}, op) // turn 2 and 3: the mark moves to the newest message. The text of the earlier ones did not change. c.do(2, body(m0, big, plain("a"), withMark("b")), body(m0, small, plain("a"), withMark("b")), - Usage{CacheRead: 3000, CacheWrite: 100}, op) + Usage{CacheRead: 100, CacheWrite: 10}, op) res := c.do(3, body(m0, big, plain("a"), plain("b"), withMark("c")), body(m0, small, plain("a"), plain("b"), withMark("c")), - Usage{CacheRead: 3100, CacheWrite: 100}, op) + Usage{CacheRead: 110, CacheWrite: 10}, op) var got *accountant.PricedOp for i := range res.Ops { @@ -259,6 +259,13 @@ func TestSavings_AMovingCacheMarkDoesNotRepriceAReplayAtTheWriteRate(t *testing. if got.Saved.CacheWrite5m != 0 || got.Saved.Input != 0 || got.Saved.CacheRead == 0 { t.Fatalf("turn 3 priced the replay at %+v, want the read tier (the provider read it)", got.Saved) } + // The mark moved; the text did not. A hash that includes the mark reads that as an edit and the + // library reports a cache break that never happened. + for _, e := range res.CacheEffects { + if e.Kind == accountant.EffectBreak { + t.Fatalf("turn 3 reports a cache break %+v: the moving mark was read as an edit", e) + } + } } // context-guru#768 B9: the cachesplit credit was the whole cached read. A replay of four Claude Code sessions booked diff --git a/proxy/accountant_shadow_test.go b/proxy/accountant_shadow_test.go index 623f23c9..debd3238 100644 --- a/proxy/accountant_shadow_test.go +++ b/proxy/accountant_shadow_test.go @@ -22,21 +22,21 @@ func (s *syncBuf) Write(p []byte) (int, error) { s.mu.Lock(); defer s.mu.Unlock( func (s *syncBuf) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() } // A acc_fallback line is logged once for each event left on the legacy rules, with the reason. The -// usage here bills far fewer tokens than the forwarded body holds, so the library's bytes-to-tokens -// ratio saves more than the request had and both events fall back (implausible_size). +// usage here bills far more tokens than the forwarded body holds, so the library refuses it +// (ratio_range) and both events fall back (unpriced). func TestFallbackReasonIsLogged(t *testing.T) { var buf syncBuf prev := slog.Default() slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) t.Cleanup(func() { slog.SetDefault(prev) }) evs, _ := accRunUsage(t, 12, 9000, 1500) - if fb := evs[0].AccountantFallback(); fb != dash.FallbackImplausibleSize { - t.Fatalf("second event: fallback %q, want %q", fb, dash.FallbackImplausibleSize) + if fb := evs[0].AccountantFallback(); fb != dash.FallbackUnpriced { + t.Fatalf("second event: fallback %q, want %q", fb, dash.FallbackUnpriced) } if n := strings.Count(buf.String(), `"msg":"acc_fallback"`); n != 2 { t.Fatalf("acc_fallback lines = %d, want 2 (one for each event):\n%s", n, buf.String()) } - if !strings.Contains(buf.String(), `"reason":"implausible_size"`) { + if !strings.Contains(buf.String(), `"reason":"unpriced"`) { t.Fatalf("acc_fallback lacks the reason:\n%s", buf.String()) } } diff --git a/proxy/accountant_wire_test.go b/proxy/accountant_wire_test.go index 76f39b29..eb1c7c4e 100644 --- a/proxy/accountant_wire_test.go +++ b/proxy/accountant_wire_test.go @@ -158,8 +158,10 @@ func extractRow(t *testing.T, e dash.Event) dash.CompRow { // usage = 20 input + 50 cache write = 70 tokens // ratio = 70 tokens / fwdBytes (the library's bytes-to-tokens ratio) // saved tokens = floor(before*70/fwdBytes) - floor(after*70/fwdBytes), in the Input tier: the -// removed span was never cached (no previous request cached it) -// SavedUSD = saved tokens * 2e-6 (fixedPricer Input rate) +// removed span was never cached (no previous request cached it). The request has no +// cache_control mark, so the provider caches nothing the client asked for and the +// library prices the removed span at the read tier (accountant#24) +// SavedUSD = saved tokens * 2e-7 (fixedPricer CacheRead rate) // // The op sizes are wire bytes, so the derivation is exact up to the two floors (1 token). // It runs through the real proxy path, and the Result must carry no failed check: before the op @@ -185,7 +187,7 @@ func TestRealisticUsageMatchesHandDerivation(t *testing.T) { return n } client, fb := msgBytes(anthropicRequest("aws/claude-sonnet-5")), msgBytes(fwd) - want := float64((client-fb)*70/fb) * 2e-6 + want := float64((client-fb)*70/fb) * 2e-7 got := extractRow(t, e).SavedUSD if math.Abs(got-want) > 2*2e-6 { t.Fatalf("extract SavedUSD = %v, hand-derived %v (client %d forwarded %d message bytes)", got, want, client, fb) @@ -193,10 +195,11 @@ func TestRealisticUsageMatchesHandDerivation(t *testing.T) { t.Logf("library %v hand %v", got, want) } -// The old fixture: 10,512 prompt tokens for a few-hundred-byte forwarded body. The library's -// token figure then exceeds the request's size, so PriceMode must book no saving, and for that -// reason: the Result has counted operations whose Saved tokens exceed TokensBefore and nothing -// unpriced, so the plausibility bound is what fired, not some earlier gate. +// Usage far from the body's size: 10,512 prompt tokens for a few-hundred-byte forwarded body, 21 +// tokens per byte. The library refuses it with the ratio_range check before it prices anything +// (accountant#28), so PriceMode books no saving and the event keeps the legacy figures. The reason +// is "unpriced": the Result is built, its checks name why. FallbackImplausibleSize is the second +// line of defence for a Result that passes every check (dash/accountant_switch_test.go). func TestAbsurdUsageBooksNoSaving(t *testing.T) { cfs, _ := accRunUsage(t, 12, 9000, 1500) if got := extractRow(t, cfs[0]).SavedUSD; got != 0 || cfs[0].BaselineCostUSD != cfs[0].CostUSD { @@ -204,20 +207,18 @@ func TestAbsurdUsageBooksNoSaving(t *testing.T) { got, cfs[0].BaselineCostUSD, cfs[0].CostUSD) } r, ok := cfs[0].AccountantResult() - if !ok || len(r.Unpriced) != 0 || len(r.Checks) != 0 { - t.Fatalf("Result %v: unpriced %+v checks %+v, want a clean Result", ok, r.Unpriced, r.Checks) + if !ok { + t.Fatal("the Result was not attached") } - saved := 0 - for _, o := range r.Ops { - if !o.DryRun && o.Call == nil { - saved += o.Saved.Input + o.Saved.CacheRead + o.Saved.CacheWrite5m + o.Saved.CacheWrite1h - } + found := false + for _, c := range r.Checks { + found = found || c.Check == "ratio_range" } - if saved <= cfs[0].TokensBefore { - t.Fatalf("counted Saved %d <= TokensBefore %d: the bound would not fire", saved, cfs[0].TokensBefore) + if !found { + t.Fatalf("checks %+v, want a failed ratio_range", r.Checks) } - if fb := cfs[0].AccountantFallback(); fb != dash.FallbackImplausibleSize { - t.Fatalf("fallback reason %q, want %q", fb, dash.FallbackImplausibleSize) + if fb := cfs[0].AccountantFallback(); fb != dash.FallbackUnpriced { + t.Fatalf("fallback reason %q, want %q", fb, dash.FallbackUnpriced) } } From 2dd567dfbeed522b3b734f8f00ac15df9db91e91 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 17:44:54 +0200 Subject: [PATCH 14/58] build: Pin accountant to main at d31126b (media and host token counts) Takes accountant#30: Size gains optional Tokens and MediaBytes, and check 9 judges the media-adjusted ratio. The integration sets neither, so no behavior changes. Proxy and dash tests pass. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8fbaccdd..92bd07ee 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.26.4 require github.com/maximhq/bifrost/core v1.7.0 // accountant prices context operations. This module's tests import it and its shared test cases. -require github.com/rossoctl/accountant v0.1.1-0.20261104152152-d3b21359717a +require github.com/rossoctl/accountant v0.1.1-0.20261104153022-d31126bbb49b require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 diff --git a/go.sum b/go.sum index 939cad01..5709e52d 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rossoctl/accountant v0.1.1-0.20261104152152-d3b21359717a h1:EkB2CqzV7Naeqfowf/A4mwZWZMATiuylWPHpJFrE8OI= -github.com/rossoctl/accountant v0.1.1-0.20261104152152-d3b21359717a/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261104153022-d31126bbb49b h1:DnS0+FiNJt4qIUHjDAQ+G6H6Lmx28Aw8hAa4rciSbvM= +github.com/rossoctl/accountant v0.1.1-0.20261104153022-d31126bbb49b/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 34804561a49f1a9474f5a06ed1200c945db02406 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 17:49:51 +0200 Subject: [PATCH 15/58] feat(proxy): Save the accountant thread state and restore it at start The library keeps each thread's active changes in memory, so every restart dropped the replay saving of every live session until the next full request (review of #768, major 2). Accountant#13 added Snapshot and Restore. The handler restores from .accountant-state.json at New, saves it every minute and at Close, and fails open: a missing, stale (over 1h), corrupt or unwritable file starts the proxy empty. The file is written through a temp file and a rename, mode 0600. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- .../accountant_state_test.go | 21 +++++ cmd/context-guru-proxy/main.go | 17 +++- docs/dashboard.md | 7 ++ proxy/accountant_state.go | 89 ++++++++++++++++++ proxy/accountant_state_test.go | 90 +++++++++++++++++++ proxy/accountant_wire_test.go | 14 ++- proxy/proxy.go | 16 ++++ 7 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 cmd/context-guru-proxy/accountant_state_test.go create mode 100644 proxy/accountant_state.go create mode 100644 proxy/accountant_state_test.go diff --git a/cmd/context-guru-proxy/accountant_state_test.go b/cmd/context-guru-proxy/accountant_state_test.go new file mode 100644 index 00000000..05aef8d5 --- /dev/null +++ b/cmd/context-guru-proxy/accountant_state_test.go @@ -0,0 +1,21 @@ +package main + +import "testing" + +func TestAccountantStatePath(t *testing.T) { + for _, tc := range []struct { + db string + dashboard bool + want string + }{ + {"/var/cg/dash.db", true, "/var/cg/dash.db.accountant-state.json"}, + {"/var/cg/dash.db", false, ""}, + {"", true, ""}, + {":memory:", true, ""}, + {"file::memory:?cache=shared", true, ""}, + } { + if got := accountantStatePath(tc.db, tc.dashboard); got != tc.want { + t.Errorf("accountantStatePath(%q, %v) = %q, want %q", tc.db, tc.dashboard, got, tc.want) + } + } +} diff --git a/cmd/context-guru-proxy/main.go b/cmd/context-guru-proxy/main.go index a796237b..343d8750 100644 --- a/cmd/context-guru-proxy/main.go +++ b/cmd/context-guru-proxy/main.go @@ -703,9 +703,11 @@ func main() { Prices: priceResolver(windows), // per-token rates, so each captured request is priced at write time Preset: cfg.Preset, Dashboard: rec, // nil unless --dashboard - Pings: kaKeepAlive.pings, - Record: traceRec, // nil unless --record - Mode: mode, // sync (default) | observe — explicit, never inferred + // The accountant's thread state survives a restart; it sits beside the dashboard database. + AccountantState: accountantStatePath(*dashDB, rec != nil), + Pings: kaKeepAlive.pings, + Record: traceRec, // nil unless --record + Mode: mode, // sync (default) | observe — explicit, never inferred Observe: proxy.ObserveOptions{ MaxQueue: cfg.Observe.MaxQueue, Workers: cfg.Observe.Workers, @@ -1535,3 +1537,12 @@ func envOr0(v, fallback string) string { } return fallback } + +// accountantStatePath is where the savings tracker saves its thread state: next to the dashboard database, +// or empty when there is no dashboard (the tracker does not run then) or the database is in memory. +func accountantStatePath(dashDB string, dashboard bool) string { + if !dashboard || dashDB == "" || strings.HasPrefix(dashDB, ":memory:") || strings.HasPrefix(dashDB, "file::memory:") { + return "" + } + return dashDB + ".accountant-state.json" +} diff --git a/docs/dashboard.md b/docs/dashboard.md index f3df7728..ea076a8e 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -141,6 +141,13 @@ Limits: in the request JSON. The request detail marks the net saving with "≈" and says why in its tooltip. The totals and charts add such a figure like any other: they do not mark it. A row written before the column existed reads as exact. +- The library keeps each thread's state in memory. The proxy saves it to `.accountant-state.json` + every minute and at a clean shutdown, and restores it at start, so a restart does not drop the active + changes of a live session. The file holds hashes, sizes and ids, no message text, and is mode 0600. + A file older than one hour is ignored (no cache entry lives longer), and so is a missing or unreadable + one: the proxy then starts empty, as before. A crash (kill -9) loses up to one minute of state. A + request that follows a lost state is priced as a first request and carries the estimate flag. The + library also drops the least recently used threads past its own bound; that is not saved or restored. - A thread that continues another by content (a rewind, a fork) is priced as a first request. The proxy passes the parent thread to the library, but the library does not read it, so the new thread does not start from a copy of the parent's cache state (accountant design §4.1). diff --git a/proxy/accountant_state.go b/proxy/accountant_state.go new file mode 100644 index 00000000..0e48952c --- /dev/null +++ b/proxy/accountant_state.go @@ -0,0 +1,89 @@ +package proxy + +import ( + "errors" + "log/slog" + "os" + "path/filepath" + "time" +) + +// A thread's state lives in memory in the library. Without a save, every restart drops the active +// changes of every live session: the next request of each one is priced as an estimate (context-guru#768, +// major 2). The snapshot holds hashes and sizes, no message text (accountant persist.go). +const ( + // accStateMaxAge is how old a snapshot may be and still be restored. No cache entry outlives the longest + // TTL (1h), so an older snapshot holds only expired state. + accStateMaxAge = time.Hour + // accStateEvery is how often the state is saved while running, so a kill -9 or a crash loses minutes, not all. + accStateEvery = time.Minute +) + +// save writes the snapshot to path through a temp file and a rename, so a crash mid-write leaves the old file. +func (s *savingsTracker) save(path string) error { + data, err := s.trackers.Snapshot() + if err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".tmp*") + if err != nil { + return err + } + _, werr := tmp.Write(data) + cerr := tmp.Close() + if err := errors.Join(werr, cerr); err != nil { + os.Remove(tmp.Name()) + return err + } + if err := os.Chmod(tmp.Name(), 0o600); err != nil { + os.Remove(tmp.Name()) + return err + } + return os.Rename(tmp.Name(), path) +} + +// restore loads a snapshot saved by save. It fails open: a missing, stale or unreadable file is logged +// and the proxy starts with empty state, as before. It reports whether it restored anything. +func (s *savingsTracker) restore(path string, now time.Time) bool { + fi, err := os.Stat(path) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + slog.Warn("accountant: state file unreadable; starting empty", "path", path, "err", err) + } + return false + } + if age := now.Sub(fi.ModTime()); age > accStateMaxAge { + slog.Info("accountant: state file is older than any cache entry; starting empty", "path", path, "age", age.Round(time.Second)) + return false + } + data, err := os.ReadFile(path) + if err == nil { + err = s.trackers.Restore(data) + } + if err != nil { + slog.Warn("accountant: state not restored; starting empty", "path", path, "err", err) + return false + } + slog.Info("accountant: state restored", "path", path, "bytes", len(data)) + return true +} + +// keepSaved saves the state every accStateEvery until stop is closed, then once more. It returns when done. +func (s *savingsTracker) keepSaved(path string, stop <-chan struct{}, done chan<- struct{}) { + defer close(done) + t := time.NewTicker(accStateEvery) + defer t.Stop() + for { + select { + case <-t.C: + case <-stop: + if err := s.save(path); err != nil { + slog.Warn("accountant: state not saved at shutdown", "path", path, "err", err) + } + return + } + if err := s.save(path); err != nil { + slog.Warn("accountant: state not saved", "path", path, "err", err) + } + } +} diff --git a/proxy/accountant_state_test.go b/proxy/accountant_state_test.go new file mode 100644 index 00000000..78ba4ecd --- /dev/null +++ b/proxy/accountant_state_test.go @@ -0,0 +1,90 @@ +package proxy + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/rossoctl/accountant" +) + +// twoTurns runs turn 1 (the change is made and reported) on a tracker, saves it, and returns the bodies +// of turn 2, on which the change is not reported again: the library carries it from the thread state. +func twoTurns(t *testing.T, path string) (turn2c, turn2f []byte, id string) { + t.Helper() + id = "cg:mask:state" + big, nBig := msg("user", strings.Repeat("tool result ", 400)) + small, nSmall := msg("user", "[masked]") + m0, m2 := mustMsg("user", "first"), mustMsg("assistant", "answer") + c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} + c.do(1, body(m0, big, m2), body(m0, small, m2), Usage{FreshInput: 10, CacheRead: 100}, rewriteOp(id, 1, nBig, nSmall)) + if err := c.s.save(path); err != nil { + t.Fatal(err) + } + next := mustMsg("user", "next") + return body(m0, big, m2, next), body(m0, small, m2, next), id +} + +func turn2(s *savingsTracker, c, f []byte) accountant.Result { + meta := accountant.RequestMeta{RequestID: "r2", Session: "s", At: time.Unix(1020, 0), Billing: accountant.Billing{Mode: accountant.BillingAPI}} + r := s.begin(meta, c) + s.end(r, accountant.Thread{ID: "main"}, f, nil, accFacts) + return s.compute(r, Usage{FreshInput: 10, CacheRead: 100}) +} + +// A restart keeps the active change of a live session: the next request still counts it. Without the restore the +// change is gone (context-guru#768, major 2). +func TestSavingsStateSurvivesARestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + c, f, id := twoTurns(t, path) + + cold := turn2(newSavingsTracker(), c, f) + if n, _ := countChange(cold, id); n != 0 { + t.Fatalf("precondition: a fresh tracker priced the change %d times, want 0", n) + } + s := newSavingsTracker() + if !s.restore(path, time.Now()) { + t.Fatal("restore reported nothing restored") + } + if n, saved := countChange(turn2(s, c, f), id); n != 1 || saved == 0 { + t.Fatalf("after the restore the change was priced %d times for %d tokens, want 1 and non-zero", n, saved) + } +} + +// A missing, stale or corrupt file starts the proxy empty. None of them may fail the start. +func TestSavingsStateFailsOpen(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + if newSavingsTracker().restore(path, time.Now()) { + t.Error("a missing file was restored") + } + twoTurns(t, path) + if newSavingsTracker().restore(path, time.Now().Add(accStateMaxAge+time.Minute)) { + t.Error("a file older than any cache entry was restored") + } + if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if newSavingsTracker().restore(path, time.Now()) { + t.Error("a corrupt file was restored") + } +} + +// Close saves the state, and the file is private: it holds session ids. +func TestHandlerCloseSavesTheState(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + up := fakeUpstream(t) + defer up.Close() + h, _, _ := accHandlerOpts(t, up.URL, "pipeline: [extract]\n", func(o *Options) { o.AccountantState = path }) + h.Close() + h.Close() // the test cleanup closes it again; that must be safe + fi, err := os.Stat(path) + if err != nil { + t.Fatalf("Close saved no state file: %v", err) + } + if fi.Mode().Perm() != 0o600 { + t.Errorf("state file mode %v, want 0600", fi.Mode().Perm()) + } +} diff --git a/proxy/accountant_wire_test.go b/proxy/accountant_wire_test.go index eb1c7c4e..9cf1b96a 100644 --- a/proxy/accountant_wire_test.go +++ b/proxy/accountant_wire_test.go @@ -41,6 +41,12 @@ func accHandler(t *testing.T, up string) (*Handler, *dash.Recorder, *accTap) { // accHandlerYAML is accHandler with the config document spelled out. func accHandlerYAML(t *testing.T, up, yaml string) (*Handler, *dash.Recorder, *accTap) { + t.Helper() + return accHandlerOpts(t, up, yaml, nil) +} + +// accHandlerOpts is accHandlerYAML with a hook that edits the handler options before New. +func accHandlerOpts(t *testing.T, up, yaml string, edit func(*Options)) (*Handler, *dash.Recorder, *accTap) { t.Helper() cfg, err := config.LoadBytes([]byte(yaml)) if err != nil { @@ -58,10 +64,14 @@ func accHandlerYAML(t *testing.T, up, yaml string) (*Handler, *dash.Recorder, *a t.Fatal(err) } t.Cleanup(func() { rec.Close() }) - h := New(pipe, store.NewMemory(store.Options{}), agg, Options{ + opts := Options{ AnthropicUpstream: up, OpenAIUpstream: up, Preset: cfg.Preset, Dashboard: rec, Prices: fixedPricer{}, - }) + } + if edit != nil { + edit(&opts) + } + h := New(pipe, store.NewMemory(store.Options{}), agg, opts) t.Cleanup(h.Close) return h, rec, tap } diff --git a/proxy/proxy.go b/proxy/proxy.go index d5c2ffe6..e1902d37 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -143,6 +143,9 @@ type Options struct { // request is captured off the hot path into a durable store and the dashboard UI // + API are mounted. nil = the proxy behaves exactly as before. Dashboard *dash.Recorder + // AccountantState, when set with Dashboard, is the file the savings tracker's thread state is saved to + // (every minute and at Close) and restored from at New. Empty = the state starts empty at every start. + AccountantState string // Record, when non-nil, records one OTLP span per proxied turn: exported to a collector // (--record-otlp), written as OTLP JSON one file per session (--record), or both. nil = no // recording, and every touchpoint is nil-guarded, so the default path costs one nil check @@ -349,6 +352,10 @@ type Handler struct { // for every send still in flight before its mail sink goes away (see waitMailForTest). mailWG sync.WaitGroup + // stateStop and stateDone run the periodic save of the savings tracker's state (Options.AccountantState). + stateStop, stateDone chan struct{} + stateOnce sync.Once + // lastSessionMu guards lastSession. lastSessionMu sync.Mutex // lastSession is the session id of the most recent REAL (non-ping) request this @@ -422,6 +429,11 @@ func New(pipe *components.Pipeline, st store.Store, agg *metrics.Aggregator, opt // The tracker needs a dashboard: the Result is attached to the recorded event. if opts.Dashboard != nil { h.savings = newSavingsTracker() + if opts.AccountantState != "" { + h.savings.restore(opts.AccountantState, time.Now()) + h.stateStop, h.stateDone = make(chan struct{}), make(chan struct{}) + go h.savings.keepSaved(opts.AccountantState, h.stateStop, h.stateDone) + } } // The restart grace decides by the thread's own last cache refresh — its last request or // a keep-alive ping that read the cache — and not by this process's start (#766). The @@ -519,6 +531,10 @@ func (h *Handler) onRegistryChange(kind string) { // sync-mode handler and safe to call twice. func (h *Handler) Close() { h.pool.Stop() + if h.stateStop != nil { + h.stateOnce.Do(func() { close(h.stateStop) }) + <-h.stateDone // saves the state once more + } // The keeper holds request bodies and, on a caller-pays upstream, callers' provider // credentials. Stopping it drops both, which is why this is not optional on a host that // builds and discards handlers. From 25f274141814b3ae3818d2a8e2463ae2e339bb3a Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 17:54:22 +0200 Subject: [PATCH 16/58] fix(dash): Give the read-only shim view a rowid OpenReadOnly shims the columns an older database lacks with a TEMP VIEW. A view has no rowid, so Request(id) failed on such a file with "no such column: rowid", and cg-cache-campaign opens databases this way (review of #768, minor). Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/openreadonly_test.go | 38 ++++++++++++++++++++++++++++++++++++++ dash/store.go | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/dash/openreadonly_test.go b/dash/openreadonly_test.go index 776578bc..72ade133 100644 --- a/dash/openreadonly_test.go +++ b/dash/openreadonly_test.go @@ -191,3 +191,41 @@ func TestOpenReadOnlyShimsAnOlderRequestComponentsTable(t *testing.T) { t.Fatalf("missing columns read as %q, %v; want their defaults", by, first) } } + +// A database written by an older proxy lacks the newest request_components columns. OpenReadOnly shims them +// with a TEMP VIEW, and a view has no rowid, so ordering a request's components by rowid failed with "no such +// column: rowid" (context-guru#768 minor). cg-cache-campaign opens databases this way. +func TestOpenReadOnlyRequestOnAnOlderDatabase(t *testing.T) { + path := filepath.Join(t.TempDir(), "old.db") + db, err := Open(path) + if err != nil { + t.Fatal(err) + } + seedRequest(t, db, "t1", "s1") + var id int64 + if err := db.sql.QueryRow(`SELECT id FROM requests`).Scan(&id); err != nil { + t.Fatal(err) + } + if _, err := db.sql.Exec(`INSERT INTO request_components(request_id, component, kind) VALUES (?, 'a', 'k'), (?, 'b', 'k')`, id, id); err != nil { + t.Fatal(err) + } + // Make it an older file: drop the newest additive column of request_components. + if _, err := db.sql.Exec(`ALTER TABLE request_components DROP COLUMN priced_by`); err != nil { + t.Fatalf("cannot build the older file: %v", err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + ro, err := OpenReadOnly(path) + if err != nil { + t.Fatal(err) + } + defer ro.Close() + e, err := ro.Request(id, false) + if err != nil { + t.Fatalf("Request on an older database: %v", err) + } + if len(e.Components) != 2 || e.Components[0].Component != "a" || e.Components[1].Component != "b" { + t.Fatalf("components %+v, want a then b in insert order", e.Components) + } +} diff --git a/dash/store.go b/dash/store.go index 0ff1a859..4d016356 100644 --- a/dash/store.go +++ b/dash/store.go @@ -374,7 +374,7 @@ func init() { } } if extra != "" && len(have) > 0 { - if _, err := conn.ExecContext(ctx, `CREATE TEMP VIEW `+table+` AS SELECT *`+extra+` FROM main.`+table, nil); err != nil { + if _, err := conn.ExecContext(ctx, `CREATE TEMP VIEW `+table+` AS SELECT rowid AS rowid, *`+extra+` FROM main.`+table, nil); err != nil { return err } } From f9f4437a2bd9bba5cf06e2a65e19386970bb6606 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 17:54:23 +0200 Subject: [PATCH 17/58] fix(dash): Count the costs in a component's replay multiple ReplayMultiple was (first + replay) / first and ignored the costs the accountant nets into saved_usd, so a component that loses money printed "1.5x". The replay cap already divides the decomposed total. Use the same numerator here and clamp it at 0 (review of #768, minor). Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accounting_test.go | 4 ++++ dash/readvalue.go | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dash/accounting_test.go b/dash/accounting_test.go index cb2f606b..30e1449e 100644 --- a/dash/accounting_test.go +++ b/dash/accounting_test.go @@ -1090,6 +1090,10 @@ func TestTheFirstRemovalVerdictIncludesTheCostsAComponentCaused(t *testing.T) { } // 0.0100 first removal - 0.0600 costs, and the component has no LLM spend of its own. nearEq(t, "verdict", c.NetUSDFirstRemoval, -0.0500) + // The amortized verdict is -0.0450 too, so the factor between the two is not a gain: it printed 1.5x. + if c.ReplayMultiple != 0 { + t.Fatalf("replay_multiple = %g for a component whose amortized verdict is %g, want 0", c.ReplayMultiple, c.SavedUSDDecomposed) + } } // A request the accountant could not price books no saving: its components carry priced_by='none' and a stored diff --git a/dash/readvalue.go b/dash/readvalue.go index 1b5fc942..ff62433a 100644 --- a/dash/readvalue.go +++ b/dash/readvalue.go @@ -2,6 +2,7 @@ package dash import ( "context" + "math" "sort" "github.com/rossoctl/context-guru/internal/modelinfo" @@ -294,7 +295,9 @@ func decomposeFromGroups(cgroups map[compGroupKey]*compGroup, p modelinfo.Pricer // component that loses money as a gain (context-guru#768 B6: saved_usd -$0.0302, this field +$0.0113). c.NetUSDFirstRemoval = c.SavedUSDFirstRemoval + c.SavedUSDCosts - c.LLMCostUSD if c.SavedUSDFirstRemoval > 0 { - c.ReplayMultiple = (c.SavedUSDFirstRemoval + c.SavedUSDReplay) / c.SavedUSDFirstRemoval + // The amortized verdict (replay and costs included) over the first-removal credit, as the replay cap + // computes it: a component whose costs exceed its credit must not print a gain. + c.ReplayMultiple = math.Max(c.SavedUSDDecomposed, 0) / c.SavedUSDFirstRemoval } // Whether this component's `unique` is a DEDUP MEASUREMENT at all. // From 0c2f66a507531009de24184fbf05f0c604e725dd Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 17:56:04 +0200 Subject: [PATCH 18/58] fix(modelinfo): Price a dated or Bedrock-wrapped model id The price lookup matched a map key against the whole id, so claude-sonnet-5-20260101 and us.anthropic.claude-sonnet-5-v1:0 found no price and no saving was booked. After the exact tiers, strip the vendor prefix, the version suffix and the release date and look the bare name up exactly. A different model (claude-sonnet-5-5) is not reached (review of #768, minor). Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- internal/modelinfo/litellm/chain_test.go | 38 ++++++++++++++++++++++++ internal/modelinfo/litellm/litellm.go | 26 ++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/internal/modelinfo/litellm/chain_test.go b/internal/modelinfo/litellm/chain_test.go index 17d9e800..4c9bb2eb 100644 --- a/internal/modelinfo/litellm/chain_test.go +++ b/internal/modelinfo/litellm/chain_test.go @@ -259,3 +259,41 @@ func TestLiteLLMBlankCacheReadFollowsTheModelTier(t *testing.T) { } } } + +// A dated id and a Bedrock or Vertex id name a model the map lists under its bare name. Neither found a +// price, because the last-resort scan needs a map key that CONTAINS the whole id, and these ids are longer +// than the key. No price means no saving is booked (context-guru#768 minor). `x` and `x[1m]` already agree. +func TestLiteLLMPriceFindsDatedAndWrappedIDs(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte(priceSample)) + })) + defer srv.Close() + l := NewLiteLLM(srv.URL, srv.Client(), time.Hour) + ctx := context.Background() + for i := 0; i < 400; i++ { + if _, ok := l.Price(ctx, "aws/claude-sonnet-5"); ok { + break + } + time.Sleep(5 * time.Millisecond) + } + want, _ := l.Price(ctx, "aws/claude-sonnet-5") + for _, id := range []string{ + "claude-sonnet-5-20260101", + "aws/claude-sonnet-5-20260101", + "us.anthropic.claude-sonnet-5-v1:0", + "anthropic.claude-sonnet-5-20260101-v2:0", + "eu.anthropic.claude-sonnet-5-v1:0[1m]", + "claude-sonnet-5@20260101", + } { + got, ok := l.Price(ctx, id) + if !ok || got != want { + t.Errorf("Price(%q) = %+v, %v; want %+v", id, got, ok, want) + } + } + // A different model must not be mistaken for it: the date and the wrapper are stripped, nothing else. + for _, id := range []string{"claude-sonnet-5-5-20260101", "us.anthropic.totally-absent-xyz-v1:0"} { + if p, ok := l.Price(ctx, id); ok && p == want { + t.Errorf("Price(%q) resolved to claude-sonnet-5's price", id) + } + } +} diff --git a/internal/modelinfo/litellm/litellm.go b/internal/modelinfo/litellm/litellm.go index b5cf8c7f..447565dc 100644 --- a/internal/modelinfo/litellm/litellm.go +++ b/internal/modelinfo/litellm/litellm.go @@ -13,6 +13,7 @@ import ( "io" "log/slog" "net/http" + "regexp" "strings" "sync" "time" @@ -259,6 +260,13 @@ func (l *LiteLLM) Price(ctx context.Context, model string) (modelinfo.Price, boo if p, ok := l.priceBy[tail]; ok { return p, true } + // A dated id (claude-sonnet-5-20260101) or a Bedrock/Vertex one (us.anthropic.claude-sonnet-5-v1:0) names a + // model the map lists under its bare name. Strip the decoration and look the exact name up again. + if bare := undecorate(tail); bare != tail { + if p, ok := l.priceBy[bare]; ok { + return p, true + } + } // Last resort: a key that CONTAINS the bare id (claude-sonnet-5 vs // anthropic.claude-sonnet-5). The SHORTEST such key wins, because the exact // matches are already handled above, so the remaining candidates differ from the @@ -316,3 +324,21 @@ func (l *LiteLLM) Window(ctx context.Context, model string) (int, bool) { } return best, found } + +var ( + // decoDate is a trailing release date: -20260101 or @20260101. + decoDate = regexp.MustCompile(`[-@]\d{8}$`) + // decoBedrock is Bedrock's version suffix: -v1:0, -v2. + decoBedrock = regexp.MustCompile(`-v\d+(:\d+)?$`) + // decoVendor is the dotted vendor prefix of a Bedrock or Vertex id: us.anthropic., eu.anthropic., anthropic. + decoVendor = regexp.MustCompile(`^(?:(?:us|eu|apac|global)\.)?(?:anthropic|amazon|meta|mistral|cohere)\.`) +) + +// undecorate removes a dated or Bedrock-wrapped id's decoration: the vendor prefix, the version suffix and the +// release date. The result is looked up as an exact key, so a different model (claude-sonnet-5-5 against +// claude-sonnet-5) is never reached by it. +func undecorate(id string) string { + id = decoVendor.ReplaceAllString(id, "") + id = decoBedrock.ReplaceAllString(id, "") + return decoDate.ReplaceAllString(id, "") +} From 94ce934336c8b46fca8335bc8e273319e3b46901 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 17:59:07 +0200 Subject: [PATCH 19/58] fix(proxy): Give the library a ping's start time, not its completion The keeper's report carries TS, the time the ping was recorded after its response. The library measures the entry gap from the start of a ping, as the provider measures the TTL. Map At to TS minus the duration (review of #768, minor). Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/accountant_ping.go | 9 ++++++++- proxy/accountant_ping_test.go | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/proxy/accountant_ping.go b/proxy/accountant_ping.go index 0443be37..04d19ef4 100644 --- a/proxy/accountant_ping.go +++ b/proxy/accountant_ping.go @@ -81,10 +81,17 @@ func pingFromReport(r components.KeepAliveReport) accountant.Ping { Session: r.Session, ThreadID: r.Thread, Model: r.Model, Host: r.Route, Usage: accountant.CallUsage{Usage: accountant.Usage{Input: int(r.FreshInput), CacheRead: int(r.CacheRead), CacheWrite: int(r.CacheWrite), CacheWrite1h: int(r.CacheWrite1h)}, Output: int(r.Output)}, - At: time.UnixMilli(r.TS), Window: r.Window, + At: pingStart(r), Window: r.Window, } } +// pingStart is when the ping began. The report's TS is when it was recorded, after the response, and the library +// measures the entry gap from the START of a request or ping, as the provider measures the TTL (accountant design +// §5.2; PingFacts.At is "when the ping started"). A ping that streams for minutes would otherwise shorten the gap. +func pingStart(r components.KeepAliveReport) time.Time { + return time.UnixMilli(r.TS).Add(-time.Duration(r.DurationMs * float64(time.Millisecond))) +} + // PingSavings prices one keep-alive ping as a spend operation and stores its facts in st for the // thread's next real request. ok is false, and nothing is stored, for a ping that is not // converted: no Window (its ChangeID would be shared by every such ping), a status outside diff --git a/proxy/accountant_ping_test.go b/proxy/accountant_ping_test.go index 376a166c..49f47f0a 100644 --- a/proxy/accountant_ping_test.go +++ b/proxy/accountant_ping_test.go @@ -323,3 +323,15 @@ func TestPingSavingsDoesNotStoreFactsForAManualPing(t *testing.T) { t.Fatal("an automatic ping stored no facts") } } + +// The report's TS is the completion time. The library's entry gap runs from the START of the ping, so the mapped +// At must be TS minus the ping's duration (context-guru#768 minor). +func TestPingAtIsTheStartOfThePing(t *testing.T) { + r := components.KeepAliveReport{TS: 1_700_000_100_000, DurationMs: 90_000} + if got, want := pingFromReport(r).At, time.UnixMilli(1_700_000_010_000); !got.Equal(want) { + t.Fatalf("At = %v, want %v (completion minus 90 s)", got, want) + } + if got := pingFromReport(components.KeepAliveReport{TS: 1_700_000_100_000}).At; !got.Equal(time.UnixMilli(1_700_000_100_000)) { + t.Fatalf("a ping with no duration: At = %v, want TS", got) + } +} From c5c7765fd14f8d02664c33afa3363113f53127b4 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 18:32:00 +0200 Subject: [PATCH 20/58] feat(dash): Count the estimated requests on Overview An estimated request is one the accountant priced without knowing whether a component broke the cache: a thread's first request, or the first after a restart that lost the state. Its net can be too high, and only the request detail said so (review of #768, minor). Add estimated_requests to the full and the lean Overview, one SUM over the stored price_estimated column in the aggregate that already runs, and a line on the Preset card when it is above zero. The three stats goldens gain the key. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accounting_test.go | 26 ++++++++++++++++++++++++ dash/overview.go | 10 +++++++-- dash/overview_lean.go | 4 +++- dash/testdata/golden/stats.json | 1 + dash/testdata/golden/stats_lean.json | 1 + dash/testdata/golden/stats_windowed.json | 1 + dash/ui/app.js | 3 +++ docs/dashboard.md | 3 ++- 8 files changed, 45 insertions(+), 4 deletions(-) diff --git a/dash/accounting_test.go b/dash/accounting_test.go index 30e1449e..b3b18dfd 100644 --- a/dash/accounting_test.go +++ b/dash/accounting_test.go @@ -5,6 +5,7 @@ import ( "math" "strings" "testing" + "time" "github.com/rossoctl/context-guru/internal/modelinfo" ) @@ -1151,3 +1152,28 @@ func TestOnlyAFallbackRowThatRemovedContentCountsAsUnpriced(t *testing.T) { t.Fatalf("saved_usd_unpriced_rows = %d, want 1 (only the fallback row that removed content)", n) } } + +// Overview counts the requests priced without knowing whether a component broke the cache, on both the full and +// the lean path. After a restart that lost the state, the next request of every live session is one of them. +func TestOverviewCountsEstimatedRequests(t *testing.T) { + db := openTestDB(t) + now := time.Now().UnixMilli() + for i, est := range []bool{true, false, true, false, false} { + e := &Event{TS: now - int64(i)*1000, SessionID: "s1", Model: "m1", TenantID: "t1", + TokensBefore: 100, TokensAfter: 90, FreshInput: 5, CacheRead: 40, OutputTokens: 5, + TokenAccounting: AccountingComplete, PriceEstimated: est} + insertReq(t, db, e) + } + f := Filter{TenantAll: true, Since: now - 60_000} + full, err := db.Overview(f) + if err != nil { + t.Fatal(err) + } + lean, err := db.OverviewLean(f, nil, nil) + if err != nil { + t.Fatal(err) + } + if full.EstimatedRequests != 2 || lean.EstimatedRequests != 2 { + t.Fatalf("estimated requests: full %d, lean %d, want 2 and 2 (of %d)", full.EstimatedRequests, lean.EstimatedRequests, full.Requests) + } +} diff --git a/dash/overview.go b/dash/overview.go index 6d37a3bc..3826489c 100644 --- a/dash/overview.go +++ b/dash/overview.go @@ -413,6 +413,10 @@ type Overview struct { PreCorrectionSavingRows int64 `json:"pre_correction_saving_rows"` PreCorrectionSavedUSD float64 `json:"pre_correction_saved_usd"` CorrectedSavingRows int64 `json:"corrected_saving_rows"` + // EstimatedRequests counts the requests whose saving the accountant library priced without knowing whether + // a component broke the cache (requests.price_estimated): the first request of a thread, and the first one + // after a restart that lost the state. Their net can be too high. Overview.Requests is the denominator. + EstimatedRequests int64 `json:"estimated_requests"` // BilledInputTokens is fresh + cache reads + cache writes: the input the PROVIDER counted. // It is here to be compared with TokensBefore, which is what our own tokenizer counted // over message text only — a different unit, roughly a third the size, and the reason no @@ -663,7 +667,8 @@ func (d *DB) Overview(f Filter) (*Overview, error) { COALESCE(SUM(CASE WHEN r.billed_token_factor = 0 THEN r.baseline_cost_usd - r.cost_usd ELSE 0 END),0), COALESCE(SUM(CASE WHEN r.billed_token_factor > 0 - AND r.baseline_cost_usd > r.cost_usd THEN 1 ELSE 0 END),0) + AND r.baseline_cost_usd > r.cost_usd THEN 1 ELSE 0 END),0), + COALESCE(SUM(r.price_estimated),0) FROM requests r WHERE `+cond, args...).Scan( &o.Requests, &o.Sessions, &o.TokensBefore, &o.TokensAfter, &o.SavedUnique, &o.AttemptedTokens, &o.FrozenTokens, &o.SavedGrossAttempted, &o.AttemptedRequests, @@ -680,7 +685,8 @@ func (d *DB) Overview(f Filter) (*Overview, error) { &o.TokensBeforeBilled, &o.BilledInputRows, &o.SSERecorded, &o.SSEStreamRows, &o.CacheTTLRecorded, &o.KeepAliveSavedUSD, &o.KeepAliveMissesAvoided, &o.CacheWrite1h, - &o.PreCorrectionSavingRows, &o.PreCorrectionSavedUSD, &o.CorrectedSavingRows) + &o.PreCorrectionSavingRows, &o.PreCorrectionSavedUSD, &o.CorrectedSavingRows, + &o.EstimatedRequests) if err != nil { return nil, err } diff --git a/dash/overview_lean.go b/dash/overview_lean.go index 8762f568..2648a14a 100644 --- a/dash/overview_lean.go +++ b/dash/overview_lean.go @@ -67,7 +67,8 @@ func (d *DB) overviewLeanWindow(f Filter) (*Overview, error) { if err := d.sql.QueryRowContext(d.readCtx(), q, args...).Scan(append([]any{ &o.Requests, &o.TokensBefore, &o.TokensAfter, &o.SavedUnique, &o.CostUSD, &o.BaselineCostUSD, &o.CGLLMCostUSD, &o.CacheSavedUSD, &o.CachesplitSavedUSD, - &o.KeepAliveSavedUSD, &o.KeepAliveMissesAvoided, &upAvg, &cgAvg, &o.KeepAlivePingUSD, &o.KeepAlivePings}, + &o.KeepAliveSavedUSD, &o.KeepAliveMissesAvoided, &upAvg, &cgAvg, &o.KeepAlivePingUSD, &o.KeepAlivePings, + &o.EstimatedRequests}, manual.dest()...)...); err != nil { return nil, err } @@ -115,6 +116,7 @@ func leanWindowMainQuery(f Filter) (string, []any) { AVG(CASE WHEN ` + agent + ` THEN r.cg_latency_ms END), COALESCE(SUM(CASE WHEN ` + autoPing + ` THEN r.cost_usd ELSE 0 END),0), COALESCE(SUM(CASE WHEN ` + autoPing + ` THEN 1 ELSE 0 END),0), + COALESCE(SUM(CASE WHEN ` + agent + ` THEN r.price_estimated ELSE 0 END),0), ` + manualSelect + ` FROM requests r WHERE ` + cond, args } diff --git a/dash/testdata/golden/stats.json b/dash/testdata/golden/stats.json index 18708d51..96562793 100644 --- a/dash/testdata/golden/stats.json +++ b/dash/testdata/golden/stats.json @@ -83,6 +83,7 @@ "percent": 2.3845059148893064 } ], + "estimated_requests": 0, "estimator_divergence": 1.001123848055743, "estimator_divergence_compacted": 1.0011973896904747, "estimator_divergence_compacted_rows": 197, diff --git a/dash/testdata/golden/stats_lean.json b/dash/testdata/golden/stats_lean.json index 18708d51..96562793 100644 --- a/dash/testdata/golden/stats_lean.json +++ b/dash/testdata/golden/stats_lean.json @@ -83,6 +83,7 @@ "percent": 2.3845059148893064 } ], + "estimated_requests": 0, "estimator_divergence": 1.001123848055743, "estimator_divergence_compacted": 1.0011973896904747, "estimator_divergence_compacted_rows": 197, diff --git a/dash/testdata/golden/stats_windowed.json b/dash/testdata/golden/stats_windowed.json index a101aab8..07516db9 100644 --- a/dash/testdata/golden/stats_windowed.json +++ b/dash/testdata/golden/stats_windowed.json @@ -83,6 +83,7 @@ "percent": 2.3779447995901024 } ], + "estimated_requests": 0, "estimator_divergence": 1.0011168816663873, "estimator_divergence_compacted": 1.0012067457085105, "estimator_divergence_compacted_rows": 79, diff --git a/dash/ui/app.js b/dash/ui/app.js index c3fe7aa0..f8475740 100644 --- a/dash/ui/app.js +++ b/dash/ui/app.js @@ -1824,6 +1824,9 @@ function renderSavedLedger(o, ka, partial) { o.requests > 0 && !partial ? ovFigure('saved-compaction-tokens', 'Tokens removed', compact(presetTok), 'of ' + compact(inputWithout) + ' tokens', share(presetTok, inputWithout)) : null, capNote(o, usd) ? el('p', { class: 'ov-none ov-small', 'data-testid': 'replay-capped', text: capNote(o, usd) }) : null, + o.estimated_requests > 0 ? el('p', { class: 'ov-none ov-small', 'data-testid': 'estimated-requests', + text: 'Includes ' + num(o.estimated_requests) + ' of ' + num(o.requests) + ' requests priced as an estimate: the first ' + + 'request of a thread, or the first after a restart that lost its state. Their net saving can be too high.' }) : null, ])); // Tools and skills filter. diff --git a/docs/dashboard.md b/docs/dashboard.md index ea076a8e..2ef683d2 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -139,7 +139,8 @@ Limits: can be too high if a component broke the cache on that request (accountant design §6.4). The event carries an estimate flag, stored in `requests.price_estimated` and sent as `price_estimated` in the request JSON. The request detail marks the net saving with "≈" and says why in its tooltip. - The totals and charts add such a figure like any other: they do not mark it. A row written before + Overview counts them (`estimated_requests`, on the full and the lean path) and the Preset card says how many of + the window's requests they are. The totals and charts add such a figure like any other: they do not mark it. A row written before the column existed reads as exact. - The library keeps each thread's state in memory. The proxy saves it to `.accountant-state.json` every minute and at a clean shutdown, and restores it at start, so a restart does not drop the active From 70501f4a35771cb165fce568ea2381fe530e1ee1 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 18:38:15 +0200 Subject: [PATCH 21/58] perf(proxy): Hash a request's body once when it is forwarded as sent begin hashes the client body and end hashed the forwarded body again, even when no component and no injection had changed a byte. On a 600 KB body that is about 4 ms of the 8 ms the accountant costs on the request path (review of #768, minor). Keep the client hash from begin to end and reuse it when the bytes are equal. A request that was rewritten hashes both, as before. The entry is dropped at end. BenchmarkBeginEndUnchanged: 8.1 ms and 10.1 MB per request before, 4.0 ms and 5.1 MB after. The rest is the library's NewBody walking each position for media, which needs a fix in accountant. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/accountant_bench_test.go | 61 ++++++++++++++++++++++++++++++++++ proxy/accountant_savings.go | 25 ++++++++++++-- proxy/accountant_state_test.go | 31 +++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 proxy/accountant_bench_test.go diff --git a/proxy/accountant_bench_test.go b/proxy/accountant_bench_test.go new file mode 100644 index 00000000..0e9f90fc --- /dev/null +++ b/proxy/accountant_bench_test.go @@ -0,0 +1,61 @@ +package proxy + +import ( + "fmt" + "github.com/rossoctl/accountant" + "strings" + "testing" + "time" +) + +// benchBody is a Claude Code shaped request of about 600 KB: 150 messages, each with a block of +// text, and a cache mark on the newest block. +func benchBody() []byte { + var b strings.Builder + b.WriteString(`{"model":"m","system":[{"type":"text","text":"sys","cache_control":{"type":"ephemeral"}}],"messages":[`) + for i := 0; i < 150; i++ { + if i > 0 { + b.WriteString(",") + } + role := "user" + if i%2 == 1 { + role = "assistant" + } + mark := "" + if i == 149 { + mark = `,"cache_control":{"type":"ephemeral"}` + } + fmt.Fprintf(&b, `{"role":%q,"content":[{"type":"text","text":%q%s}]}`, role, strings.Repeat("word ", 800)+fmt.Sprint(i), mark) + } + b.WriteString(`]}`) + return []byte(b.String()) +} + +func BenchmarkAccBodies(b *testing.B) { + raw := benchBody() + b.SetBytes(int64(len(raw))) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + accBodies(raw) + } +} + +func BenchmarkAccBreakpoints(b *testing.B) { + raw := benchBody() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + accBreakpoints(raw) + } +} + +// BenchmarkBeginEndUnchanged is the accountant's cost on the request path for a request forwarded as sent. +func BenchmarkBeginEndUnchanged(b *testing.B) { + raw := benchBody() + s := newSavingsTracker() + meta := accountant.RequestMeta{RequestID: "r", Session: "s", At: time.Unix(1000, 0), Billing: accountant.Billing{Mode: accountant.BillingAPI}} + b.ReportAllocs() + for i := 0; i < b.N; i++ { + r := s.begin(meta, raw) + s.end(r, accountant.Thread{ID: "main"}, raw, nil, accFacts) + } +} diff --git a/proxy/accountant_savings.go b/proxy/accountant_savings.go index 2630cfba..090e9e1d 100644 --- a/proxy/accountant_savings.go +++ b/proxy/accountant_savings.go @@ -1,7 +1,9 @@ package proxy import ( + "bytes" "log/slog" + "sync" "github.com/rossoctl/accountant" ) @@ -11,10 +13,20 @@ import ( type savingsTracker struct { trackers *accountant.Trackers siblings *siblingStore // each thread's last forwarded body, for EntryFacts.SharedWarm + // clients holds the hash of each open request's client body, by request, from begin to end. + clients sync.Map // *accountant.Request -> clientHash // onEnd, when set, is called with each record after EndRequest. Tests only. onEnd func(*accountant.Request) } +// clientHash is what begin built from the client body, kept until end so a request that was forwarded unchanged does +// not hash the same bytes again. Hashing a 600 KB body costs 3 to 4 ms. +type clientHash struct { + raw []byte + body accountant.Body + bps []accountant.Breakpoint +} + func newSavingsTracker() *savingsTracker { return &savingsTracker{trackers: accountant.NewTrackers(), siblings: newSiblingStore()} } @@ -22,7 +34,10 @@ func newSavingsTracker() *savingsTracker { // begin opens the request record with the client body, before any component ran. The Trackers bound // their own memory (accountant design §6.1, Memory), so there is nothing to sweep here. func (s *savingsTracker) begin(meta accountant.RequestMeta, client []byte) *accountant.Request { - return s.trackers.BeginRequest(meta, accBodies(client), accBreakpoints(client)) + body, bps := accBodies(client), accBreakpoints(client) + req := s.trackers.BeginRequest(meta, body, bps) + s.clients.Store(req, clientHash{raw: client, body: body, bps: bps}) + return req } // end records the operations, then the thread and the forwarded body. Every op is added, replays @@ -33,7 +48,13 @@ func (s *savingsTracker) end(req *accountant.Request, th accountant.Thread, forw for _, op := range ops { req.Add(op) } - fwd, bps := accBodies(forwarded), accBreakpoints(forwarded) + var fwd accountant.Body + var bps []accountant.Breakpoint + if v, ok := s.clients.LoadAndDelete(req); ok && bytes.Equal(v.(clientHash).raw, forwarded) { + fwd, bps = v.(clientHash).body, v.(clientHash).bps + } else { + fwd, bps = accBodies(forwarded), accBreakpoints(forwarded) + } facts.SharedWarm = s.siblings.sharedWarm(req.Meta.Session, th.ID, req.Meta.At, facts.TTL, fwd, bps) req.EndRequest(th, fwd, bps, facts) if s.onEnd != nil { diff --git a/proxy/accountant_state_test.go b/proxy/accountant_state_test.go index 78ba4ecd..c0673343 100644 --- a/proxy/accountant_state_test.go +++ b/proxy/accountant_state_test.go @@ -1,6 +1,7 @@ package proxy import ( + "fmt" "os" "path/filepath" "strings" @@ -88,3 +89,33 @@ func TestHandlerCloseSavesTheState(t *testing.T) { t.Errorf("state file mode %v, want 0600", fi.Mode().Perm()) } } + +// begin keeps the client body's hash for end, so a request forwarded unchanged does not hash 600 KB twice. The +// reuse must not change the answer, and the entry must not outlive the request. +func TestSavingsEndReusesTheClientHashOnlyForIdenticalBytes(t *testing.T) { + s := newSavingsTracker() + client := benchBody() + meta := accountant.RequestMeta{RequestID: "r1", Session: "s", At: time.Unix(1000, 0), Billing: accountant.Billing{Mode: accountant.BillingAPI}} + held := func() (n int) { s.clients.Range(func(any, any) bool { n++; return true }); return } + + r := s.begin(meta, client) + if held() != 1 { + t.Fatalf("begin kept %d client hashes, want 1", held()) + } + s.end(r, accountant.Thread{ID: "main"}, client, nil, accFacts) + if held() != 0 { + t.Fatalf("end left %d client hashes behind", held()) + } + same := s.compute(r, Usage{FreshInput: 100, CacheWrite: 30000}) + + // A changed body takes the other path, and the answer is the one a fresh hash gives. + changed := []byte(strings.Replace(string(client), "word word", "wurd word", -1)) + r2 := s.begin(meta, client) + s.end(r2, accountant.Thread{ID: "main"}, changed, nil, accFacts) + if r2.Forwarded.Pos[len(r2.Forwarded.Pos)-1].Hash == r2.Client.Pos[len(r2.Client.Pos)-1].Hash { + t.Fatal("a changed forwarded body reused the client hash") + } + if want := accBodies(client); fmt.Sprint(r.Forwarded) != fmt.Sprint(want) || len(same.Checks) > 1 { + t.Fatalf("the reused hash differs from a fresh one") + } +} From ad7a7e91848b4edbebe977fbd3aa72dafe552e69 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 18:44:13 +0200 Subject: [PATCH 22/58] docs: Say that long-context price tiers are not applied The price lookup reads the base rates only, so a model that charges more above 100k or 200k prompt tokens is under-priced on those requests. Of the bare Claude ids in LiteLLM's map, two carry such a tier. Applying it means choosing the tier from each request's prompt size at 29 pricing call sites, which moves every cost figure; that is left as a decision (review of #768, major 3). Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- docs/dashboard.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/dashboard.md b/docs/dashboard.md index 2ef683d2..4b8120ec 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -152,6 +152,11 @@ Limits: - A thread that continues another by content (a rewind, a fork) is priced as a first request. The proxy passes the parent thread to the library, but the library does not read it, so the new thread does not start from a copy of the parent's cache state (accountant design §4.1). +- Every request is priced at the model's base rates. A model whose provider charges more above a prompt size + (LiteLLM's `*_above_100k_tokens` and `*_above_200k_tokens` fields, for example `claude-haiku-5-5` above 100k + and `claude-sonnet-4-5` above 200k) is under-priced on those requests: its cost and its saving both use the + base rate, so their ratio looks right and the dollars are low. Only a few Claude ids carry such a tier. + `MODEL_PRICES` sets one flat rate for a model, so it does not fix a tiered price. - The cache lifetime comes from the provider rule the cold gate uses (`apply.CacheLifetime`): an exact expiry for Anthropic, a minimum for OpenAI's 30-minute entries. Where it is unknown, the library does not credit a keep-alive rescue after a gap past the TTL. From f95f439e246c6e729634829fb0ea850b9b9e563b Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 18:45:59 +0200 Subject: [PATCH 23/58] feat(proxy): Say why an event has no accountant result no_result stands for four unrelated causes: no record was opened, the response carried no usage, an expand loop ran several upstream rounds, or Compute panicked. The acc_fallback line gives the reason only. Add a "why" attribute for no_result, so the first request of a session that reads a warm cache and books nothing (review of #768, major 5) names its cause the next time it happens. I could not reproduce that case without the trace. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- docs/dashboard.md | 3 +++ proxy/accountant_shadow.go | 12 +++++++++--- proxy/accountant_state_test.go | 22 ++++++++++++++++++++++ proxy/dashcapture.go | 23 ++++++++++++++++++++++- 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/dashboard.md b/docs/dashboard.md index 4b8120ec..42d610d6 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -126,6 +126,9 @@ When the library cannot price an event, the event books **no saving**, and the p `figure_without_row`, `duplicate_name`, `acted_without_figure` or `unsupported_wire`. A saving nothing measured is not shown. +For `no_result` the line also carries `why`: `no_record` (no session, or the pipeline did not run), `no_usage`, +`upstream_rounds=N` (an expand loop) or `compute_panicked`. + `unsupported_wire` is the OpenAI Responses API. It keeps the conversation in `input`, and the proxy hashes `messages` and reports no operations there, so **every saving on that wire is $0**. Chat Completions and the Anthropic Messages API are priced. Until Responses is read, a Responses account sees no saving and the diff --git a/proxy/accountant_shadow.go b/proxy/accountant_shadow.go index a911bfc9..d1b464b8 100644 --- a/proxy/accountant_shadow.go +++ b/proxy/accountant_shadow.go @@ -11,10 +11,16 @@ import ( // logFallback writes the acc_fallback line for an event the accountant library could not price, so it // books no saving, // with the reason (dash.Fallback*). -func logFallback(e *dash.Event) { +// +// why is set for no_result only: it names which of that reason's causes applied (noResultWhy). +func logFallback(e *dash.Event, why string) { r, _ := e.AccountantResult() - slog.Info("acc_fallback", "session", e.SessionID, "thread", e.ThreadID, - "reason", e.AccountantFallback(), "checks", checkNames(r), "recomputed", r.Recomputed) + attrs := []any{"session", e.SessionID, "thread", e.ThreadID, + "reason", e.AccountantFallback(), "checks", checkNames(r), "recomputed", r.Recomputed} + if why != "" && e.AccountantFallback() == dash.FallbackNoResult { + attrs = append(attrs, "why", why) + } + slog.Info("acc_fallback", attrs...) } // checkNames lists the names of the failed library checks, comma-separated, in order. diff --git a/proxy/accountant_state_test.go b/proxy/accountant_state_test.go index c0673343..259e6a64 100644 --- a/proxy/accountant_state_test.go +++ b/proxy/accountant_state_test.go @@ -119,3 +119,25 @@ func TestSavingsEndReusesTheClientHashOnlyForIdenticalBytes(t *testing.T) { t.Fatalf("the reused hash differs from a fresh one") } } + +// The reason no_result stands for four unrelated things. The log line names which one applied, so the next +// first-request-on-a-warm-cache case (review of #768, major 5) names its own cause. +func TestNoResultWhyNamesTheBranch(t *testing.T) { + rec := &accountant.Request{} + for _, tc := range []struct { + name string + c capture + ok bool + want string + }{ + {"no record", capture{}, true, "no_record"}, + {"no usage", capture{accReq: rec}, false, "no_usage"}, + {"expand rounds", capture{accReq: rec, rounds: 3}, true, "upstream_rounds=3"}, + {"compute panicked", capture{accReq: rec, rounds: 1, accNoResult: "compute_panicked"}, true, "compute_panicked"}, + {"nothing wrong", capture{accReq: rec, rounds: 1}, true, ""}, + } { + if got := tc.c.noResultWhy(tc.ok); got != tc.want { + t.Errorf("%s: why = %q, want %q", tc.name, got, tc.want) + } + } +} diff --git a/proxy/dashcapture.go b/proxy/dashcapture.go index c0b9eebf..0083d109 100644 --- a/proxy/dashcapture.go +++ b/proxy/dashcapture.go @@ -3,6 +3,7 @@ package proxy import ( "context" "net/http" + "strconv" "strings" "time" @@ -130,6 +131,8 @@ type capture struct { accReq *accountant.Request // accUnsupportedWire is set for a request on a wire the accountant does not read (OpenAI Responses). accUnsupportedWire bool + // accNoResult says why no Result was attached when Compute was reached and gave none. + accNoResult string // accClient is the body the agent sent, kept for accEnd's envelope diff (envelopeOps). accClient []byte // accThread is the thread id accEnd gave the library, the second half of the ping store's key. @@ -727,6 +730,8 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content if c.rounds <= 1 { if res, ok := c.savings.computeSafe(c.accReq, usage); ok { e.SetAccountant(res) + } else { + c.accNoResult = "compute_panicked" } // The ping facts were read for this request; now that it is priced they are spent. c.pings.drop(accPingKey(c.accReq.Meta.Session, c.accThread), c.accPingWindow) @@ -738,7 +743,7 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content } e.PriceMode(price, usageOK && priced) if (c.accReq != nil || c.accUnsupportedWire) && e.AccountantFallback() != "" { - logFallback(e) + logFallback(e, c.noResultWhy(usageOK)) } if cgIn > 0 || cgOut > 0 || cgCacheWrite > 0 || cgCacheRead > 0 { cgPrice, cgPriced := price, priced @@ -852,3 +857,19 @@ func (c *capture) accDrop() { c.accReq = nil } } + +// noResultWhy names the branch that left an event with no accountant Result, for the acc_fallback line. The reason +// no_result covers four unrelated things, and the log line used to say only that. +func (c *capture) noResultWhy(usageOK bool) string { + switch { + case c.accReq == nil: + return "no_record" // no session, or the pipeline did not run: accBegin never opened a record + case !usageOK: + return "no_usage" + case c.rounds > 1: + return "upstream_rounds=" + strconv.Itoa(c.rounds) + case c.accNoResult != "": + return c.accNoResult + } + return "" +} From a311defabcbd6d69d6d7c8541fd2be9f37465ad4 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 19:13:05 +0200 Subject: [PATCH 24/58] fix(proxy): Review fixes for the accountant integration (round 1 of #893) Both reviewers found the same three blocking problems. - accBodies kept the library's cumulative byte table, built from the mark-free input, while Pos.Bytes held the wire size. A splice that removed a message with a cache mark failed the sum check (sawOriginal compares the operation's wire SizeBefore with that table), so the request booked no saving. The Body is now built again from its exported fields, which drops the table; every byte count the library reads is the wire size, as the operations' sizes are. The test passes with wire bytes hashed, fails with the old Body, and passes now. - The client hash begin keeps for end leaked on the recovered panic before the forward. accDrop and finish now drop it. - docs/data-dependence.md said the hashes are held in memory only; they are also saved to the state file. The row says so. Also: the replay cap path clamps the replay multiple at 0 as the read path does; undecorate strips a version suffix only from a vendor-prefixed id; the long-context tier note names the models; two proxies must not share a dashboard database. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/replaycap.go | 2 +- dash/replaycap_test.go | 15 ++++++++++++ docs/dashboard.md | 8 ++++--- docs/data-dependence.md | 2 +- internal/modelinfo/litellm/chain_test.go | 3 ++- internal/modelinfo/litellm/litellm.go | 11 ++++++--- proxy/accountant_savings.go | 8 +++++++ proxy/accountant_savings_body.go | 13 ++++++---- proxy/accountant_savings_test.go | 30 ++++++++++++++++++++++++ proxy/accountant_state_test.go | 18 ++++++++++++++ proxy/dashcapture.go | 4 ++++ 11 files changed, 101 insertions(+), 13 deletions(-) diff --git a/dash/replaycap.go b/dash/replaycap.go index a08f907a..e535f071 100644 --- a/dash/replaycap.go +++ b/dash/replaycap.go @@ -496,7 +496,7 @@ func applyReplayLedgerToRow(c *ComponentRow, l *ReplayLedger) { c.NetUSDFirstRemoval = c.SavedUSDFirstRemoval + c.SavedUSDCosts - c.LLMCostUSD c.ReplayMultiple = 0 if c.SavedUSDFirstRemoval > 0 { - c.ReplayMultiple = c.SavedUSDDecomposed / c.SavedUSDFirstRemoval + c.ReplayMultiple = math.Max(c.SavedUSDDecomposed, 0) / c.SavedUSDFirstRemoval } c.ReplayCreditPastCapUSD = cl.PastCapUSD } diff --git a/dash/replaycap_test.go b/dash/replaycap_test.go index a597a094..942a782d 100644 --- a/dash/replaycap_test.go +++ b/dash/replaycap_test.go @@ -704,3 +704,18 @@ func TestReplayCapStillRemovesAPositiveCreditPastTheCap(t *testing.T) { nearEq(t, "past cap", c.PastCapUSD, 4) nearEq(t, "counted", c.CountedUSD, -1) // +1 counted, -2 kept } + +// After the cap removes credit, a component whose costs exceed what is left must not print a gain: its replay +// multiple is the amortized verdict over the first-removal credit, and a negative verdict is 0, as in +// decomposeFromGroups (review of #893). +func TestReplayLedgerRowKeepsTheReplayMultipleAtZeroForALoss(t *testing.T) { + row := &ComponentRow{Component: "extract_llm", SavedUSDFirstRemoval: 1, SavedUSDReplay: 3, SavedUSDCosts: -9} + l := walkReplayCap([]capRow{ + capTestRow(1, 100_000, 50_000, 400_000, "extract_llm", 1), + capTestRow(2, 110_000, 500_000, 500_000, "extract_llm", 3), + }, fixedWindow(1_000_000)) + applyReplayLedgerToRow(row, l) + if row.ReplayMultiple < 0 { + t.Fatalf("ReplayMultiple = %g for a loss, want 0", row.ReplayMultiple) + } +} diff --git a/docs/dashboard.md b/docs/dashboard.md index 42d610d6..5185fa95 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -152,13 +152,15 @@ Limits: one: the proxy then starts empty, as before. A crash (kill -9) loses up to one minute of state. A request that follows a lost state is priced as a first request and carries the estimate flag. The library also drops the least recently used threads past its own bound; that is not saved or restored. + Two proxies must not share one dashboard database: they would write the same state file and the last writer wins. - A thread that continues another by content (a rewind, a fork) is priced as a first request. The proxy passes the parent thread to the library, but the library does not read it, so the new thread does not start from a copy of the parent's cache state (accountant design §4.1). - Every request is priced at the model's base rates. A model whose provider charges more above a prompt size - (LiteLLM's `*_above_100k_tokens` and `*_above_200k_tokens` fields, for example `claude-haiku-5-5` above 100k - and `claude-sonnet-4-5` above 200k) is under-priced on those requests: its cost and its saving both use the - base rate, so their ratio looks right and the dollars are low. Only a few Claude ids carry such a tier. + (LiteLLM's `*_above_100k_tokens` and `*_above_200k_tokens` fields) is under-priced on those requests: its + cost and its saving both use the base rate, so their ratio looks right and the dollars are low. Among Claude + models, `claude-haiku-5-5` carries a 100k tier (input at 5x) and `claude-sonnet-4` and `claude-sonnet-4-5` a + 200k tier (2x); Sonnet 5, Opus and Fable carry none. Models of other families carry tiers too. `MODEL_PRICES` sets one flat rate for a model, so it does not fix a tiered price. - The cache lifetime comes from the provider rule the cold gate uses (`apply.CacheLifetime`): an exact expiry for Anthropic, a minimum for OpenAI's 30-minute entries. Where it is unknown, the diff --git a/docs/data-dependence.md b/docs/data-dependence.md index 46b3040f..7c2bca9d 100644 --- a/docs/data-dependence.md +++ b/docs/data-dependence.md @@ -226,7 +226,7 @@ One row for each registered component. The test `components/all/datadep_test.go` | Thread tracker | metadata | 🟡 Works, metadata | 🟡 Works, metadata | 🟡 Works, metadata | It tells the main agent from its subagents. It uses hashes to check if a new request continues an earlier one. | | Session tracker | metadata | 🟡 Works, metadata | 🟡 Works, metadata | 🟡 Works, metadata | It needs a stable label for each conversation. When the client sends none, it hashes the start of the conversation. | | Cache tracker | metadata | 🟡 Works, metadata | 🟡 Works, metadata | 🟡 Works, metadata | It must know how much of the conversation the provider already cached, so components never edit that part. After a restart it asks the dashboard (when on) for the time each thread's cached prefix was last refreshed — by its own last request or a keep-alive ping that read the cache, under either session id the conversation uses (the row stores the content-derived alias id, a hash like the session id itself); timestamps and ids, metadata only — and treats a thread refreshed inside the request's cache lifetime as cached, not as new, until that thread's first request. Without the dashboard — and for a tenant running observe mode, whose rows carry no session id — it uses the proxy's start time instead. | -| Accountant savings (per-component figures from the accountant library) | metadata | ❌ Off. It needs the dashboard, because the figure is attached to a dashboard event. | 🟡 Works, metadata | 🟡 Works, metadata | To price a saving it must know which part of the conversation a rewrite changed and whether the provider had that part cached. It reads each request once and keeps only a hash and a size for each part, plus the cache breakpoints. The text is dropped as soon as it is hashed, and the hashes are held in memory only, one set per conversation thread, plus each thread's last set for the sibling check (SharedWarm). It reads the tools and system sections the same way, as names, hashes and sizes, to report the envelope changes. The result it writes to the dashboard is dollar amounts for each component, with their split by tier. No level changes what it does. | +| Accountant savings (per-component figures from the accountant library) | metadata | ❌ Off. It needs the dashboard, because the figure is attached to a dashboard event. | 🟡 Works, metadata | 🟡 Works, metadata | To price a saving it must know which part of the conversation a rewrite changed and whether the provider had that part cached. It reads each request once and keeps only a hash and a size for each part, plus the cache breakpoints. The text is dropped as soon as it is hashed, and the hashes are held in memory, one set per conversation thread, plus each thread's last set for the sibling check (SharedWarm). So a restart does not lose them, it also saves them to one file beside the dashboard database (`.accountant-state.json`, readable by its owner only), every minute and when it stops, and loads them again when it starts. The file holds hashes, sizes, positions, and the ids and names the state needs (session and thread ids, tool and component names, model names, times). It holds no message text. It is ignored when it is more than one hour old. It is not written when there is no dashboard. It reads the tools and system sections the same way, as names, hashes and sizes, to report the envelope changes. The result it writes to the dashboard is dollar amounts for each component, with their split by tier. No level changes what it does. | | Wait-cause and lifecycle signals | metadata | ❌ Off (it needs the dashboard) | 🟡 Works, metadata | 🟡 Works, metadata | To tell why the agent paused, it reads the request once. It keeps only counts and ids. Since #857 it also reads the **tool name** of the last tool call, to tell that an agent has finished (`done`). The keep-alive then sends no pings for that thread. A tool name is metadata, and the rule works at every storage level. Since #858 the keep-alive also uses the **wait cause** (from the stop reason, the tool names and the open-task counts), whether the thread id starts with `a:`, the token count of the prefix and the time of day, to pick the ping count, to ignore a strategy's time windows for a wait that is not for a person, and to skip the sweep question during a tool wait. It also keeps one flag per thread: "has a tool call been seen". All of it is metadata, and it works at every storage level. | | Costly settings check (`/context-guru:save-more settings`, `GET /api/adapt/settings`) | metadata | ❌ Off (the cost needs the dashboard). The plugin still finds the settings and says "cost not measured yet". | 🟡 Works, metadata | 🟡 Works, metadata | It counts and prices dashboard rows by request shape: model name, tool count, `max_tokens` and the stream flag. One check also uses the upstream status (400) of a Haiku row in the same session. It reads no request text and keeps nothing new. The plugin reads only the named keys of the user's own settings files (the betas setting, the recap setting, `ANTHROPIC_SMALL_FAST_MODEL` and `ANTHROPIC_DEFAULT_HAIKU_MODEL`) on the user's machine, and sends none of them to the proxy (#615). | | Beep (`beep.sh`, the `beep` plugin option) | none | 🟡 Works, no data | 🟡 Works, no data | 🟡 Works, no data | It plays a sound when Claude finishes or waits for you. It reads no stored data and never reads the hook payload. It sends nothing to the proxy or anywhere else. The level changes nothing. | diff --git a/internal/modelinfo/litellm/chain_test.go b/internal/modelinfo/litellm/chain_test.go index 4c9bb2eb..bdc37511 100644 --- a/internal/modelinfo/litellm/chain_test.go +++ b/internal/modelinfo/litellm/chain_test.go @@ -291,7 +291,8 @@ func TestLiteLLMPriceFindsDatedAndWrappedIDs(t *testing.T) { } } // A different model must not be mistaken for it: the date and the wrapper are stripped, nothing else. - for _, id := range []string{"claude-sonnet-5-5-20260101", "us.anthropic.totally-absent-xyz-v1:0"} { + // A plain id that ends in a version is a model of its own, not Bedrock's suffix. + for _, id := range []string{"claude-sonnet-5-5-20260101", "us.anthropic.totally-absent-xyz-v1:0", "claude-sonnet-5-v2"} { if p, ok := l.Price(ctx, id); ok && p == want { t.Errorf("Price(%q) resolved to claude-sonnet-5's price", id) } diff --git a/internal/modelinfo/litellm/litellm.go b/internal/modelinfo/litellm/litellm.go index 447565dc..38bc5bd4 100644 --- a/internal/modelinfo/litellm/litellm.go +++ b/internal/modelinfo/litellm/litellm.go @@ -337,8 +337,13 @@ var ( // undecorate removes a dated or Bedrock-wrapped id's decoration: the vendor prefix, the version suffix and the // release date. The result is looked up as an exact key, so a different model (claude-sonnet-5-5 against // claude-sonnet-5) is never reached by it. +// +// The version suffix (-v1:0) is stripped only from an id that carried a vendor prefix: that is Bedrock's form. A +// plain id such as claude-x-v2 may be a model of its own, and would take the price of claude-x. func undecorate(id string) string { - id = decoVendor.ReplaceAllString(id, "") - id = decoBedrock.ReplaceAllString(id, "") - return decoDate.ReplaceAllString(id, "") + bare := decoVendor.ReplaceAllString(id, "") + if bare != id { + bare = decoBedrock.ReplaceAllString(bare, "") + } + return decoDate.ReplaceAllString(bare, "") } diff --git a/proxy/accountant_savings.go b/proxy/accountant_savings.go index 090e9e1d..4220307e 100644 --- a/proxy/accountant_savings.go +++ b/proxy/accountant_savings.go @@ -27,6 +27,14 @@ type clientHash struct { bps []accountant.Breakpoint } +// drop forgets the client hash of a request that will never end: a panic between accBegin and accEnd makes the proxy +// send the original body, and end is the only other place that removes the entry. Nil-safe. +func (s *savingsTracker) drop(req *accountant.Request) { + if s != nil && req != nil { + s.clients.Delete(req) + } +} + func newSavingsTracker() *savingsTracker { return &savingsTracker{trackers: accountant.NewTrackers(), siblings: newSiblingStore()} } diff --git a/proxy/accountant_savings_body.go b/proxy/accountant_savings_body.go index 924119c5..00f86a56 100644 --- a/proxy/accountant_savings_body.go +++ b/proxy/accountant_savings_body.go @@ -38,11 +38,16 @@ func accBodies(raw []byte) accountant.Body { add(&system, s) } gjson.GetBytes(raw, "messages").ForEach(func(_, v gjson.Result) bool { add(&msgs, v); return true }) - b := accountant.NewBody(tools, system, msgs) - for i := range b.Pos { - b.Pos[i].Bytes = int32(sizes[i]) + built := accountant.NewBody(tools, system, msgs) + for i := range built.Pos { + built.Pos[i].Bytes = int32(sizes[i]) } - return b + // NewBody also keeps a table of cumulative bytes, built from the mark-free input, which the library prefers to + // summing Pos[i].Bytes. A Body without that table sums the positions, so every byte count the library reads + // (the token ratio, the span of a splice, sawOriginal) is the wire size, as the operations' sizes are. A body + // decoded from a snapshot has no table either, so a restart changes nothing. The table is unexported: build the + // value again from the exported fields, which drops it (review of #893, finding 2). + return accountant.Body{Start: built.Start, Pos: built.Pos} } // withoutCacheMarks is one element's JSON without any cache_control: the element's own, and the ones on diff --git a/proxy/accountant_savings_test.go b/proxy/accountant_savings_test.go index 800bebe4..ea15ac96 100644 --- a/proxy/accountant_savings_test.go +++ b/proxy/accountant_savings_test.go @@ -347,3 +347,33 @@ func TestComputeSafeTurnsAPanicIntoNoResult(t *testing.T) { t.Fatalf("a normal request did not price through computeSafe: ok=%v phase=%q", ok, res.Phase) } } + +// A summary splices away messages and one of them carries the client's cache mark. The splice's SizeBefore is the +// wire size (marks included), and the library compares it with the client span's bytes to guess whether the splice +// saw the original (sawOriginal). If the Body's byte table leaves the marks out while Pos.Bytes keeps them, the +// guess is wrong and the sum check fails: the request books no saving (review of #893, finding 2). +func TestSavings_ASpliceOverAMarkedMessageKeepsTheSumCheck(t *testing.T) { + c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} + long := strings.Repeat("tool result ", 300) + m0, n0 := msg("user", long) + m1, n1 := msg("assistant", long) + masked, nMasked := msg("assistant", "[masked]") + m2raw := fmt.Sprintf(`{"role":"user","content":[{"type":"text","text":%q,"cache_control":{"type":"ephemeral"}}]}`, long) + n2 := len(m2raw) + m3, _ := msg("assistant", "answer") + sum, nSum := msg("user", "[summary]") + + // Request 1: a mask reduces message 1. + c.do(1, body(m0, m1, m2raw, m3), body(m0, masked, m2raw, m3), Usage{CacheWrite: 1800}, + rewriteOp("cg:mask:a", 1, n1, nMasked)) + // Request 2: a summary replaces messages 0 to 2 with one. Its SizeBefore counts the wire bytes of those + // three messages as the pipeline saw them: message 1 is already masked. + splice := accountant.Operation{Kind: accountant.KindRewrite, Operator: "context-guru.summarize", ChangeID: "cg:sum:a", + At: accountant.Position{Section: accountant.SectionMessages, Index: 0}, + SizeBefore: accountant.Size{Bytes: n0 + nMasked + n2}, SizeAfter: accountant.Size{Bytes: nSum}, + MessageSplice: &accountant.MessageSplice{At: 0, Removed: 3, Inserted: 1}} + res := c.do(2, body(m0, m1, m2raw, m3), body(sum, m3), Usage{CacheWrite: 30}, splice) + for _, ch := range res.Checks { + t.Errorf("check %s failed: %+v", ch.Check, ch) + } +} diff --git a/proxy/accountant_state_test.go b/proxy/accountant_state_test.go index 259e6a64..4e192fdd 100644 --- a/proxy/accountant_state_test.go +++ b/proxy/accountant_state_test.go @@ -141,3 +141,21 @@ func TestNoResultWhyNamesTheBranch(t *testing.T) { } } } + +// A panic between accBegin and accEnd makes the proxy forward the original body and call accDrop. The client hash +// begin kept for end must go with it: it holds the whole body (review of #893, finding 1). +func TestAccDropForgetsTheClientHash(t *testing.T) { + s := newSavingsTracker() + held := func() (n int) { s.clients.Range(func(any, any) bool { n++; return true }); return } + meta := accountant.RequestMeta{RequestID: "r1", Session: "s", At: time.Unix(1000, 0), Billing: accountant.Billing{Mode: accountant.BillingAPI}} + c := &capture{savings: s} + c.accBegin(meta, benchBody()) + if held() != 1 { + t.Fatalf("accBegin kept %d client hashes, want 1", held()) + } + c.accDrop() + if held() != 0 || c.accReq != nil { + t.Fatalf("accDrop left %d client hashes and accReq %v", held(), c.accReq) + } + (*capture)(nil).accDrop() // nil-safe, as before +} diff --git a/proxy/dashcapture.go b/proxy/dashcapture.go index 0083d109..2ab23ed9 100644 --- a/proxy/dashcapture.go +++ b/proxy/dashcapture.go @@ -725,6 +725,9 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content // (acc_fallback reason no_result). The thread state is left as the // previous request left it; the next request then sees a longer gap, which the library // handles (a phase it cannot prove is unknown and falls back too). + // Whatever way the request ended, the client hash begin kept for end is not needed past here. end removes it + // on the normal path and accDrop on a recovered panic; this covers any path that reached neither. + defer c.savings.drop(c.accReq) if c.accReq != nil && usageOK && !c.accDone { c.accDone = true if c.rounds <= 1 { @@ -854,6 +857,7 @@ func (c *capture) noteRounds(n int) { // accDrop forgets the savings record: the pipeline's rewrite did not go upstream. Nil-safe. func (c *capture) accDrop() { if c != nil { + c.savings.drop(c.accReq) c.accReq = nil } } From a3c815ff700f59226310a4dd97e17d7c804f2f30 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 19:25:30 +0200 Subject: [PATCH 25/58] fix: Review notes of round 2 on #893 - undecorate also strips the au., jp. and us-gov. region prefixes of a Bedrock id, which LiteLLM's own keys carry. - The long-context note says where Sonnet 4 and 4.5 carry a tier: on the Bedrock, Vertex and Azure listings, not on the Anthropic API ids. - The comment of the splice test names the check that fires (conservation). Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- docs/dashboard.md | 5 +++-- internal/modelinfo/litellm/chain_test.go | 3 +++ internal/modelinfo/litellm/litellm.go | 2 +- proxy/accountant_savings_test.go | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/dashboard.md b/docs/dashboard.md index 5185fa95..e79002dc 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -159,8 +159,9 @@ Limits: - Every request is priced at the model's base rates. A model whose provider charges more above a prompt size (LiteLLM's `*_above_100k_tokens` and `*_above_200k_tokens` fields) is under-priced on those requests: its cost and its saving both use the base rate, so their ratio looks right and the dollars are low. Among Claude - models, `claude-haiku-5-5` carries a 100k tier (input at 5x) and `claude-sonnet-4` and `claude-sonnet-4-5` a - 200k tier (2x); Sonnet 5, Opus and Fable carry none. Models of other families carry tiers too. + models, `claude-haiku-5-5` carries a 100k tier (input at 5x) on every listing, and Sonnet 4 and 4.5 carry a + 200k tier (2x) on their Bedrock, Vertex and Azure listings but not on the Anthropic API ids; Sonnet 5, Opus and + Fable carry none. Models of other families carry tiers too. `MODEL_PRICES` sets one flat rate for a model, so it does not fix a tiered price. - The cache lifetime comes from the provider rule the cold gate uses (`apply.CacheLifetime`): an exact expiry for Anthropic, a minimum for OpenAI's 30-minute entries. Where it is unknown, the diff --git a/internal/modelinfo/litellm/chain_test.go b/internal/modelinfo/litellm/chain_test.go index bdc37511..a3e8a0f5 100644 --- a/internal/modelinfo/litellm/chain_test.go +++ b/internal/modelinfo/litellm/chain_test.go @@ -284,6 +284,9 @@ func TestLiteLLMPriceFindsDatedAndWrappedIDs(t *testing.T) { "anthropic.claude-sonnet-5-20260101-v2:0", "eu.anthropic.claude-sonnet-5-v1:0[1m]", "claude-sonnet-5@20260101", + "au.anthropic.claude-sonnet-5-v1:0", + "jp.anthropic.claude-sonnet-5-v1:0", + "us-gov.anthropic.claude-sonnet-5-v1:0", } { got, ok := l.Price(ctx, id) if !ok || got != want { diff --git a/internal/modelinfo/litellm/litellm.go b/internal/modelinfo/litellm/litellm.go index 38bc5bd4..81bfb910 100644 --- a/internal/modelinfo/litellm/litellm.go +++ b/internal/modelinfo/litellm/litellm.go @@ -331,7 +331,7 @@ var ( // decoBedrock is Bedrock's version suffix: -v1:0, -v2. decoBedrock = regexp.MustCompile(`-v\d+(:\d+)?$`) // decoVendor is the dotted vendor prefix of a Bedrock or Vertex id: us.anthropic., eu.anthropic., anthropic. - decoVendor = regexp.MustCompile(`^(?:(?:us|eu|apac|global)\.)?(?:anthropic|amazon|meta|mistral|cohere)\.`) + decoVendor = regexp.MustCompile(`^(?:(?:us|eu|apac|au|jp|us-gov|global)\.)?(?:anthropic|amazon|meta|mistral|cohere)\.`) ) // undecorate removes a dated or Bedrock-wrapped id's decoration: the vendor prefix, the version suffix and the diff --git a/proxy/accountant_savings_test.go b/proxy/accountant_savings_test.go index ea15ac96..667e5778 100644 --- a/proxy/accountant_savings_test.go +++ b/proxy/accountant_savings_test.go @@ -351,7 +351,8 @@ func TestComputeSafeTurnsAPanicIntoNoResult(t *testing.T) { // A summary splices away messages and one of them carries the client's cache mark. The splice's SizeBefore is the // wire size (marks included), and the library compares it with the client span's bytes to guess whether the splice // saw the original (sawOriginal). If the Body's byte table leaves the marks out while Pos.Bytes keeps them, the -// guess is wrong and the sum check fails: the request books no saving (review of #893, finding 2). +// byte counts disagree and the library fails a check (conservation here: its client token total is built from +// mark-free bytes, the operations' deltas from wire bytes): the request books no saving (review of #893, finding 2). func TestSavings_ASpliceOverAMarkedMessageKeepsTheSumCheck(t *testing.T) { c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} long := strings.Repeat("tool result ", 300) From 851ad21bd772ade366a0323851423b942990c2f2 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 06:39:49 +0200 Subject: [PATCH 26/58] revert: Revert the revert of the accountant pricing of proxy savings (#768) This reverts commit 38c00e137e8bf1400b674569e10ff96016f66a04 (#903). It brings back #768 on top of current main. Conflicts in keepalive/keeper.go and keepalive/manual.go: main added touchAlias (#904) and #768 adds gapStart to the same struct literals. Both fields are kept. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- apply/apply.go | 87 +- apply/ops.go | 280 ++++ apply/ops_integration_test.go | 44 + apply/ops_phantom_test.go | 82 + apply/ops_restore_test.go | 77 + apply/ops_splice_test.go | 59 + apply/ops_test.go | 197 +++ apply/ops_wiresize_test.go | 154 ++ .../dashemitter_savings_test.go | 63 + cmd/context-guru-proxy/main.go | 16 +- components/component.go | 28 + components/pipeline.go | 92 +- components/pipeline_changes_test.go | 201 +++ dash/accountant_agree_helpers_test.go | 60 + dash/accountant_agree_test.go | 218 +++ dash/accountant_event.go | 28 + dash/accountant_price.go | 358 +++++ dash/accountant_price_test.go | 281 ++++ dash/accountant_switch_test.go | 340 ++++ dash/accounting_test.go | 49 +- dash/adaptcompaction.go | 43 +- dash/adaptcompaction_test.go | 35 + dash/billedfactor_test.go | 81 - dash/cachecost_test.go | 120 +- dash/capture.go | 18 +- dash/capture_test.go | 104 +- dash/compactepisode.go | 22 +- dash/event.go | 324 +--- dash/honesty_test.go | 71 +- dash/keepalive.go | 8 +- dash/keepalive_ttl_test.go | 23 - dash/keepalivecost_test.go | 75 - dash/openreadonly_test.go | 34 + dash/overview.go | 10 +- dash/query.go | 33 +- dash/readvalue.go | 29 +- dash/replaycap.go | 18 +- dash/schema.go | 23 +- dash/store.go | 60 +- dash/summarizeattr.go | 14 +- dash/summarizeattr_test.go | 47 +- dash/summarizercomponents_test.go | 20 - dash/sweepattr_test.go | 15 +- dash/testdata/accountant/README.md | 25 + dash/ui/app.js | 8 +- docs/accountant/cg-differences-root-causes.md | 194 +++ docs/accountant/cg-differences.md | 343 ++++ docs/dashboard.md | 78 + docs/data-dependence.md | 1 + .../2026-10-28-proxy-accountant-savings.md | 1394 +++++++++++++++++ ...6-10-28-proxy-accountant-savings-design.md | 128 ++ go.mod | 3 + go.sum | 6 + internal/modelinfo/cachefrac.go | 5 +- internal/modelinfo/cachefrac_test.go | 2 +- internal/modelinfo/modelinfo.go | 1 + keepalive/keeper.go | 19 +- keepalive/manual.go | 5 +- keepalive/manual_test.go | 73 + keepalive/substitute.go | 1 + keepalive/substitute_test.go | 21 + keepalive/sweeptextturn_test.go | 31 + mkdocs.yml | 2 + proxy/accountant_envelope.go | 283 ++++ proxy/accountant_envelope_test.go | 230 +++ proxy/accountant_import_test.go | 13 + proxy/accountant_inject_test.go | 104 ++ proxy/accountant_ping.go | 101 ++ proxy/accountant_ping_test.go | 302 ++++ proxy/accountant_presets_test.go | 90 ++ proxy/accountant_rounds_test.go | 56 + proxy/accountant_savings.go | 46 + proxy/accountant_savings_body.go | 85 + proxy/accountant_savings_body_test.go | 97 ++ proxy/accountant_savings_test.go | 220 +++ proxy/accountant_shadow.go | 27 + proxy/accountant_shadow_test.go | 42 + proxy/accountant_shared.go | 153 ++ proxy/accountant_shared_test.go | 112 ++ proxy/accountant_wire.go | 88 ++ proxy/accountant_wire_test.go | 238 +++ proxy/dash_test.go | 80 +- proxy/dashcapture.go | 68 +- proxy/proxy.go | 38 + proxy/savings_test.go | 2 +- 85 files changed, 7847 insertions(+), 909 deletions(-) create mode 100644 apply/ops.go create mode 100644 apply/ops_integration_test.go create mode 100644 apply/ops_phantom_test.go create mode 100644 apply/ops_restore_test.go create mode 100644 apply/ops_splice_test.go create mode 100644 apply/ops_test.go create mode 100644 apply/ops_wiresize_test.go create mode 100644 cmd/context-guru-proxy/dashemitter_savings_test.go create mode 100644 components/pipeline_changes_test.go create mode 100644 dash/accountant_agree_helpers_test.go create mode 100644 dash/accountant_agree_test.go create mode 100644 dash/accountant_event.go create mode 100644 dash/accountant_price.go create mode 100644 dash/accountant_price_test.go create mode 100644 dash/accountant_switch_test.go delete mode 100644 dash/billedfactor_test.go create mode 100644 dash/testdata/accountant/README.md create mode 100644 docs/accountant/cg-differences-root-causes.md create mode 100644 docs/accountant/cg-differences.md create mode 100644 docs/superpowers/plans/2026-10-28-proxy-accountant-savings.md create mode 100644 docs/superpowers/specs/2026-10-28-proxy-accountant-savings-design.md create mode 100644 proxy/accountant_envelope.go create mode 100644 proxy/accountant_envelope_test.go create mode 100644 proxy/accountant_import_test.go create mode 100644 proxy/accountant_inject_test.go create mode 100644 proxy/accountant_ping.go create mode 100644 proxy/accountant_ping_test.go create mode 100644 proxy/accountant_presets_test.go create mode 100644 proxy/accountant_rounds_test.go create mode 100644 proxy/accountant_savings.go create mode 100644 proxy/accountant_savings_body.go create mode 100644 proxy/accountant_savings_body_test.go create mode 100644 proxy/accountant_savings_test.go create mode 100644 proxy/accountant_shadow.go create mode 100644 proxy/accountant_shadow_test.go create mode 100644 proxy/accountant_shared.go create mode 100644 proxy/accountant_shared_test.go create mode 100644 proxy/accountant_wire.go create mode 100644 proxy/accountant_wire_test.go diff --git a/apply/apply.go b/apply/apply.go index e66b83a7..1daa4363 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -32,6 +32,7 @@ import ( "unicode/utf8" bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/accountant" "github.com/rossoctl/context-guru/components" "github.com/rossoctl/context-guru/internal/logging" "github.com/rossoctl/context-guru/internal/modelinfo" @@ -292,6 +293,10 @@ type Trace struct { Restored, RestoredTokens int // Run is the pipeline's aggregate report (nil when the pipeline never ran). Run *components.RunReport + // Ops are the rewrite operations of this request for the savings computation, in pipeline + // order, with wire positions. OpsDropped counts changes that could not be placed. + Ops []accountant.Operation + OpsDropped int // Changes lists each rewritten message's before/after text (clipped). Changes []Change } @@ -470,8 +475,9 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o // than re-parse the whole messages array to find each message again — that parse // measured ~3x the cost of the whole-body copy it was meant to save. shiftAt, shift := 0, 0 - // splitStableTokens is what the split rescued from the cache-creation tier; the - // dashboard prices it (see dash.Event.cachesplitSavedUSD). 0 when nothing split. + // splitStableTokens is the size of the stable half the split moved the breakpoint onto, recorded + // on the dashboard row. The split's saving is the accountant library's split effect. 0 when + // nothing split. splitStableTokens := 0 // splitTailHash identifies the volatile half, so a later turn can tell whether the // snapshot MOVED. Only on such a turn would the unsplit block have been re-created. @@ -806,6 +812,18 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o } rr := pipe.Run(chat, c) tr.Run = rr + // Operations are published only once the writeback has settled and we know which rewrites + // reached the forwarded body (Trace.Ops never describes a rewrite that is not on the wire). + // applied: the rewritten body is the one forwarded, which is every mode but observe. + wire, applied := wireGroups(slots), mode != components.ModeObserve + // giveUp is every return that forwards the original body: no operation is real. + giveUp := func() { + tr.Ops = nil + if rr != nil { + ops, d := opsFromRun(*rr, wire, applied, nil) + tr.OpsDropped = len(ops) + d + } + } if debug && rr != nil { logDecisions(lg, rr) } @@ -829,6 +847,15 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o // OpenAI-shaped body has neither, and searching it would count every splice as not found. nb = anchorCacheBreakpoint(nb, c.CacheAnchorToolID) } + if ok && rr != nil { + tr.Ops, tr.OpsDropped = OpsFromRun(*rr, wire, applied) + outMsgs := messagesArray(nb).Array() + if !wireSizes(tr.Ops, rawLen(msgs), rawLen(outMsgs), rawTotal(msgs), rawTotal(outMsgs)) { + giveUp() + } + } else { + giveUp() + } if !ok && systemSplit { res.Body, res.Changed = body, true // keep the split even when the rebuild declined return res @@ -884,6 +911,8 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o } return bi, rel, cur, true } + // written: the req.Input indices whose content change really went into an edit. + written := map[int]bool{} for i := range chat.Input { s := slots[i] switch s.kind { @@ -900,15 +929,18 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o bi, rel, cur, ok := editOf(s.path, s.raw) if !ok { res.Body = body + giveUp() return res } nb, err := sjson.SetBytes(cur, rel, newText) if err != nil { res.Body = body + giveUp() return res } edits[bi] = nb changed = true + written[i] = true changes = append(changes, mkChange(s.path, s.preText, newText, touched[i])) default: // wholeMessage if _, was := touched[i]; trustTouched && !was { @@ -917,6 +949,7 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o post, err := json.Marshal(chat.Input[i]) if err != nil { res.Body = body + giveUp() return res } if bytes.Equal(post, s.pre) { @@ -925,6 +958,7 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o bi, _, cur, ok := editOf(s.path, s.raw) if !ok { res.Body = body + giveUp() return res } if !s.lossless() { @@ -950,6 +984,7 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o // that splice, at the cost of copying the whole body. edits[bi] = post changed = true + written[i] = true var pm bschemas.ChatMessage _ = json.Unmarshal(s.pre, &pm) changes = append(changes, @@ -976,10 +1011,34 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o nb, ok := spliceMessages(body, msgs, edits, shiftAt, shift) if !ok { res.Body = body + giveUp() return res } out = nb } + if rr != nil { + tr.Ops, tr.OpsDropped = opsFromRun(*rr, wire, applied, func(idx int) bool { return written[idx] }) + // The forwarded messages are the inbound ones with each edit swapped in (spliceMessages). + outLen := func(i int) (int, bool) { + if e, ok := edits[i]; ok { + return len(e), true + } + return rawLen(msgs)(i) + } + inTotal, outTotal := rawTotal(msgs), rawTotal(msgs) + for i, e := range edits { + if i >= 0 && i < len(msgs) { + outTotal += len(e) - len(msgs[i].Raw) + } + } + // outLen is keyed by the INPUT index, and wireSizes asks it for the OUTPUT index. They differ + // when a splice inserts a different count than it removed, which this equal-count writeback + // can carry when two count-changing components cancel out. Sizes taken from the wrong + // message would be published with no check able to catch them, so give up here. + if shiftingSplice(tr.Ops) || !wireSizes(tr.Ops, rawLen(msgs), outLen, inTotal, outTotal) { + giveUp() + } + } tr.Changes = changes if changed && dumpPath != "" { dumpChanges(c.Session, changes) @@ -2056,6 +2115,25 @@ func splitSlotPath(path string) (idx int, rel string, ok bool) { return i, rel, true } +// rawLen is the wire length of message i of msgs, for wireSizes. +func rawLen(msgs []gjson.Result) func(i int) (int, bool) { + return func(i int) (int, bool) { + if i < 0 || i >= len(msgs) { + return 0, false + } + return len(msgs[i].Raw), true + } +} + +// rawTotal is the wire length of every message of msgs. +func rawTotal(msgs []gjson.Result) int { + n := 0 + for _, m := range msgs { + n += len(m.Raw) + } + return n +} + // wireGroups builds components.Ctx.WireGroup (#457): for each slot, the wire (body) message // index its path names — i.e. the thing that must never be split by a span boundary. A // wholeMessage slot's path is always "messages." for a DISTINCT i (one wholeMessage slot per @@ -2129,6 +2207,11 @@ func finishRestore(res Result, plan restorePlan, wire string, fallback []byte, c if len(plan) == 0 { return res } + // Either branch below changes the forwarded body after Trace.Ops was computed: the fallback + // forwards the pre-pipeline body, and an insertion shifts the wire index of every later + // message. No operation can be trusted to describe the wire any more, so all are dropped. + res.Trace.OpsDropped += len(res.Trace.Ops) + res.Trace.Ops = nil nb, tokens, ok := plan.insert(wire, res.Body, client) if !ok { res.Body, res.Changed = fallback, true diff --git a/apply/ops.go b/apply/ops.go new file mode 100644 index 00000000..0da12ba0 --- /dev/null +++ b/apply/ops.go @@ -0,0 +1,280 @@ +package apply + +import ( + "strconv" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/components" +) + +// changeID is the accountant ChangeID of one change: cg:::. idx is the +// req.Input index (the slot, the start of a splice), so two identical tool outputs that one +// component rewrites in one request get two ids (unique_change would fire on one id). The index is +// stable across the turns of a transcript that grows by appending, which is the only case in which +// a change stays active: a client edit or compaction at or before the position ends the change in +// the library anyway (doc.go decision 4). The slot index and not the wire index, because several +// slots can share one wire message (an Anthropic user message with several tool_result blocks). +// The hash is of the text as the client sent it (components.changesOf). +func changeID(component string, c components.Change) string { + return "cg:" + component + ":" + strconv.Itoa(c.Idx) + ":" + c.Hash +} + +// OpsFromRun turns a run's per-message changes into accountant operations, in pipeline +// order. The sizes are the components' TEXT sizes; the caller must convert them to wire bytes +// (wireSizes) before the operations reach the library. wire maps a req.Input index to the wire message index (wireGroups). applied is false when +// the rewritten body was not the one forwarded; a report that ran in observe mode is DryRun too, +// whatever applied says: its operations are computed but never sent. +// +// A reverted or skipped report changed nothing upstream and reports nothing. The first report that +// changed the message count carries one splice change, mapped to a MessageSplice operation when it +// covers whole wire messages (else dropped). Every report after it indexes a req.Input that no +// longer lines up with wire, so those changes are dropped, never guessed. +func OpsFromRun(rr components.RunReport, wire []int, applied bool) (ops []accountant.Operation, dropped int) { + return opsFromRun(rr, wire, applied, nil) +} + +// opsFromRun is OpsFromRun plus keep: when non-nil, a change whose req.Input index it rejects is +// counted as dropped. The equal-count writeback passes the set of messages that really reached the +// forwarded body, so an operation never describes a rewrite that is not on the wire. +func opsFromRun(rr components.RunReport, wire []int, applied bool, keep func(idx int) bool) (ops []accountant.Operation, dropped int) { + countChanged := false + for _, r := range rr.Components { + if r.Skipped || r.Reverted { + continue + } + for _, c := range r.Changes { + if c.Splice != nil { + if countChanged { + dropped++ + continue + } + op, ok := spliceOp(r, c, wire, applied) + if !ok { + dropped++ + continue + } + ops = append(ops, op) + continue + } + if wire == nil || countChanged || c.Idx < 0 || c.Idx >= len(wire) || (keep != nil && !keep(c.Idx)) { + dropped++ + continue + } + ops = append(ops, accountant.Operation{ + Kind: accountant.KindRewrite, + Operator: "context-guru." + r.Component, + DryRun: !applied || r.Mode == components.ModeObserve, + At: accountant.Position{Section: accountant.SectionMessages, Index: wire[c.Idx]}, + SizeBefore: accountant.Size{Bytes: c.BytesBefore}, + SizeAfter: accountant.Size{Bytes: c.BytesAfter}, + ChangeID: changeID(r.Component, c), + }) + } + if r.CountChanged { + countChanged = true + } + } + return ops, dropped +} + +// spliceOp maps a run's splice, in req.Input (slot) space, to a MessageSplice operation in wire +// space. It reports false when the range does not cover whole wire messages: several slots can +// share one wire message, and a splice that cuts one in two cannot be expressed in messages. +func spliceOp(r components.Report, c components.Change, wire []int, applied bool) (accountant.Operation, bool) { + sp := c.Splice + n := len(wire) + if wire == nil || sp.At < 0 || sp.Removed < 0 || sp.Inserted < 0 || sp.At+sp.Removed > n { + return accountant.Operation{}, false + } + // A boundary is clean when the slots on either side of it belong to different wire messages. + // The range is [At, At+Removed); an insertion (Removed == 0) is the empty range at At. + if sp.At > 0 && sp.At < n && wire[sp.At-1] == wire[sp.At] { + return accountant.Operation{}, false + } + end := sp.At + sp.Removed + if sp.Removed > 0 && end < n && wire[end-1] == wire[end] { + return accountant.Operation{}, false + } + if !denseAcross(wire, sp.At, end) { + return accountant.Operation{}, false + } + var at, removed int + switch { + case sp.Removed > 0: + at = wire[sp.At] + removed = wire[end-1] - wire[sp.At] + 1 + case sp.At < n: + at = wire[sp.At] + default: + // An append: one past the last wire message, not len(slots). + at = 0 + if n > 0 { + at = wire[n-1] + 1 + } + } + op := accountant.Operation{ + Kind: accountant.KindRewrite, + Operator: "context-guru." + r.Component, + DryRun: !applied || r.Mode == components.ModeObserve, + At: accountant.Position{Section: accountant.SectionMessages, Index: at}, + SizeBefore: accountant.Size{Bytes: c.BytesBefore}, + SizeAfter: accountant.Size{Bytes: c.BytesAfter}, + ChangeID: changeID(r.Component, c), + MessageSplice: &accountant.MessageSplice{At: at, Removed: removed, Inserted: sp.Inserted}, + } + return op, true +} + +// denseAcross reports whether wire has no hole over the slots [at, end) and the pairs bordering +// that range. normalize skips a message it cannot parse and gives it no slot, so wire can read +// {0, 3, 3, 4}: wire messages 1 and 2 are in the body but not in req.Input. A splice that crosses +// or touches such a hole would count wire messages the component never held, and the position of +// what it inserted would be ambiguous, so the caller drops it. Every consecutive pair wire[k-1], +// wire[k] for k in [max(at,1), min(end,len-1)] must differ by 0 or 1. An append at the end has no +// pair to check, so a trailing unparseable message cannot be detected here (known limit). +func denseAcross(wire []int, at, end int) bool { + lo := at + if lo < 1 { + lo = 1 + } + hi := end + if hi > len(wire)-1 { + hi = len(wire) - 1 + } + for k := lo; k <= hi; k++ { + if d := wire[k] - wire[k-1]; d < 0 || d > 1 { + return false + } + } + return true +} + +// wireSizes replaces the text sizes OpsFromRun puts on ops with WIRE-byte sizes: the bytes of the +// JSON message elements, which is what the host measures the bodies in (proxy cfBodies). The +// library's sum check compares the two, so text sizes fail it on every request (escaping, role and +// ids are in the wire bytes and not in the text). +// +// inLen(i) is the length of wire message i of the body the pipeline read, outLen(i) the length of +// wire message i of the body forwarded. inTotal and outTotal are the bytes of all messages of each. +// +// Per wire message, the first operation's SizeBefore is the message before and the last one's +// SizeAfter the message after; with several operations on one message (several components, or +// several slots), the boundaries between them split the wire delta in proportion to each +// operation's text delta, so the chain is contiguous and its total exact. A splice's SizeBefore is +// the bytes of the wire messages it removed, its SizeAfter the bytes of those it inserted. An +// operation on a message a later splice removed keeps an approximate size (its text delta). The +// library does not end it: the net check below fails instead, and every operation is dropped. +// +// It reports false when the sizes do not reconcile: the operations' net change must equal +// inTotal - outTotal. A message that changed with no operation (a metadata-only write, a cache +// anchor, a rewritten survivor of a splice) makes it false, and the caller then drops every +// operation, so the event falls back to the legacy figures rather than publish a figure the bytes +// do not support. +func wireSizes(ops []accountant.Operation, inLen, outLen func(i int) (int, bool), inTotal, outTotal int) bool { + var sp *accountant.MessageSplice + for _, o := range ops { + if o.MessageSplice != nil { + sp = o.MessageSplice + } + } + outIdx := func(i int) int { + switch { + case sp == nil || i < sp.At: + return i + case i >= sp.At+sp.Removed: + return i - sp.Removed + sp.Inserted + } + return -1 // removed by the splice + } + net := 0 + groups := map[int][]int{} // wire index -> op indices, in pipeline order + var order []int + for k := range ops { + o := &ops[k] + if o.MessageSplice != nil { + b, a := 0, 0 + for i := o.MessageSplice.At; i < o.MessageSplice.At+o.MessageSplice.Removed; i++ { + n, ok := inLen(i) + if !ok { + return false + } + b += n + } + for i := o.MessageSplice.At; i < o.MessageSplice.At+o.MessageSplice.Inserted; i++ { + n, ok := outLen(i) + if !ok { + return false + } + a += n + } + o.SizeBefore, o.SizeAfter = accountant.Size{Bytes: b}, accountant.Size{Bytes: a} + net += b - a + continue + } + if _, seen := groups[o.At.Index]; !seen { + order = append(order, o.At.Index) + } + groups[o.At.Index] = append(groups[o.At.Index], k) + } + for _, wi := range order { + ks := groups[wi] + b, ok := inLen(wi) + if !ok { + return false + } + oi := outIdx(wi) + if oi < 0 { + // Removed by the later splice: the library ends these operations (see above). + cur := b + for _, k := range ks { + d := ops[k].SizeBefore.Bytes - ops[k].SizeAfter.Bytes + next := max(cur-d, 0) + ops[k].SizeBefore, ops[k].SizeAfter = accountant.Size{Bytes: cur}, accountant.Size{Bytes: next} + cur = next + } + continue + } + a, ok := outLen(oi) + if !ok { + return false + } + chain(ops, ks, b, a) + net += b - a + } + return net == inTotal-outTotal +} + +// shiftingSplice is true when an operation is a splice that inserts a different number of messages +// than it removed: every message after it moves to a new wire index. +func shiftingSplice(ops []accountant.Operation) bool { + for _, o := range ops { + if o.MessageSplice != nil && o.MessageSplice.Removed != o.MessageSplice.Inserted { + return true + } + } + return false +} + +// chain gives the operations ks, all on one wire message of b bytes before and a bytes after, +// contiguous wire sizes from b to a, splitting b - a in proportion to their text deltas. With no +// text delta at all, the whole wire delta goes to the last operation. +func chain(ops []accountant.Operation, ks []int, b, a int) { + t := 0 + for _, k := range ks { + t += ops[k].SizeBefore.Bytes - ops[k].SizeAfter.Bytes + } + w, cum, cur := b-a, 0, b + for n, k := range ks { + next := a + if n < len(ks)-1 { + next = b + if t != 0 { + cum += ops[k].SizeBefore.Bytes - ops[k].SizeAfter.Bytes + next = b - int(int64(w)*int64(cum)/int64(t)) + } + next = max(next, 0) + } + ops[k].SizeBefore, ops[k].SizeAfter = accountant.Size{Bytes: cur}, accountant.Size{Bytes: next} + cur = next + } +} diff --git a/apply/ops_integration_test.go b/apply/ops_integration_test.go new file mode 100644 index 00000000..183fbec2 --- /dev/null +++ b/apply/ops_integration_test.go @@ -0,0 +1,44 @@ +package apply_test + +import ( + "context" + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/store" +) + +// Two tool_results in ONE wire message (index 2) become two req.Input entries (2 and 3); both +// operations must land on wire index 2, and neither on the req.Input index. +func TestBodyOptsTraceOpsCarryWireIndex(t *testing.T) { + long := strings.Repeat("a long original tool output line that the component will rewrite\\n", 8) + body := []byte(`{"model":"claude-x","messages":[ + {"role":"user","content":"go"}, + {"role":"assistant","content":[{"type":"tool_use","id":"a","name":"R","input":{}},{"type":"tool_use","id":"b","name":"R","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"a","content":"` + long + `"},{"type":"tool_result","tool_use_id":"b","content":"` + long + `x"}]} + ]}`) + for _, tc := range []struct { + mode components.Mode + dryRun bool + }{{components.ModeSync, false}, {components.ModeObserve, true}} { + p, err := pipe(t, "pipeline: [attr2]\n").Build(nil) + if err != nil { + t.Fatal(err) + } + res := apply.BodyOpts(context.Background(), p, store.NewMemory(store.Options{}), apply.Opts{ + Provider: bschemas.Anthropic, Body: body, Mode: tc.mode, + }) + if len(res.Trace.Ops) == 0 { + t.Fatalf("%s: Trace.Ops empty (dropped %d)", tc.mode, res.Trace.OpsDropped) + } + for _, o := range res.Trace.Ops { + if o.At.Index != 2 || o.DryRun != tc.dryRun || o.Operator != "context-guru.attr2" { + t.Fatalf("%s: op = %+v", tc.mode, o) + } + } + } +} diff --git a/apply/ops_phantom_test.go b/apply/ops_phantom_test.go new file mode 100644 index 00000000..debabe38 --- /dev/null +++ b/apply/ops_phantom_test.go @@ -0,0 +1,82 @@ +package apply_test + +import ( + "context" + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" +) + +// userShrinker rewrites every non-tool message, which includes the read-only sibling text slots +// of an Anthropic user message that also carries a tool_result. +type userShrinker struct{} + +func (userShrinker) Name() string { return "usershrink" } +func (userShrinker) Enabled(*components.Ctx) bool { return true } +func (userShrinker) Reformat(req *bschemas.BifrostChatRequest, _ *components.Report, _ *components.Ctx) error { + for i := range req.Input { + if req.Input[i].Role != bschemas.ChatMessageRoleTool { + schema.SetMessageText(&req.Input[i], "x") + } + } + return nil +} + +func init() { + components.Register("usershrink", func([]byte) (components.Component, error) { return userShrinker{}, nil }) +} + +func traceOf(t *testing.T, comp string, provider bschemas.ModelProvider, body string) apply.Result { + t.Helper() + p, err := pipe(t, "pipeline: ["+comp+"]\n").Build(nil) + if err != nil { + t.Fatal(err) + } + return apply.BodyOpts(context.Background(), p, store.NewMemory(store.Options{}), apply.Opts{ + Provider: provider, Body: []byte(body), + }) +} + +// A change on a read-only anthropicSiblingText slot is never written back, so no op may claim it. +func TestTraceOpsExcludeSiblingTextChange(t *testing.T) { + long := strings.Repeat("a long sibling text that the component rewrites to nothing\\n", 8) + res := traceOf(t, "usershrink", bschemas.Anthropic, `{"model":"claude-x","messages":[ + {"role":"user","content":[{"type":"text","text":"`+long+`"},{"type":"tool_result","tool_use_id":"a","content":"out"}]} + ]}`) + if res.Changed { + t.Fatalf("precondition: the sibling change must not be written back") + } + if len(res.Trace.Ops) != 0 || res.Trace.OpsDropped == 0 { + t.Fatalf("ops = %+v dropped = %d, want none and >0", res.Trace.Ops, res.Trace.OpsDropped) + } +} + +// A message bifrost cannot round-trip (an unmodelled field) has its change discarded and the +// original bytes forwarded; no op may claim that saving. +func TestTraceOpsExcludeDiscardedChange(t *testing.T) { + res := traceOf(t, "attr1", bschemas.OpenAI, `{"model":"gpt-x","messages":[ + {"role":"user","content":"run it"}, + {"role":"tool","tool_call_id":"a","unmodelled_field":{"k":1},"content":"`+ + strings.Repeat("a long original tool output line that the component rewrites\\n", 8)+`"} + ]}`) + if res.Changed { + t.Fatalf("precondition: the change must be discarded, not written") + } + if len(res.Trace.Ops) != 0 || res.Trace.OpsDropped == 0 { + t.Fatalf("ops = %+v dropped = %d, want none and >0", res.Trace.Ops, res.Trace.OpsDropped) + } +} + +// Control: a lossless change that is written keeps its op and drops nothing. +func TestTraceOpsKeepWrittenChange(t *testing.T) { + res := traceOf(t, "attr1", bschemas.OpenAI, string(toolBody())) + if !res.Changed || len(res.Trace.Ops) != 1 || res.Trace.OpsDropped != 0 || res.Trace.Ops[0].At.Index != 1 { + t.Fatalf("changed=%v ops = %+v dropped = %d", res.Changed, res.Trace.Ops, res.Trace.OpsDropped) + } +} diff --git a/apply/ops_restore_test.go b/apply/ops_restore_test.go new file mode 100644 index 00000000..4aebe6ad --- /dev/null +++ b/apply/ops_restore_test.go @@ -0,0 +1,77 @@ +package apply + +import ( + "context" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/tidwall/gjson" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/expand" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" +) + +type toolCut struct{} + +func (toolCut) Name() string { return "toolcut" } +func (toolCut) Enabled(*components.Ctx) bool { return true } +func (toolCut) Reformat(req *bschemas.BifrostChatRequest, _ *components.Report, _ *components.Ctx) error { + for i := range req.Input { + if req.Input[i].Role == bschemas.ChatMessageRoleTool { + schema.SetMessageText(&req.Input[i], "cut") + } + } + return nil +} + +func restoreRun(t *testing.T, withAnchor bool) Result { + t.Helper() + pipe := components.NewPipeline([]components.Component{toolCut{}}, nil) + pipe.SetFixedRestore(true) + st := store.NewMemory(store.Options{}) + if withAnchor { + msgs := gjson.Get(restoreBody, "messages").Array() + sess := SessionIDFor("", "", bschemas.Anthropic, []byte(restoreBody)) + expand.RecordAnchor(st, sess, expand.Anchor{ID: "H1", At: 2, Fp: expand.Fingerprint(msgs[1])}) + st.Put("H1", []byte("ORIGINAL OUTPUT")) + } + return BodyOpts(context.Background(), pipe, st, Opts{Provider: bschemas.Anthropic, Body: []byte(restoreBody)}) +} + +// Control: with an empty plan the op for the rewritten tool_result (wire message 1) is kept. +func TestOpsKeptWhenRestorePlanEmpty(t *testing.T) { + res := restoreRun(t, false) + if res.Restored != 0 || len(res.Trace.Ops) != 1 || res.Trace.Ops[0].At.Index != 1 || res.Trace.OpsDropped != 0 { + t.Fatalf("restored=%d ops=%+v dropped=%d", res.Restored, res.Trace.Ops, res.Trace.OpsDropped) + } +} + +// A successful insertion shifts later wire indices, so every op is dropped, not misplaced. +func TestOpsDroppedWhenRestoreInserts(t *testing.T) { + res := restoreRun(t, true) + if res.Restored != 1 { + t.Fatalf("precondition: a copy must be inserted, Restored=%d", res.Restored) + } + if len(res.Trace.Ops) != 0 || res.Trace.OpsDropped != 1 { + t.Fatalf("ops=%+v dropped=%d, want none and 1", res.Trace.Ops, res.Trace.OpsDropped) + } +} + +// A failed insertion forwards the pre-pipeline body: no op describes it. +func TestOpsDroppedWhenRestoreInsertFallsBack(t *testing.T) { + orig := []byte(restoreBody) + res := Result{Body: []byte(`{"model":"claude-x"}`)} + res.Trace.Ops = []accountant.Operation{{Kind: accountant.KindRewrite, At: accountant.Position{Section: accountant.SectionMessages, Index: 1}}} + res.Trace.OpsDropped = 2 + plan := restorePlan{{anchor: expand.Anchor{ID: "H1", At: 2}, original: "ORIGINAL OUTPUT"}} + got := finishRestore(res, plan, "anthropic", orig, gjson.GetBytes(orig, "messages").Array()) + if string(got.Body) != restoreBody || got.Restored != 0 { + t.Fatalf("want the original body and no restore, got %s restored=%d", got.Body, got.Restored) + } + if len(got.Trace.Ops) != 0 || got.Trace.OpsDropped != 3 { + t.Fatalf("ops=%+v dropped=%d, want none and 3", got.Trace.Ops, got.Trace.OpsDropped) + } +} diff --git a/apply/ops_splice_test.go b/apply/ops_splice_test.go new file mode 100644 index 00000000..4d0a01e1 --- /dev/null +++ b/apply/ops_splice_test.go @@ -0,0 +1,59 @@ +package apply_test + +import ( + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/tidwall/gjson" + + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/schema" +) + +// midSummarizer replaces messages 1..2 with one summary message: a count-changing run. +type midSummarizer struct{} + +func (midSummarizer) Name() string { return "midsum" } +func (midSummarizer) Enabled(*components.Ctx) bool { return true } +func (midSummarizer) Reformat(req *bschemas.BifrostChatRequest, _ *components.Report, _ *components.Ctx) error { + if len(req.Input) < 4 { + return nil + } + sum := req.Input[1] + schema.SetMessageText(&sum, "s") + out := []bschemas.ChatMessage{req.Input[0], sum} + req.Input = append(out, req.Input[3:]...) + return nil +} + +func init() { + components.Register("midsum", func([]byte) (components.Component, error) { return midSummarizer{}, nil }) +} + +func TestBodyOptsCountChangeEmitsSplice(t *testing.T) { + long := strings.Repeat("some long earlier turn text ", 20) + body := `{"model":"claude-x","messages":[ + {"role":"user","content":"first"}, + {"role":"assistant","content":"` + long + `"}, + {"role":"user","content":"` + long + `"}, + {"role":"assistant","content":"tail one"}, + {"role":"user","content":"final"} + ]}` + res := traceOf(t, "midsum", bschemas.Anthropic, body) + if !res.Changed { + t.Fatal("precondition: the count-changing rewrite must be written back") + } + if res.Trace.OpsDropped != 0 || len(res.Trace.Ops) != 1 || res.Trace.Ops[0].MessageSplice == nil { + t.Fatalf("ops = %+v dropped = %d", res.Trace.Ops, res.Trace.OpsDropped) + } + s := res.Trace.Ops[0].MessageSplice + // Wire bytes: the two removed JSON message elements, and the one inserted. + in := gjson.Get(body, "messages").Array() + out := gjson.GetBytes(res.Body, "messages").Array() + if s.At != 1 || s.Removed != 2 || s.Inserted != 1 || + res.Trace.Ops[0].SizeBefore.Bytes != len(in[1].Raw)+len(in[2].Raw) || + res.Trace.Ops[0].SizeAfter.Bytes != len(out[1].Raw) { + t.Fatalf("splice = %+v op = %+v", s, res.Trace.Ops[0]) + } +} diff --git a/apply/ops_test.go b/apply/ops_test.go new file mode 100644 index 00000000..5312588f --- /dev/null +++ b/apply/ops_test.go @@ -0,0 +1,197 @@ +package apply + +import ( + "testing" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/components" +) + +func TestOpsFromRun_MapsToWirePositionAndSkipsRevertedAndSkipped(t *testing.T) { + rr := components.RunReport{Components: []components.Report{ + {Component: "mask", Changes: []components.Change{{Idx: 1, BytesBefore: 4000, BytesAfter: 100, Hash: "aa11"}}}, + {Component: "dedup", Reverted: true, Changes: []components.Change{{Idx: 0, BytesBefore: 9, BytesAfter: 1, Hash: "bb22"}}}, + {Component: "toon", Skipped: true}, + }} + ops, dropped := OpsFromRun(rr, []int{0, 3}, true) + if dropped != 0 || len(ops) != 1 { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } + o := ops[0] + if o.Kind != accountant.KindRewrite || o.Operator != "context-guru.mask" || o.ChangeID != "cg:mask:1:aa11" { + t.Fatalf("op = %+v", o) + } + if o.At != (accountant.Position{Section: accountant.SectionMessages, Index: 3}) { + t.Fatalf("At = %+v, want messages[3] (wire index, not req.Input index)", o.At) + } + if o.SizeBefore.Bytes != 4000 || o.SizeAfter.Bytes != 100 || o.DryRun { + t.Fatalf("sizes/dryrun = %+v", o) + } +} + +func TestOpsFromRun_DryRunWhenNotApplied(t *testing.T) { + rr := components.RunReport{Components: []components.Report{ + {Component: "mask", Changes: []components.Change{{Idx: 0, BytesBefore: 10, BytesAfter: 1, Hash: "cc"}}}, + }} + ops, _ := OpsFromRun(rr, []int{0}, false) + if len(ops) != 1 || !ops[0].DryRun { + t.Fatalf("a run that was not applied must be DryRun: %+v", ops) + } +} + +func TestOpsFromRun_ObserveReportIsDryRunEvenWhenApplied(t *testing.T) { + rr := components.RunReport{Components: []components.Report{ + {Component: "mask", Mode: components.ModeObserve, Changes: []components.Change{{Idx: 0, BytesBefore: 10, BytesAfter: 1, Hash: "cc"}}}, + }} + ops, _ := OpsFromRun(rr, []int{0}, true) + if len(ops) != 1 || !ops[0].DryRun { + t.Fatalf("an observe-mode report must be DryRun: %+v", ops) + } +} + +func TestOpsFromRun_NilWireDropsEverything(t *testing.T) { + rr := components.RunReport{Components: []components.Report{ + {Component: "mask", Changes: []components.Change{{Idx: 0, BytesBefore: 10, BytesAfter: 1, Hash: "cc"}}}, + }} + ops, dropped := OpsFromRun(rr, nil, true) + if len(ops) != 0 || dropped != 1 { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } +} + +func TestOpsFromRun_CountChangedRule(t *testing.T) { + ch := func(h string) []components.Change { + return []components.Change{{Idx: 0, BytesBefore: 10, BytesAfter: 1, Hash: h}} + } + sum := func() components.Report { + return components.Report{Component: "sum", CountChanged: true, Changes: []components.Change{ + {Idx: 1, BytesBefore: 8, BytesAfter: 1, Hash: "ss", Splice: &components.Splice{At: 1, Removed: 2, Inserted: 1}}}} + } + mid := components.RunReport{Components: []components.Report{ + {Component: "a", Changes: ch("1")}, sum(), {Component: "b", Changes: ch("2")}, + }} + ops, dropped := OpsFromRun(mid, []int{0, 1, 2, 3}, true) + if len(ops) != 2 || ops[0].ChangeID != "cg:a:0:1" || ops[1].MessageSplice == nil || dropped != 1 { + t.Fatalf("CountChanged in the middle: ops = %+v dropped = %d", ops, dropped) + } + last := components.RunReport{Components: []components.Report{{Component: "a", Changes: ch("1")}, sum()}} + ops, dropped = OpsFromRun(last, []int{0, 1, 2, 3}, true) + if len(ops) != 2 || dropped != 0 { + t.Fatalf("CountChanged last: ops = %+v dropped = %d", ops, dropped) + } +} + +func spliceReport(at, removed, inserted int) components.RunReport { + return components.RunReport{Components: []components.Report{{Component: "sum", CountChanged: true, + Changes: []components.Change{{Idx: at, BytesBefore: 8, BytesAfter: 2, Hash: "ss", + Splice: &components.Splice{At: at, Removed: removed, Inserted: inserted}}}}}} +} + +func TestOpsFromRun_SpliceMapsToWire(t *testing.T) { + // slots 1 and 2 are one wire message (1); slot 0 is wire 0, slot 3 is wire 2. Dense. + wire := []int{0, 1, 1, 2} + ops, dropped := OpsFromRun(spliceReport(1, 2, 1), wire, true) + if dropped != 0 || len(ops) != 1 { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } + o := ops[0] + if *o.MessageSplice != (accountant.MessageSplice{At: 1, Removed: 1, Inserted: 1}) || o.At.Index != 1 || + o.SizeBefore.Bytes != 8 || o.SizeAfter.Bytes != 2 || o.ChangeID != "cg:sum:1:ss" { + t.Fatalf("op = %+v splice %+v", o, o.MessageSplice) + } +} + +func TestOpsFromRun_SpliceAtZero(t *testing.T) { + ops, dropped := OpsFromRun(spliceReport(0, 1, 0), []int{0, 1, 2}, true) + if dropped != 0 || len(ops) != 1 || *ops[0].MessageSplice != (accountant.MessageSplice{At: 0, Removed: 1, Inserted: 0}) { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } +} + +func TestOpsFromRun_SpliceAppendedAtTheEnd(t *testing.T) { + // the last wire message (5) spans two slots: one past it is 6, not len(wire) = 3. + ops, dropped := OpsFromRun(spliceReport(3, 0, 1), []int{0, 5, 5}[:3], true) + if dropped != 0 || len(ops) != 1 || *ops[0].MessageSplice != (accountant.MessageSplice{At: 6, Removed: 0, Inserted: 1}) { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } + if ops[0].At.Index != 6 { + t.Fatalf("At = %+v", ops[0].At) + } +} + +func TestOpsFromRun_SplicePartOfASharedWireMessageIsDropped(t *testing.T) { + wire := []int{0, 1, 1, 2} + for _, c := range []struct{ at, rm, in int }{{1, 1, 1}, {2, 1, 0}, {2, 0, 1}, {0, 2, 1}} { + ops, dropped := OpsFromRun(spliceReport(c.at, c.rm, c.in), wire, true) + if len(ops) != 0 || dropped != 1 { + t.Fatalf("splice %+v: ops = %+v dropped = %d, want dropped", c, ops, dropped) + } + } +} + +func TestOpsFromRun_ChangeAfterASpliceStaysDropped(t *testing.T) { + rr := spliceReport(1, 1, 1) + rr.Components = append(rr.Components, components.Report{Component: "mask", + Changes: []components.Change{{Idx: 0, BytesBefore: 5, BytesAfter: 1, Hash: "m"}}}) + ops, dropped := OpsFromRun(rr, []int{0, 1, 2}, true) + if len(ops) != 1 || ops[0].MessageSplice == nil || dropped != 1 { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } +} + +func TestOpsFromRun_SecondSpliceIsDropped(t *testing.T) { + rr := spliceReport(1, 1, 1) + rr.Components = append(rr.Components, spliceReport(0, 1, 0).Components...) + ops, dropped := OpsFromRun(rr, []int{0, 1, 2}, true) + if len(ops) != 1 || dropped != 1 { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } +} + +// wire {0,3,3,4} means wire messages 1 and 2 were unparseable and have no slot. +func TestOpsFromRun_SpliceAcrossAWireHoleIsDropped(t *testing.T) { + wire := []int{0, 3, 3, 4} + for _, c := range []struct{ at, rm, in int }{{1, 2, 1}, {0, 1, 1}, {0, 1, 0}, {1, 0, 1}} { + ops, dropped := OpsFromRun(spliceReport(c.at, c.rm, c.in), wire, true) + if len(ops) != 0 || dropped != 1 { + t.Fatalf("splice %+v: ops = %+v dropped = %d, want dropped", c, ops, dropped) + } + } +} + +func TestOpsFromRun_SpliceBeforeAWireHoleIsKept(t *testing.T) { + wire := []int{0, 1, 2, 5, 6} + ops, dropped := OpsFromRun(spliceReport(0, 2, 1), wire, true) + // end=2: pair (wire[1], wire[2]) = (1,2) is dense, so it is kept. + if dropped != 0 || len(ops) != 1 || *ops[0].MessageSplice != (accountant.MessageSplice{At: 0, Removed: 2, Inserted: 1}) { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } + // ending at slot 3 touches the pair (2,5): dropped. + ops, dropped = OpsFromRun(spliceReport(1, 2, 1), wire, true) + if len(ops) != 0 || dropped != 1 { + t.Fatalf("ops = %+v dropped = %d, want dropped", ops, dropped) + } +} + +// shiftingSplice is what keeps the equal-count writeback from sizing a message with the wrong +// index: it is true only for a splice that inserts a different count than it removed. +func TestShiftingSplice(t *testing.T) { + sp := func(removed, inserted int) accountant.Operation { + return accountant.Operation{MessageSplice: &accountant.MessageSplice{At: 1, Removed: removed, Inserted: inserted}} + } + for _, tc := range []struct { + name string + ops []accountant.Operation + want bool + }{ + {"no operations", nil, false}, + {"a rewrite", []accountant.Operation{{}}, false}, + {"a splice that keeps the count", []accountant.Operation{sp(2, 2)}, false}, + {"a splice that removes more", []accountant.Operation{{}, sp(3, 1)}, true}, + {"a splice that inserts more", []accountant.Operation{sp(1, 2)}, true}, + } { + if got := shiftingSplice(tc.ops); got != tc.want { + t.Errorf("%s: shiftingSplice = %v, want %v", tc.name, got, tc.want) + } + } +} diff --git a/apply/ops_wiresize_test.go b/apply/ops_wiresize_test.go new file mode 100644 index 00000000..c01f4972 --- /dev/null +++ b/apply/ops_wiresize_test.go @@ -0,0 +1,154 @@ +package apply_test + +import ( + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/tidwall/gjson" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/apply" +) + +// wireDelta is, per wire message index, len(in) - len(out) of an equal-count rewrite. +func wireDelta(t *testing.T, in, out []byte) map[int]int { + t.Helper() + a, b := gjson.GetBytes(in, "messages").Array(), gjson.GetBytes(out, "messages").Array() + if len(a) != len(b) { + t.Fatalf("message count changed: %d -> %d", len(a), len(b)) + } + d := map[int]int{} + for i := range a { + if a[i].Raw != b[i].Raw { + d[i] = len(a[i].Raw) - len(b[i].Raw) + } + } + return d +} + +// checkChains asserts the library's sum invariant on Trace.Ops: per wire message, the ops form a +// contiguous chain from the inbound message's bytes to the forwarded one's, and nothing else moved. +func checkChains(t *testing.T, body []byte, res apply.Result) { + t.Helper() + in := gjson.GetBytes(body, "messages").Array() + out := gjson.GetBytes(res.Body, "messages").Array() + want := wireDelta(t, body, res.Body) + got := map[int]int{} + last := map[int]int{} + for _, o := range res.Trace.Ops { + i := o.At.Index + if prev, ok := last[i]; ok { + if o.SizeBefore.Bytes != prev { + t.Fatalf("op chain broken at wire %d: %d after %d", i, o.SizeBefore.Bytes, prev) + } + } else if o.SizeBefore.Bytes != len(in[i].Raw) { + t.Fatalf("first op on wire %d: SizeBefore %d, message is %d bytes", i, o.SizeBefore.Bytes, len(in[i].Raw)) + } + last[i] = o.SizeAfter.Bytes + got[i] += o.SizeBefore.Bytes - o.SizeAfter.Bytes + } + for i, a := range last { + if a != len(out[i].Raw) { + t.Fatalf("last op on wire %d: SizeAfter %d, forwarded message is %d bytes", i, a, len(out[i].Raw)) + } + } + for i, d := range want { + if got[i] != d { + t.Fatalf("wire %d: ops remove %d bytes, bodies differ by %d", i, got[i], d) + } + } + for i, g := range got { + if want[i] != g { + t.Fatalf("wire %d: ops remove %d bytes, bodies differ by %d", i, g, want[i]) + } + } +} + +// One component, one OpenAI tool message: the op carries the JSON message element's bytes (role, +// tool_call_id, escaped newlines), not the decoded text. +func TestTraceOpsCarryWireBytesNotTextBytes(t *testing.T) { + res := traceOf(t, "attr1", bschemas.OpenAI, string(toolBody())) + if len(res.Trace.Ops) != 1 || res.Trace.OpsDropped != 0 { + t.Fatalf("ops = %+v dropped = %d", res.Trace.Ops, res.Trace.OpsDropped) + } + checkChains(t, toolBody(), res) +} + +// Two components on one message: the boundary between them splits the wire delta, the chain stays +// contiguous and the total exact. +func TestTraceOpsTwoComponentsOnOneMessageChain(t *testing.T) { + res := traceOf(t, "attr1, attr2", bschemas.OpenAI, string(toolBody())) + if len(res.Trace.Ops) != 2 || res.Trace.OpsDropped != 0 { + t.Fatalf("ops = %+v dropped = %d", res.Trace.Ops, res.Trace.OpsDropped) + } + checkChains(t, toolBody(), res) + if res.Trace.Ops[0].SizeAfter.Bytes == res.Trace.Ops[0].SizeBefore.Bytes { + t.Fatalf("the first component's share must not be zero: %+v", res.Trace.Ops[0]) + } +} + +// Two tool_results in one Anthropic wire message, rewritten by two components: four ops on one wire +// message, one chain. Also: the two slots get two ChangeIDs although the component is the same. +func TestTraceOpsSlotsSharingAWireMessageChain(t *testing.T) { + long := strings.Repeat("a long original tool output line that the component will rewrite\\n", 8) + body := `{"model":"claude-x","messages":[ + {"role":"user","content":"go"}, + {"role":"assistant","content":[{"type":"tool_use","id":"a","name":"R","input":{}},{"type":"tool_use","id":"b","name":"R","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"a","content":"` + long + `"},{"type":"tool_result","tool_use_id":"b","content":"` + long + `"}]} + ]}` + res := traceOf(t, "attr1, attr2", bschemas.Anthropic, body) + if len(res.Trace.Ops) != 4 || res.Trace.OpsDropped != 0 { + t.Fatalf("ops = %+v dropped = %d", res.Trace.Ops, res.Trace.OpsDropped) + } + checkChains(t, []byte(body), res) + seen := map[string]bool{} + for _, o := range res.Trace.Ops { + if seen[o.ChangeID] { + t.Fatalf("duplicate ChangeID %q for two identical outputs", o.ChangeID) + } + seen[o.ChangeID] = true + } +} + +// A count-changing run: the splice op's sizes are the removed and inserted JSON elements, and the +// library's sum check passes on the bodies apply produced. +func TestTraceOpsSpliceWireBytesPassTheSumCheck(t *testing.T) { + long := strings.Repeat("some long earlier turn text ", 20) + body := `{"model":"claude-x","messages":[ + {"role":"user","content":"first"}, + {"role":"assistant","content":"` + long + `"}, + {"role":"user","content":"` + long + `"}, + {"role":"assistant","content":"tail one"}, + {"role":"user","content":"final"} + ]}` + res := traceOf(t, "midsum", bschemas.Anthropic, body) + if len(res.Trace.Ops) != 1 { + t.Fatalf("ops = %+v dropped %d", res.Trace.Ops, res.Trace.OpsDropped) + } + r := libraryResult([]byte(body), res.Body, res.Trace.Ops) + for _, c := range r.Checks { + if c.Check == "sum" || c.Check == "unique_change" { + t.Fatalf("checks = %+v", r.Checks) + } + } +} + +// libraryResult runs the library on the inbound and forwarded bodies, positions measured as raw +// JSON elements (the proxy's cfBodies does the same). +func libraryResult(client, fwd []byte, ops []accountant.Operation) accountant.Result { + b := func(raw []byte) accountant.Body { + var m [][]byte + for _, v := range gjson.GetBytes(raw, "messages").Array() { + m = append(m, []byte(v.Raw)) + } + return accountant.NewBody(nil, nil, m) + } + ts := accountant.NewTrackers() + r := ts.BeginRequest(accountant.RequestMeta{RequestID: "r1", Session: "s", Billing: accountant.Billing{Mode: accountant.BillingAPI}}, b(client), nil) + for _, o := range ops { + r.Add(o) + } + r.EndRequest(accountant.Thread{ID: "main"}, b(fwd), nil, accountant.EntryFacts{Lifetime: accountant.LifetimeUnknown}) + return r.Compute(accountant.Usage{Input: 1000}) +} diff --git a/cmd/context-guru-proxy/dashemitter_savings_test.go b/cmd/context-guru-proxy/dashemitter_savings_test.go new file mode 100644 index 00000000..f2d77c7e --- /dev/null +++ b/cmd/context-guru-proxy/dashemitter_savings_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "path/filepath" + "sync" + "testing" + "time" + + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/dash" + "github.com/rossoctl/context-guru/proxy" +) + +func pingReport() components.KeepAliveReport { + now := time.Now() + return components.KeepAliveReport{Tenant: "t", Session: "s1", Model: "aws/claude-sonnet-5", + Provider: "anthropic", Route: "/v1/messages", CacheRead: 48576, CostUSD: 0.0098, Status: 200, + TS: now.UnixMilli(), Window: now.Add(-4 * time.Minute)} +} + +// tapped runs one ping through a dashEmitter and returns the recorded event as the recorder's tap +// saw it (the tap, unlike the store, keeps the accountant Result). +func tapped(t *testing.T) dash.Event { + t.Helper() + var mu sync.Mutex + var got []dash.Event + rec, err := dash.NewRecorder(dash.Options{DBPath: filepath.Join(t.TempDir(), "d.db"), + BatchSize: 1, FlushInterval: time.Millisecond, + Tap: func(e *dash.Event) { mu.Lock(); got = append(got, *e); mu.Unlock() }}) + if err != nil { + t.Fatal(err) + } + defer rec.Close() + em := &dashEmitter{rec: rec, pings: proxy.NewPingStore()} + em.KeepAlivePing(pingReport()) + for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + mu.Lock() + n := len(got) + mu.Unlock() + if n == 1 { + break + } + } + mu.Lock() + defer mu.Unlock() + if len(got) != 1 { + t.Fatalf("events = %d", len(got)) + } + return got[0] +} + +func TestDashEmitterAttachesTheAccountantResultToAPing(t *testing.T) { + ev := tapped(t) + res, ok := ev.AccountantResult() + if !ok || len(res.Ops) != 1 || res.Ops[0].Call == nil { + t.Fatalf("result %+v ok=%v, want one spend with its call", res, ok) + } + // The override changes components and the baseline of a REQUEST; a ping is not one, so the + // legacy figures stay as Price set them. + if ev.CostUSD != pingReport().CostUSD || ev.CacheRead != pingReport().CacheRead { + t.Errorf("a ping's cost or cache read changed: cost %v read %v", ev.CostUSD, ev.CacheRead) + } +} diff --git a/cmd/context-guru-proxy/main.go b/cmd/context-guru-proxy/main.go index 40addfa4..a796237b 100644 --- a/cmd/context-guru-proxy/main.go +++ b/cmd/context-guru-proxy/main.go @@ -321,7 +321,7 @@ func main() { // `windows` is resolved — rather than reordering boot around a Recorder nothing else here // needs yet. A pointer, so the SAME object threaded into `emitter` right now keeps // working once both fields are set. - kaKeepAlive := &dashEmitter{} + kaKeepAlive := &dashEmitter{pings: proxy.NewPingStore()} // metrics.Slog is deliberately NOT wired in here any more. It emitted one line per // component plus one per run, at INFO, with no tenant and no session on any of them — // so a busy proxy buried its own request lifecycle, and none of it could be correlated @@ -422,9 +422,8 @@ func main() { // means no ping ever reaches this far without --dashboard on. kaKeepAlive.rec = rec // Recover which sessions were live before this restart. Without it every conversation - // in flight reports a cold start on its next turn — and that flag also decides whether - // a cache hit counts as our saving, so a restart handed out one bonus credit per live - // session (see dash.Event.cachesplitSavedUSD). + // in flight reports a cold start on its next turn, which mislabels the cache verdict of + // that turn. if n, err := rec.SeedSessions(time.Now().UnixMilli()); err != nil { slog.Warn("dashboard: could not recover session recency; the first turn of each live "+ "session will be reported as a cold start", "err", err) @@ -703,7 +702,8 @@ func main() { Windows: windows, // dynamic context-window resolver (fraction triggers) Prices: priceResolver(windows), // per-token rates, so each captured request is priced at write time Preset: cfg.Preset, - Dashboard: rec, // nil unless --dashboard + Dashboard: rec, // nil unless --dashboard + Pings: kaKeepAlive.pings, Record: traceRec, // nil unless --record Mode: mode, // sync (default) | observe — explicit, never inferred Observe: proxy.ObserveOptions{ @@ -955,6 +955,9 @@ func spendChecker(rec *dash.Recorder) proxy.SpendChecker { type dashEmitter struct { rec *dash.Recorder prices modelinfo.Pricer + // pings is the store shared with the Handler (proxy.Options.Pings). Each converted ping + // carries an accountant Result on its event. + pings *proxy.PingStore } // Component and Run are no-ops: dashEmitter exists only to be tee'd in for its @@ -1005,6 +1008,9 @@ func (d *dashEmitter) KeepAlivePing(r components.KeepAliveReport) { // derivation of the same number from the same inputs to agree by construction. ev.Price(price, priced && (r.CacheRead > 0 || r.CacheWrite > 0 || r.Output > 0)) ev.CostUSD = r.CostUSD + if res, ok := proxy.PingSavings(r, d.pings); ok { + ev.SetAccountant(res) + } d.rec.Record(ev) } diff --git a/components/component.go b/components/component.go index a459adbe..5a151325 100644 --- a/components/component.go +++ b/components/component.go @@ -872,6 +872,23 @@ func (c *Ctx) Stats() FilterStatsSink { return c.FilterStats } +// Change is one message a component rewrote, in bytes. The pipeline fills it, so no component +// has to. Hash is of the text BEFORE the run: the same content re-sent next turn gets the same +// hash, which is what lets the savings code recognise a replay (accountant ChangeID). +type Change struct { + Idx int // index in req.Input; for a splice, where the splice starts + BytesBefore, BytesAfter int + Hash string + // Splice is set, on the one Change of a count-changing run, to the contiguous range of + // messages the run replaced. BytesBefore/After are then the bytes of the removed and the + // inserted messages. + Splice *Splice +} + +// Splice says a run replaced Removed messages of its input, starting at At, with Inserted +// messages. At is a req.Input index (the same before and after the range). +type Splice struct{ At, Removed, Inserted int } + // Report is the per-component result, modelled after lean-ctx's ToolOutput // token accounting and headroom's record_pipeline_run inputs. The pipeline // fills TokensBefore/After/DurationMs; the component fills CacheKey and may set @@ -899,6 +916,12 @@ type Report struct { // pipeline. The writeback layer uses them to attribute a discarded change back to // the component that made it (see Pipeline.RecordDiscards). ChangedIdx []int + // Changes is ChangedIdx with sizes and a content hash. When CountChanged it holds exactly one + // Change, with Splice set, describing the replaced range. + Changes []Change + // CountChanged says the run added or removed messages (a summary, a drop). ChangedIdx is + // empty then, because indices before and after do not line up; Changes carries the splice. + CountChanged bool // Discarded counts changes this component made that the WRITEBACK layer then threw // away (bifrost could not round-trip the message, so splicing would have dropped // provider fields). A report carrying Discarded > 0 is a follow-up attribution, not @@ -1207,6 +1230,11 @@ type KeepAliveReport struct { // any fresh input or cache write. Booked here once, on the ping row the question replaced, as the // summary's call is (#458). Part of CostUSD, not added to it. 0 for every other ping. SweepAnswerUSD float64 + // Window is the start of the idle gap this ping served: the start of the last upstream + // request the entry was kept warm after (kaEntry.startedAt as it stood before this ping + // began). Zero when the emitter site cannot know it; consumers must then not treat the ping + // as belonging to a window. + Window time.Time } // KeepAliveEmitter is an OPTIONAL capability an Emitter implementation may add. Most Emitter diff --git a/components/pipeline.go b/components/pipeline.go index e14e0926..143a5c6d 100644 --- a/components/pipeline.go +++ b/components/pipeline.go @@ -106,11 +106,14 @@ func (p *Pipeline) Run(req *schemas.BifrostChatRequest, c *Ctx) *RunReport { // messages were re-cloned unchanged six times over. snap := schema.CloneMessages(req.Input) own := ownOutputs(req) + // clientHash is the hash of each changed message's text as the client sent it, filled by the + // first component that changes the message (see changesOf). + clientHash := map[int]string{} for _, comp := range p.comps { if !comp.Enabled(c) { continue } - rep := p.runOne(comp, req, c, &snap, own) + rep := p.runOne(comp, req, c, &snap, own, clientHash) rr.Components = append(rr.Components, rep) rr.DurationMs += rep.DurationMs safeEmit(func() { p.emitter.Component(rep) }) @@ -134,7 +137,7 @@ func safeEmit(fn func()) { // snap is the run's live snapshot: on entry it holds an independent deep copy of // req.Input, and runOne leaves it holding one of req.Input on exit (re-cloning only // what changed, or the lot after a revert or a count change). -func (p *Pipeline) runOne(comp Component, req *schemas.BifrostChatRequest, c *Ctx, snap *[]schemas.ChatMessage, own map[int]bool) (rep Report) { +func (p *Pipeline) runOne(comp Component, req *schemas.BifrostChatRequest, c *Ctx, snap *[]schemas.ChatMessage, own map[int]bool, clientHash map[int]string) (rep Report) { rep = Report{Component: comp.Name(), Mode: c.effMode()} before := *snap // revert installs the snapshot and re-establishes an independent one, because @@ -217,6 +220,18 @@ func (p *Pipeline) runOne(comp Component, req *schemas.BifrostChatRequest, c *Ct if restoreOwn(&rep, before, req.Input, own) { after = tokensOf(req.Input) } + // After restoreOwn: it can rewrite req.Input once changedIdx has run. + rep.CountChanged = len(before) != len(req.Input) + rep.Changes = changesOf(before, req.Input, rep.ChangedIdx, clientHash) + if rep.CountChanged { + if c, ok := spliceOf(before, req.Input); ok { + rep.Changes = []Change{c} + } + // Indices before and after a count change do not line up, so the client hashes no + // longer name the right messages. Every later change is dropped by the savings code + // anyway (apply.OpsFromRun); clearing keeps a stale hash from being reused. + clear(clientHash) + } rep.TokensBefore, rep.TokensAfter = baseline, after // A Reformat stashes nothing, and neither does an Offload that dropped content on purpose // (rep.Irreversible: marker_mode summary/off). Without a key, every consumer of CacheKeys @@ -230,6 +245,79 @@ func (p *Pipeline) runOne(comp Component, req *schemas.BifrostChatRequest, c *Ct return rep } +// changesOf sizes each changed message and hashes its text AS THE CLIENT SENT IT. clientHash +// holds that hash for every message an earlier component of the run already changed; the first +// component to change a message sees the client's own text in before, so it records the hash. The +// hash therefore does not depend on what earlier components did: if one of them starts or stops +// acting between turns, a later component keeps the same ChangeID (accountant replay). +func changesOf(before, after []schemas.ChatMessage, idx []int, clientHash map[int]string) []Change { + if len(idx) == 0 { + return nil + } + out := make([]Change, 0, len(idx)) + for _, i := range idx { + if i < 0 || i >= len(before) || i >= len(after) { + continue + } + bt, at := schema.MessageText(before[i]), schema.MessageText(after[i]) + h, ok := clientHash[i] + if !ok { + sum := sha256.Sum256([]byte(bt)) + h = hex.EncodeToString(sum[:6]) + if clientHash != nil { + clientHash[i] = h + } + } + out = append(out, Change{Idx: i, BytesBefore: len(bt), BytesAfter: len(at), Hash: h}) + } + return out +} + +// spliceOf finds the one contiguous range that differs between before and after, by a common +// prefix and suffix scan with the same equality changedIdx uses (reflect.DeepEqual). The scan +// always yields a single range; if the true change was several, the range is wider than needed, +// which only makes the operation coarser, never wrong. The hash covers the removed messages' text +// before the run (the inserted text when nothing was removed, so the ChangeID is not constant). +func spliceOf(before, after []schemas.ChatMessage) (Change, bool) { + p := 0 + for p < len(before) && p < len(after) && reflect.DeepEqual(before[p], after[p]) { + p++ + } + s := 0 + for s < len(before)-p && s < len(after)-p && + reflect.DeepEqual(before[len(before)-1-s], after[len(after)-1-s]) { + s++ + } + removed, inserted := before[p:len(before)-s], after[p:len(after)-s] + if len(removed) == 0 && len(inserted) == 0 { + return Change{}, false + } + var bb, ba int + h := sha256.New() + // A 0x00 byte separates the messages, so ["ab","c"] and ["a","bc"] hash apart. + for k, m := range removed { + t := schema.MessageText(m) + bb += len(t) + if k > 0 { + h.Write([]byte{0}) + } + h.Write([]byte(t)) + } + for k, m := range inserted { + t := schema.MessageText(m) + ba += len(t) + if len(removed) == 0 { + if k > 0 { + h.Write([]byte{0}) + } + h.Write([]byte(t)) + } + } + return Change{Idx: p, BytesBefore: bb, BytesAfter: ba, + Hash: hex.EncodeToString(h.Sum(nil)[:6]), + Splice: &Splice{At: p, Removed: len(removed), Inserted: len(inserted)}}, true +} + // reformatKeys gives a Reformat the content-derived dedup keys an Offload gets for free // from its stashes: one per message it rewrote, hashed over that message's text BEFORE // the fold. diff --git a/components/pipeline_changes_test.go b/components/pipeline_changes_test.go new file mode 100644 index 00000000..31d8598d --- /dev/null +++ b/components/pipeline_changes_test.go @@ -0,0 +1,201 @@ +package components + +import ( + "strings" + "testing" + + "github.com/maximhq/bifrost/core/schemas" +) + +func twoMsgReq(second string) *schemas.BifrostChatRequest { + r := reqWith("hi") + s := second + r.Input = append(r.Input, schemas.ChatMessage{ + Role: schemas.ChatMessageRoleTool, Content: &schemas.ChatMessageContent{ContentStr: &s}}) + return r +} + +// dropFirst is a Reformat that changes the message count. +type dropFirst struct{} + +func (dropFirst) Name() string { return "dropfirst" } +func (dropFirst) Enabled(*Ctx) bool { return true } +func (dropFirst) Reformat(req *schemas.BifrostChatRequest, _ *Report, _ *Ctx) error { + req.Input = req.Input[1:] + return nil +} + +func TestRunOne_RecordsByteChangesAndStableHash(t *testing.T) { + big := strings.Repeat("word ", 400) + run := func() Report { + rr := NewPipeline([]Component{shrink{}}, nil).Run(twoMsgReq(big), testCtx()) + return rr.Components[0] + } + a, b := run(), run() + if len(a.Changes) != 1 { + t.Fatalf("Changes = %+v, want one", a.Changes) + } + c := a.Changes[0] + if c.Idx != 1 || c.BytesBefore != len(big) || c.BytesAfter <= 0 || c.BytesBefore <= c.BytesAfter { + t.Fatalf("change = %+v", c) + } + if len(c.Hash) != 12 || c.Hash != b.Changes[0].Hash { + t.Fatalf("hash %q vs %q: must be 12 chars and equal for equal input", c.Hash, b.Changes[0].Hash) + } + if a.CountChanged { + t.Fatal("CountChanged must be false when the count is unchanged") + } +} + +func TestRunOne_CountChangeRecordsOneSplice(t *testing.T) { + rep := NewPipeline([]Component{dropFirst{}}, nil).Run(twoMsgReq("x y z"), testCtx()).Components[0] + if rep.Reverted { + t.Fatal("reverted by guard") + } + if !rep.CountChanged || len(rep.Changes) != 1 || rep.Changes[0].Splice == nil { + t.Fatalf("CountChanged=%v Changes=%+v", rep.CountChanged, rep.Changes) + } + if s := *rep.Changes[0].Splice; s != (Splice{At: 0, Removed: 1, Inserted: 0}) { + t.Fatalf("splice = %+v", s) + } +} + +func msgOf(role schemas.ChatMessageRole, text string) schemas.ChatMessage { + t := text + return schemas.ChatMessage{Role: role, Content: &schemas.ChatMessageContent{ContentStr: &t}} +} + +// rewriter is a Reformat that replaces req.Input with a fixed function of it. +type rewriter struct { + f func([]schemas.ChatMessage) []schemas.ChatMessage +} + +func (rewriter) Name() string { return "rewriter" } +func (rewriter) Enabled(*Ctx) bool { return true } +func (r rewriter) Reformat(req *schemas.BifrostChatRequest, _ *Report, _ *Ctx) error { + req.Input = r.f(req.Input) + return nil +} + +func spliceReq() *schemas.BifrostChatRequest { + return &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{ + msgOf(schemas.ChatMessageRoleUser, "a"), msgOf(schemas.ChatMessageRoleTool, "bbbb"), + msgOf(schemas.ChatMessageRoleTool, "cccc"), msgOf(schemas.ChatMessageRoleUser, "dddddddddd"), + }} +} + +func runSplice(t *testing.T, f func([]schemas.ChatMessage) []schemas.ChatMessage) Report { + t.Helper() + rep := NewPipeline([]Component{rewriter{f}}, nil).Run(spliceReq(), testCtx()).Components[0] + if rep.Reverted || !rep.CountChanged || len(rep.Changes) != 1 || rep.Changes[0].Splice == nil { + t.Fatalf("report = %+v", rep) + } + return rep +} + +func TestRunOne_SpliceInTheMiddle(t *testing.T) { + rep := runSplice(t, func(in []schemas.ChatMessage) []schemas.ChatMessage { + return []schemas.ChatMessage{in[0], msgOf(schemas.ChatMessageRoleUser, "s"), in[3]} + }) + c := rep.Changes[0] + if *c.Splice != (Splice{At: 1, Removed: 2, Inserted: 1}) || c.Idx != 1 { + t.Fatalf("splice = %+v idx %d", c.Splice, c.Idx) + } + if c.BytesBefore != 8 || c.BytesAfter != 1 || len(c.Hash) != 12 { + t.Fatalf("change = %+v", c) + } +} + +func TestRunOne_SplicePureRemoval(t *testing.T) { + rep := runSplice(t, func(in []schemas.ChatMessage) []schemas.ChatMessage { + return []schemas.ChatMessage{in[0], in[3]} + }) + c := rep.Changes[0] + if *c.Splice != (Splice{At: 1, Removed: 2, Inserted: 0}) || c.BytesBefore != 8 || c.BytesAfter != 0 { + t.Fatalf("change = %+v splice %+v", c, c.Splice) + } +} + +// A pure insertion cannot come through Run (the never-worse guard reverts a run that grows the +// request), so spliceOf is exercised directly. +func TestSpliceOf_PureInsertion(t *testing.T) { + in := spliceReq().Input + out := append([]schemas.ChatMessage{}, in[:2]...) + out = append(out, msgOf(schemas.ChatMessageRoleUser, "new")) + out = append(out, in[2:]...) + c, ok := spliceOf(in, out) + if !ok || *c.Splice != (Splice{At: 2, Removed: 0, Inserted: 1}) || c.BytesBefore != 0 || c.BytesAfter != 3 { + t.Fatalf("change = %+v ok %v", c, ok) + } + if c.Hash == "e3b0c44298fc" { + t.Fatal("hash of nothing: the ChangeID would be constant") + } +} + +func TestRunOne_RevertedRunRecordsNoChanges(t *testing.T) { + rep := NewPipeline([]Component{grow{}}, nil).Run(reqWith("small"), testCtx()).Components[0] + if !rep.Reverted || rep.Changes != nil || rep.CountChanged { + t.Fatalf("reverted report = %+v", rep) + } +} + +// T3: two different message texts give different hashes. +func TestRunOne_DifferentTextGivesDifferentHash(t *testing.T) { + run := func(s string) string { + return NewPipeline([]Component{shrink{}}, nil).Run(twoMsgReq(s), testCtx()).Components[0].Changes[0].Hash + } + a, b := run(strings.Repeat("word ", 400)), run(strings.Repeat("other ", 400)) + if a == b { + t.Fatalf("hash %q for two different texts", a) + } +} + +// stampTail is a Reformat that appends a marker to the tool message only when on is true: an +// earlier component whose output varies between turns. +type stampTail struct{ on bool } + +func (stampTail) Name() string { return "stamp" } +func (stampTail) Enabled(*Ctx) bool { return true } +func (s stampTail) Reformat(req *schemas.BifrostChatRequest, _ *Report, _ *Ctx) error { + if s.on { + t := msgText(req.Input[1]) + setText(&req.Input[1], t[:len(t)-10]) // drop the last word: smaller, so never-worse keeps it + } + return nil +} + +// M2: a later component's hash is of the text the CLIENT sent, so it does not change when an +// earlier component starts acting on the same message between turns. +func TestRunOne_HashIsOfTheClientTextNotTheEarlierComponentsOutput(t *testing.T) { + big := strings.Repeat("word ", 400) + hashOf := func(on bool) string { + rr := NewPipeline([]Component{stampTail{on: on}, shrink{}}, nil).Run(twoMsgReq(big), testCtx()) + last := rr.Components[len(rr.Components)-1] + if len(last.Changes) != 1 || last.Changes[0].Idx != 1 { + t.Fatalf("on=%v: shrink changes = %+v", on, last.Changes) + } + if on && (len(rr.Components[0].Changes) != 1 || rr.Components[0].Changes[0].Hash != last.Changes[0].Hash) { + t.Fatalf("both components must hash the client text: %+v vs %+v", rr.Components[0].Changes, last.Changes) + } + return last.Changes[0].Hash + } + if off, on := hashOf(false), hashOf(true); off != on { + t.Fatalf("shrink hash changed with the earlier component: %q vs %q", off, on) + } +} + +// T5: the removed messages are separated in the splice hash, so ["ab","c"] and ["a","bc"] differ. +func TestSpliceOf_HashSeparatesMessages(t *testing.T) { + keep := msgOf(schemas.ChatMessageRoleUser, "keep") + h := func(a, b string) string { + in := []schemas.ChatMessage{keep, msgOf(schemas.ChatMessageRoleTool, a), msgOf(schemas.ChatMessageRoleTool, b), keep} + c, ok := spliceOf(in, []schemas.ChatMessage{keep, keep}) + if !ok { + t.Fatal("no splice") + } + return c.Hash + } + if h("ab", "c") == h("a", "bc") { + t.Fatal("the splice hash runs the messages together") + } +} diff --git a/dash/accountant_agree_helpers_test.go b/dash/accountant_agree_helpers_test.go new file mode 100644 index 00000000..52ec4674 --- /dev/null +++ b/dash/accountant_agree_helpers_test.go @@ -0,0 +1,60 @@ +package dash_test + +import ( + "math" + + "github.com/rossoctl/accountant/test/cases" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +// The helpers accountant_agree_test.go reuses. accountant_play_test.go holds same-named ones typed for the +// accountant module's cases, so these carry an Acc suffix. + +func priceOfAcc(r cases.Rate) modelinfo.Price { + return modelinfo.Price{ + Input: float64(r.Input) / 1e12, + CacheWrite: float64(r.CacheWrite5m) / 1e12, + CacheRead: float64(r.CacheRead) / 1e12, + Output: float64(r.Output) / 1e12, + } +} + +func microsAcc(usd float64) int64 { return int64(math.Round(usd * 1e6)) } + +func absAcc(x int64) int64 { + if x < 0 { + return -x + } + return x +} + +// expectedAcc is the case's expected figure for one step, in micro-dollars. +func expectedAcc(c cases.Case, s cases.Step) (net int64, byChange map[string]int64) { + byChange = map[string]int64{} + rate := c.StepRate(s) + for _, e := range s.Expected.Ops { + picos := rate.Picos(e.Saved) - rate.Picos(e.Caused) + if e.Call != nil { + picos -= c.ModelRate(e.Call.Model).CallPicos(e.Call.Usage) + } + byChange[e.ChangeID] += cases.RoundMicros(picos) + net += picos + } + for _, e := range s.Expected.Effects { // breaks and rescues are cache effects, not operations + if !e.Counted() { + continue + } + picos := rate.Picos(e.Saved) - rate.Picos(e.Caused) + byChange[e.ChangeID] += cases.RoundMicros(picos) + net += picos + } + return cases.RoundMicros(net), byChange +} + +// diffAcc is one disagreement. change is the ChangeID it is about, or "" when it is about a step or +// the thread. +type diffAcc struct { + step int // -1 for the thread total + change string + text string +} diff --git a/dash/accountant_agree_test.go b/dash/accountant_agree_test.go new file mode 100644 index 00000000..4d6120d5 --- /dev/null +++ b/dash/accountant_agree_test.go @@ -0,0 +1,218 @@ +package dash_test + +import ( + "fmt" + "sort" + "strings" + "testing" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/accountant/test/cases" + "github.com/rossoctl/context-guru/dash" +) + +// This file plays every cases.Case through the path the proxy uses (Trackers, Request.Add of +// every op with replays included, Compute, Event.SetAccountant, Event.AccountantFigure) and +// compares the dollars with the case's expected figures. It lives in package dash_test, with +// the helpers it reuses (priceOfAcc, microsAcc, expectedAcc, absAcc) in accountant_agree_helpers_test.go. + +// PlayedAcc is the proxy path's figure for one step, in micro-dollars. +type PlayedAcc struct { + Standalone bool + NetMicro int64 + // ByComponent is AccFigure.ByComponent in micro-dollars: the ops plus the breaks charged to a + // component. A per-ChangeID figure is not derivable from it, so the comparison is per component. + ByComponent map[string]int64 + Unpriced []string +} + +// PlayAcc drives the new path over a case. Net = DeltaUSD + KeepAliveSavedUSD + SplitUSD + TTLUSD, +// minus every spend op priced with its own model's rate (the host prices spends). +func PlayAcc(c cases.Case) []PlayedAcc { + trackers := accountant.NewTrackers() + var out []PlayedAcc + for _, s := range c.Steps { + if s.IsStandalone() { + _ = accountant.ComputeStandalone(*s.Standalone) // the library returns the Call; the host prices it + var spend int64 + for _, op := range s.Standalone.Ops { + if op.Kind == accountant.KindSpend && op.Call != nil { + spend += c.ModelRate(op.Call.Model).CallPicos(op.Call.Usage) + } + } + out = append(out, PlayedAcc{Standalone: true, NetMicro: -cases.RoundMicros(spend), ByComponent: map[string]int64{}}) + continue + } + req := trackers.BeginRequest(s.Begin.Meta, s.Begin.Client, s.Begin.ClientBreakpoints) + var spend int64 + for _, op := range s.Add { + req.Add(op) + if op.Kind == accountant.KindSpend && op.Call != nil { + spend += c.ModelRate(op.Call.Model).CallPicos(op.Call.Usage) + } + } + req.EndRequest(s.End.Thread, s.End.Forwarded, s.End.Breakpoints, s.End.Facts.EntryFacts()) + res := req.Compute(s.Usage) + e := &dash.Event{} + e.SetAccountant(res) + fig := e.AccountantFigure(priceOfAcc(c.StepRate(s))) + p := PlayedAcc{ByComponent: map[string]int64{}, Unpriced: fig.Unpriced} + for k, v := range fig.ByComponent { + p.ByComponent[k] = microsAcc(v) + } + p.NetMicro = microsAcc(fig.DeltaUSD+fig.KeepAliveSavedUSD+fig.SplitUSD+fig.TTLUSD) - cases.RoundMicros(spend) + out = append(out, p) + } + return out +} + +// componentsOf maps every ChangeID a case reports to its component, across all steps (a replayed +// change appears in Expected.Ops of a later step, but only in Add of the step that reported it). +func componentsOf(c cases.Case) map[string]string { + m := map[string]string{} + shared := map[string]bool{} // a ChangeID two components report: no per-component split is derivable + for _, s := range c.Steps { + ops := s.Add + if s.Standalone != nil { + ops = s.Standalone.Ops + } + for _, op := range ops { + if op.Kind == accountant.KindSpend || op.ChangeID == "" { + continue + } + comp := strings.TrimPrefix(op.Operator, "context-guru.") + if prev, ok := m[op.ChangeID]; ok && prev != comp { + shared[op.ChangeID] = true + } + if _, ok := m[op.ChangeID]; !ok { + m[op.ChangeID] = comp + } + } + } + for id := range shared { + delete(m, id) + } + return m +} + +// sharedComp reports whether the component reports a ChangeID that another component reports too, +// so its figure cannot be told from the other's by ChangeID (check/changeid-shared-between-operators). +func sharedComp(c cases.Case, comp string) bool { + byID := map[string]map[string]bool{} + for _, s := range c.Steps { + for _, op := range s.Add { + if op.Kind == accountant.KindSpend || op.ChangeID == "" { + continue + } + if byID[op.ChangeID] == nil { + byID[op.ChangeID] = map[string]bool{} + } + byID[op.ChangeID][strings.TrimPrefix(op.Operator, "context-guru.")] = true + } + } + for _, set := range byID { + if len(set) > 1 && set[comp] { + return true + } + } + return false +} + +// expectedByComponent is the case's expected figure of each component, the way AccFigure.ByComponent +// builds it: the priced rewrite and setting ops (a spend is priced by the host, so it is left out) +// plus the counted breaks whose ChangeID a component owns. Rescue, split and TTL effects are their +// own AccFigure fields, not component figures; a counted break nobody owns is EndedBreakUSD. +func expectedByComponent(c cases.Case, s cases.Step, comps map[string]string) map[string]int64 { + picos := map[string]int64{} + rate := c.StepRate(s) + for _, e := range s.Expected.Ops { + if e.Call != nil { + continue + } + if comp, ok := comps[e.ChangeID]; ok { + picos[comp] += rate.Picos(e.Saved) - rate.Picos(e.Caused) + } + } + for _, e := range s.Expected.Effects { + if !e.Counted() || e.Kind != accountant.EffectBreak { + continue + } + if comp, ok := comps[e.ChangeID]; ok && e.ChangeID != "" { + picos[comp] += rate.Picos(e.Saved) - rate.Picos(e.Caused) + } + } + out := map[string]int64{} + for k, v := range picos { + out[k] = cases.RoundMicros(v) + } + return out +} + +// CompareAcc lists the disagreements: the step net, the thread total, and the figure of each +// component. The legacy baseline bill is not compared: AccFigure has none. A per-ChangeID figure is +// not derivable from AccFigure (it sums by component), so the expected per-ChangeID figures are +// summed to their component, and a ChangeID that names no component (a keep-alive window, a +// spend) is skipped. +func CompareAcc(c cases.Case, played []PlayedAcc) (diffs []diffAcc) { + comps := componentsOf(c) + var wantTotal, gotTotal int64 + for i, s := range c.Steps { + want, _ := expectedAcc(c, s) + wantComp := expectedByComponent(c, s, comps) + got := played[i] + wantTotal += want + gotTotal += got.NetMicro + if absAcc(got.NetMicro-want) > 1 { + diffs = append(diffs, diffAcc{i, "", fmt.Sprintf("step %d: expected %d µ$, proxy path %d µ$", i, want, got.NetMicro)}) + } + names := map[string]bool{} + for k := range wantComp { + names[k] = true + } + for k := range got.ByComponent { + names[k] = true + } + var sorted []string + for k := range names { + sorted = append(sorted, k) + } + sort.Strings(sorted) + for _, k := range sorted { + if sharedComp(c, k) { + continue // see componentsOf + } + if absAcc(got.ByComponent[k]-wantComp[k]) > 1 { + diffs = append(diffs, diffAcc{i, k, fmt.Sprintf("step %d component %s: expected %d µ$, proxy path %d µ$", i, k, wantComp[k], got.ByComponent[k])}) + } + } + } + if absAcc(gotTotal-wantTotal) > 1 { + diffs = append(diffs, diffAcc{-1, "", fmt.Sprintf("thread total: expected %d µ$, proxy path %d µ$", wantTotal, gotTotal)}) + } + return diffs +} + +// accPlace is a step and a component ("" for the step net). Step -1 is the thread total. +type accPlace struct { + step int + comp string +} + +// accLibraryDisagrees are cases where Compute itself returns something other than the case expects. +// They are skipped, with the reason, so the suite stays green and the finding stays visible. +var accLibraryDisagrees = map[string]string{} + +// TestProxyPathMatchesTheAccountantCases plays every shared case through the proxy path and wants +// every step net, every component figure and the thread total to match. No difference is allowed. +func TestProxyPathMatchesTheAccountantCases(t *testing.T) { + for _, c := range cases.All() { + t.Run(c.ID, func(t *testing.T) { + if why, ok := accLibraryDisagrees[c.ID]; ok { + t.Skipf("library disagrees with its own case: %s", why) + } + for _, x := range CompareAcc(c, PlayAcc(c)) { + t.Error(x.text) + } + }) + } +} diff --git a/dash/accountant_event.go b/dash/accountant_event.go new file mode 100644 index 00000000..8eab47ea --- /dev/null +++ b/dash/accountant_event.go @@ -0,0 +1,28 @@ +package dash + +import "github.com/rossoctl/accountant" + +// The accountant Result of the request, set by the proxy when a dashboard is on. It is unexported, +// so neither JSON nor the SQLite writer sees it. + +// SetAccountant attaches the library's Result to the event. +func (e *Event) SetAccountant(r accountant.Result) { e.accRes = &r } + +// HasAccountant reports whether a Result is attached. +func (e *Event) HasAccountant() bool { return e.accRes != nil } + +// AccountantResult returns a copy of the attached Result and whether one is attached. It is how a +// test or a host reads what the store does not keep. +func (e *Event) AccountantResult() (accountant.Result, bool) { + if e.accRes == nil { + return accountant.Result{}, false + } + return *e.accRes, true +} + +// SetAccountantContext records what qualifies the attached Result: how many of the request's +// changes the trace could not place as operations, and whether the run was observe mode. Both +// gate the accountant override in PriceMode. +func (e *Event) SetAccountantContext(opsDropped int, observe bool) { + e.accOpsDropped, e.accObserve = opsDropped, observe +} diff --git a/dash/accountant_price.go b/dash/accountant_price.go new file mode 100644 index 00000000..6bf17b23 --- /dev/null +++ b/dash/accountant_price.go @@ -0,0 +1,358 @@ +package dash + +import ( + "sort" + "strings" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +// oneHourWriteInputMultiple is the 1h cache-write premium: such a token bills at 2x the input +// rate. It is the rule of modelinfo.Price.CostWithCacheWrite1h (which adds 2*Input - CacheWrite +// per 1h token on top of the 5m write rate); keep the two in step. +const oneHourWriteInputMultiple = 2 + +// tierUSD prices tokens by tier. +func tierUSD(p modelinfo.Price, t accountant.TierTokens) float64 { + return float64(t.Input)*p.Input + float64(t.CacheRead)*p.CacheRead + + float64(t.CacheWrite5m)*p.CacheWrite + float64(t.CacheWrite1h)*oneHourWriteInputMultiple*p.Input +} + +// AccFigure is the request's savings in dollars, from the library's tokens. Only dollars are +// summed; token tiers are never collapsed into one total. +type AccFigure struct { + ByComponent map[string]float64 // net of the break each component caused + // ByComponentFirst and ByComponentReplay split each component's SAVED tokens by tier: the write + // and input tiers (content the provider had not cached: a first removal) and the read tier (a + // replay of content in the cached prefix). ByComponent minus both is its costs (Caused, breaks). + ByComponentFirst, ByComponentReplay map[string]float64 + // DeltaUSD is the sum of ByComponent plus EndedBreakUSD. It excludes UnattributedBreakUSD, + // KeepAliveSavedUSD, SplitUSD and TTLUSD. + DeltaUSD float64 + KeepAliveSavedUSD float64 // rescue effects + SplitUSD float64 // cachesplit effects (Saved - Caused): CachesplitSavedUSD + TTLUSD float64 // headttl effects: premium (negative) plus rescue: the headttl row + // EndedBreakUSD is the break a change left behind when it ended (accountant design §5.2 row 1, + // Q12): it has no ChangeID, so no component owns it, but the client body would have read that + // prefix, so it is part of the bill difference. Negative: a cost. + EndedBreakUSD float64 + // UnattributedBreakUSD is the breaks the client body has too (InBaseline: client edits, expired + // caches, entries lost early): shown, never counted, never charged to a component. + UnattributedBreakUSD float64 + Unpriced []string // "opID:reason"; an unpriced operation is not a zero saving + Estimated bool // any counted operation or effect is an estimate +} + +// AccountantFigure prices the attached Result. A DryRun operation is skipped: computed, +// never counted. A spend operation (Call != nil) is not a saving and is priced elsewhere. +// Result.Interaction is ignored on purpose: the library attributes it to nobody. +func (e *Event) AccountantFigure(p modelinfo.Price) AccFigure { + f := AccFigure{ByComponent: map[string]float64{}, ByComponentFirst: map[string]float64{}, ByComponentReplay: map[string]float64{}} + if e.accRes == nil { + return f + } + // An unpriced op is also in Ops, with zero tokens (compute.go price()/notEnded()). Counting it + // would show "saved $0" for something unmeasured. OpIDs are host-supplied: they may be empty or + // duplicated, and an Unpriced entry with an empty OpID can be request-level. So an op is skipped + // only when its OpID is in the set AND it carries no tokens: that can never drop a real dollar. + unpricedIDs := map[string]bool{} + for _, u := range e.accRes.Unpriced { + unpricedIDs[u.OpID] = true + } + owner := map[string]string{} // ChangeID -> component; empty ChangeIDs are never owners + for _, o := range e.accRes.Ops { + if o.DryRun || o.Call != nil { + continue + } + c := strings.TrimPrefix(o.Operator, "context-guru.") + // An unpriced op still owns the break it caused: a setting with no position is unpriced + // (penalty_unknown) and its break is priced from the read the usage shows. + if o.ChangeID != "" { + if _, dup := owner[o.ChangeID]; !dup { // first owner wins; the library flags duplicates (unique_change) + owner[o.ChangeID] = c + } + } + if unpricedIDs[o.OpID] && o.Saved == (accountant.TierTokens{}) && o.Caused == (accountant.TierTokens{}) { + continue + } + net := tierUSD(p, o.Saved) - tierUSD(p, o.Caused) + f.ByComponent[c] += net + read := tierUSD(p, accountant.TierTokens{CacheRead: o.Saved.CacheRead}) + f.ByComponentReplay[c] += read + f.ByComponentFirst[c] += tierUSD(p, o.Saved) - read + f.DeltaUSD += net + f.Estimated = f.Estimated || o.Estimated + } + for _, ce := range e.accRes.CacheEffects { + // Sign convention (accountant internal/engine/compute.go): value = tierUSD(Saved) - tierUSD(Caused). + // break: Saved is the read that did not happen, Caused the write that did: a cost, + // tierUSD(Caused) - tierUSD(Saved) > 0, charged to the owner of ChangeID. + // rescue: Saved is the write avoided, Caused the read paid instead: a saving. + // split: same shape as rescue: a saving. + // ttlPremium: Saved is the 5m write, Caused the dearer 1h write: value < 0 (a cost). + // ttlRescue: Saved is the 5m write, Caused the read: a saving. + value := tierUSD(p, ce.Saved) - tierUSD(p, ce.Caused) + switch ce.Kind { + case accountant.EffectBreak: + switch c, ok := owner[ce.ChangeID]; { + case ce.InBaseline: // the client body has this break too: shown, not counted + f.UnattributedBreakUSD -= value + case ok && ce.ChangeID != "": + f.ByComponent[c] += value + f.DeltaUSD += value + case ce.ChangeID == "": // an ended change left it behind: counted, owned by no component + f.EndedBreakUSD += value + f.DeltaUSD += value + default: // a ChangeID no counted op owns (a dry run): shown, never charged + f.UnattributedBreakUSD -= value + } + case accountant.EffectRescue: + f.KeepAliveSavedUSD += value + case accountant.EffectSplit: + f.SplitUSD += value + case accountant.EffectTTLPremium, accountant.EffectTTLRescue: + f.TTLUSD += value + default: + f.Unpriced = append(f.Unpriced, "effect:"+string(ce.Kind)+":unknown_kind") + continue + } + f.Estimated = f.Estimated || ce.Estimated + } + for _, u := range e.accRes.Unpriced { + f.Unpriced = append(f.Unpriced, u.OpID+":"+u.Reason) + } + return f +} + +// Fallback reasons: why PriceMode booked no saving on an event. AccountantFallback +// returns one of them ("" when the override applied). +const ( + FallbackNoResult = "no_result" // no Result attached (no usage, skipped, expand rounds) + FallbackOpsDropped = "ops_dropped" // a change could not be placed as an operation + FallbackObserve = "observe" // the run was observe mode + FallbackUnpriced = "unpriced" // no usage or no price for the model, or an operation or effect is unpriced + FallbackCheckSum = "check_sum" // the library's sum check failed + FallbackCheckUniqueChange = "check_unique_change" // two operations share a ChangeID + FallbackCheckSpans = "check_spans" // an operation's spans do not add up to its sizes + FallbackSizeUnknown = "size_unknown" // TokensBefore is 0: the plausibility bound cannot be checked + FallbackImplausibleSize = "implausible_size" // the library saved more tokens than the request had + FallbackFigureWithoutRow = "figure_without_row" // a library figure for a component with no row + FallbackDuplicateName = "duplicate_name" // two rows share a component name + FallbackActedWithoutFigure = "acted_without_figure" // a component acted and the library has no figure for it +) + +// gatedChecks maps each library check (accountant internal/engine/compute.go checks() and conservation()) that +// means "the operation list or its attribution is wrong" to its fallback reason. A check not in the +// map is informational: it never gates. +// +// - sum: the operations' byte changes per section do not equal the client-minus-forwarded body +// bytes. The op list is wrong (an op missing, a change reported twice under two ChangeIDs, sizes +// in the wrong unit, thread state lost on restart). Gated. +// - unique_change: two live operations share a ChangeID; the figure would follow whichever owner +// comes first. Gated. +// - spans: an operation's spans do not add up to its own sizes; its per-span tiers are then +// priced from numbers that contradict its total. The proxy reports no spans today, so it cannot +// fire, but if it does the op list is malformed. Gated. +// - conservation: the token form of sum (client tokens minus billed tokens against the net of the +// operations). With sum passing it differs only by the ratio's rounding beyond its tolerance, or +// by a usage that does not bill the forwarded body; the second case is what the plausibility +// bound below catches where it matters. Informational, not gated. +// - order, attribution: named in result.go but never emitted by compute.go (attribution holds by +// construction). Not gated; would be informational if ever emitted. +var gatedChecks = map[string]string{ + "sum": FallbackCheckSum, + "unique_change": FallbackCheckUniqueChange, + "spans": FallbackCheckSpans, +} + +// PricedByAccountant is CompRow.PricedBy on a row the accountant library priced. +const PricedByAccountant = "accountant" + +// PricedByNone is CompRow.PricedBy on a row of a request the accountant path handled but could not +// price (a fallback): it books no saving, and the read-time passes must not value it with the +// legacy rule. Any non-empty PricedBy means the stored split stands as written; only a row written +// before the switch has it empty. +const PricedByNone = "none" + +// KindEnvelope is the CompRow.Kind of a row the accountant override adds: an envelope operator or +// the ended-change break, neither of which is a pipeline component. +const KindEnvelope = "envelope" + +// EndedChangeComponent is the row that carries the break an ended change left behind. +const EndedChangeComponent = "ended_change" + +// envelopeComponents are the operators that change the request envelope (tools, system, cache +// marks) and may have no pipeline row: the proxy reports them from a body diff +// (proxy/accountant_envelope.go). expand and adjudicate are never pipeline components; headttl is +// a host option; toolschema, toolfilter and cachesplit are components whose row can be absent when +// the rewrite ran from an equivalent gate (cacheinject runs the split too). +var envelopeComponents = map[string]bool{ + "expand": true, "adjudicate": true, "headttl": true, + "toolschema": true, "toolfilter": true, "cachesplit": true, +} + +// AccountantEstimated reports whether the applied figure is an estimate: the library could not rule +// out a break (penalty_unknown, accountant design §6.4). It is false when the override did not apply. +// It reads PriceEstimated, the stored field the dashboard marks the figure by. +func (e *Event) AccountantEstimated() bool { return e.PriceEstimated } + +// AccountantFallback is the reason PriceMode booked no saving, or "" when the override +// applied. It is "" too on an event priced with Price alone, which never tries the override. +// An event with no usage or no price books no saving too, and its reason is FallbackUnpriced. +func (e *Event) AccountantFallback() string { return e.accFallback } + +// PriceMode is Price, then an override of each component's SavedUSD, of BaselineCostUSD, of +// KeepAliveSavedUSD and of CachesplitSavedUSD from the library's figure. CacheSavedUSD (the +// provider's own cache, a measurement), CostUSD and CGLLMCostUSD always stay as Price set them. +// +// The override is all-or-nothing per event, because a partial list would show an unmeasured +// component as a $0 saving. It applies only when: a Result is attached; no change was dropped +// from the operation list; the run was not observe mode; no operation or effect is unpriced; no +// gated library check failed; the figure is plausible; and every component that acted has a +// library figure. Otherwise the event books no saving (Price set every saving to 0) and +// AccountantFallback says why: a figure nothing measured is not shown. +func (e *Event) PriceMode(p modelinfo.Price, accountingComplete bool) { + e.accFallback, e.PriceEstimated = "", false + if !e.priceCost(p, accountingComplete) { + e.accFallback = FallbackUnpriced + return + } + e.accFallback = e.accOverride(p) + if e.accFallback != "" { + for i := range e.Components { + e.Components[i].PricedBy = PricedByNone + } + } +} + +// accOverride applies the accountant override, or returns the reason it cannot and changes nothing. +func (e *Event) accOverride(p modelinfo.Price) string { + switch { + case e.accRes == nil: + return FallbackNoResult + case e.accOpsDropped != 0: + return FallbackOpsDropped + case e.accObserve: + return FallbackObserve + } + for _, c := range e.accRes.Checks { + if r, ok := gatedChecks[c.Check]; ok { + return r + } + } + fig := e.AccountantFigure(p) + // penalty_unknown is not a missing price: the library priced every operation and could not + // rule out a break, because the thread has no history or a setting has no position. The design + // says to show such a figure and mark it estimated (accountant design §6.4), so it does not gate. + // Any other unpriced reason means an operation has no price, and it does. + estimated := false + for _, u := range fig.Unpriced { + if !strings.HasSuffix(u, ":"+accountant.ReasonPenaltyUnknown) { + return FallbackUnpriced + } + estimated = true + } + // Plausibility bound on the library's INPUT: a request cannot remove more tokens than it had. + // The library converts removed bytes to tokens with the request's own billed-token ratio, so a + // usage that does not match the forwarded body (a wrong bytes-to-tokens ratio) would otherwise + // publish an inflated figure. Over the bound, or with TokensBefore unknown (0), the bound cannot + // be met or checked and the event books no saving. This is a gate, never a rescale of the figure. + if e.TokensBefore <= 0 { + return FallbackSizeUnknown + } + if e.countedSavedTokens() > e.TokensBefore { + return FallbackImplausibleSize + } + // A library figure for a component the event has no row for would leave the rows short of + // the baseline: the sum invariant would break. An envelope operator (envelopeComponents) gets + // its own row; for any other name the event books no saving. + rows := map[string]bool{} + for _, c := range e.Components { + rows[c.Component] = true + } + var added []CompRow + for name := range fig.ByComponent { + switch { + case rows[name]: + case envelopeComponents[name]: + added = append(added, CompRow{Component: name, Kind: KindEnvelope, Acted: true, Mutated: true}) + default: + return FallbackFigureWithoutRow + } + } + // The head-TTL legs (premium and rescue) are a lever's figure like any component's: they go on + // the headttl row, inside the baseline delta. Legacy had no figure for them. + if fig.TTLUSD != 0 { + fig.ByComponent["headttl"] += fig.TTLUSD + fig.DeltaUSD += fig.TTLUSD + if !rows["headttl"] && !inAdded(added, "headttl") { + added = append(added, CompRow{Component: "headttl", Kind: KindEnvelope, Acted: true, Mutated: true}) + } + } + // The break an ended change left behind is counted and owned by no component: it gets a row + // of its own, so the rows still sum to the baseline delta. + if fig.EndedBreakUSD != 0 { + if rows[EndedChangeComponent] { + return FallbackDuplicateName + } + added = append(added, CompRow{Component: EndedChangeComponent, Kind: KindEnvelope, Acted: true}) + fig.ByComponent[EndedChangeComponent] = fig.EndedBreakUSD + } + sort.Slice(added, func(i, j int) bool { return added[i].Component < added[j].Component }) + // Two rows with one name would each take the full figure and break the sum invariant. + seen := map[string]bool{} + for _, c := range e.Components { + if _, ok := fig.ByComponent[c.Component]; ok { + if seen[c.Component] { + return FallbackDuplicateName + } + seen[c.Component] = true + } + } + saved := make([]float64, len(e.Components)) + for i, c := range e.Components { + v, ok := fig.ByComponent[c.Component] + if !ok && c.Acted { + return FallbackActedWithoutFigure + } + saved[i] = v + } + for _, r := range added { + e.Components = append(e.Components, r) + saved = append(saved, fig.ByComponent[r.Component]) + } + for i := range e.Components { + c := &e.Components[i] + c.SavedUSD, c.PricedBy = saved[i], PricedByAccountant + c.SavedFirstUSD, c.SavedReplayUSD = fig.ByComponentFirst[c.Component], fig.ByComponentReplay[c.Component] + } + e.BaselineCostUSD = e.CostUSD + fig.DeltaUSD + // The keep-alive rescue and the split saving keep their own columns, outside the baseline, as + // the legacy rules had them: every reader of keepalive_saved_usd and cachesplit_saved_usd stays valid. + e.KeepAliveSavedUSD, e.CachesplitSavedUSD = fig.KeepAliveSavedUSD, fig.SplitUSD + e.PriceEstimated = estimated + return "" +} + +func inAdded(rows []CompRow, name string) bool { + for _, r := range rows { + if r.Component == name { + return true + } + } + return false +} + +// countedSavedTokens is the Saved tokens, all four tiers, of the operations that feed +// AccountantFigure.ByComponent: not DryRun, not a spend. +func (e *Event) countedSavedTokens() int { + n := 0 + for _, o := range e.accRes.Ops { + if o.DryRun || o.Call != nil { + continue + } + n += o.Saved.Input + o.Saved.CacheRead + o.Saved.CacheWrite5m + o.Saved.CacheWrite1h + } + return n +} diff --git a/dash/accountant_price_test.go b/dash/accountant_price_test.go new file mode 100644 index 00000000..fb6f4896 --- /dev/null +++ b/dash/accountant_price_test.go @@ -0,0 +1,281 @@ +package dash + +import ( + "math" + "reflect" + "testing" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +var accTestPrice = modelinfo.Price{Input: 3e-6, CacheRead: 0.3e-6, CacheWrite: 3.75e-6, Output: 15e-6} + +func accNear(a, b float64) bool { return math.Abs(a-b) < 1e-12 } + +func figOf(r accountant.Result) AccFigure { + e := &Event{} + e.SetAccountant(r) + return e.AccountantFigure(accTestPrice) +} + +func TestTierUSD(t *testing.T) { + got := tierUSD(accTestPrice, accountant.TierTokens{Input: 1000, CacheRead: 1000, CacheWrite5m: 1000, CacheWrite1h: 1000}) + want := 1000*3e-6 + 1000*0.3e-6 + 1000*3.75e-6 + 1000*6e-6 + if !accNear(got, want) { + t.Fatalf("got %v want %v", got, want) + } +} + +func TestAccountantResultReturnsACopy(t *testing.T) { + e := &Event{} + if _, ok := e.AccountantResult(); ok { + t.Fatal("no result attached, ok must be false") + } + e.SetAccountant(accountant.Result{Version: 7}) + r, ok := e.AccountantResult() + if !ok || r.Version != 7 { + t.Fatalf("%+v %v", r, ok) + } + r.Version = 9 + if r2, _ := e.AccountantResult(); r2.Version != 7 { + t.Fatal("accessor must return a copy") + } +} + +func TestFigure_NetOfBreakAttributedByChangeID(t *testing.T) { + f := figOf(accountant.Result{ + Ops: []accountant.PricedOp{{Operator: "context-guru.mask", ChangeID: "c1", + Saved: accountant.TierTokens{CacheWrite5m: 1000}}}, + CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectBreak, ChangeID: "c1", + Caused: accountant.TierTokens{CacheWrite5m: 200}, Saved: accountant.TierTokens{CacheRead: 200}}}, + }) + want := 1000*3.75e-6 - (200*3.75e-6 - 200*0.3e-6) + if !accNear(f.ByComponent["mask"], want) || !accNear(f.DeltaUSD, want) { + t.Fatalf("component = %v delta = %v, want %v", f.ByComponent["mask"], f.DeltaUSD, want) + } +} + +func TestPriceAcc_DryRunNeverCounted(t *testing.T) { + f := figOf(accountant.Result{Ops: []accountant.PricedOp{{Operator: "context-guru.mask", ChangeID: "c1", + DryRun: true, Saved: accountant.TierTokens{Input: 5000}}}}) + if f.DeltaUSD != 0 || len(f.ByComponent) != 0 { + t.Fatalf("a dry run must not reach a total: %+v", f) + } +} + +func TestPriceAcc_DryRunEstimatedDoesNotMarkFigure(t *testing.T) { + f := figOf(accountant.Result{Ops: []accountant.PricedOp{{Operator: "context-guru.mask", DryRun: true, Estimated: true}}}) + if f.Estimated { + t.Fatal("a skipped dry run must not mark the figure estimated") + } +} + +func TestFigure_SpendOpIsNotASaving(t *testing.T) { + f := figOf(accountant.Result{Ops: []accountant.PricedOp{{Operator: "context-guru.summarize", ChangeID: "s1", + Call: &accountant.ModelCall{}, Saved: accountant.TierTokens{Input: 9000}}}}) + if f.DeltaUSD != 0 || len(f.ByComponent) != 0 { + t.Fatalf("a spend op must be skipped: %+v", f) + } +} + +func TestFigure_UnattributedBreakStaysOutOfComponents(t *testing.T) { + f := figOf(accountant.Result{ + Ops: []accountant.PricedOp{{Operator: "context-guru.mask", ChangeID: "c1", DryRun: true}}, + CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectBreak, ChangeID: "c1", + Caused: accountant.TierTokens{CacheWrite5m: 100}}}}) + if f.UnattributedBreakUSD <= 0 || len(f.ByComponent) != 0 || f.DeltaUSD != 0 { + t.Fatalf("%+v", f) + } + if !accNear(f.UnattributedBreakUSD, 100*3.75e-6) { + t.Fatalf("got %v", f.UnattributedBreakUSD) + } +} + +func TestFigure_UnpricedOperationIsListedNotZeroed(t *testing.T) { + f := figOf(accountant.Result{Unpriced: []accountant.Unpriced{{OpID: "x", Reason: "no_usage"}}}) + if !reflect.DeepEqual(f.Unpriced, []string{"x:no_usage"}) { + t.Fatalf("%+v", f) + } +} + +func TestFigure_RescueIsKeepAliveSaved(t *testing.T) { + f := figOf(accountant.Result{CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectRescue, ChangeID: "w1", + Saved: accountant.TierTokens{CacheWrite5m: 1000}, Caused: accountant.TierTokens{CacheRead: 1000}}}}) + want := 1000*3.75e-6 - 1000*0.3e-6 + if !accNear(f.KeepAliveSavedUSD, want) || f.DeltaUSD != 0 || len(f.ByComponent) != 0 { + t.Fatalf("%+v want %v", f, want) + } +} + +func TestFigure_SplitIsReportedApart(t *testing.T) { + f := figOf(accountant.Result{CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectSplit, ChangeID: "cg:cachesplit:1", + Saved: accountant.TierTokens{CacheWrite5m: 1000}, Caused: accountant.TierTokens{CacheRead: 1000}}}}) + want := 1000*3.75e-6 - 1000*0.3e-6 + if !accNear(f.SplitUSD, want) || f.DeltaUSD != 0 || f.KeepAliveSavedUSD != 0 || len(f.ByComponent) != 0 { + t.Fatalf("%+v want %v", f, want) + } +} + +func TestFigure_TTLPremiumIsANegativeSaving(t *testing.T) { + f := figOf(accountant.Result{CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectTTLPremium, ChangeID: "headttl:1", + Saved: accountant.TierTokens{CacheWrite5m: 1000}, Caused: accountant.TierTokens{CacheWrite1h: 1000}}}}) + want := 1000*3.75e-6 - 1000*6e-6 + if !accNear(f.TTLUSD, want) || f.TTLUSD >= 0 { + t.Fatalf("%+v want %v", f, want) + } +} + +func TestFigure_TTLRescueIsAPositiveSaving(t *testing.T) { + f := figOf(accountant.Result{CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectTTLRescue, ChangeID: "headttl:1", + Saved: accountant.TierTokens{CacheWrite5m: 1000}, Caused: accountant.TierTokens{CacheRead: 1000}}}}) + want := 1000*3.75e-6 - 1000*0.3e-6 + if !accNear(f.TTLUSD, want) || f.TTLUSD <= 0 { + t.Fatalf("%+v want %v", f, want) + } +} + +func TestFigure_UnknownEffectKindIsUnpriced(t *testing.T) { + f := figOf(accountant.Result{CacheEffects: []accountant.CacheEffect{{Kind: "future", Saved: accountant.TierTokens{Input: 1}}}}) + if !reflect.DeepEqual(f.Unpriced, []string{"effect:future:unknown_kind"}) || f.DeltaUSD != 0 { + t.Fatalf("%+v", f) + } +} + +func TestFigure_EstimatedFlag(t *testing.T) { + if figOf(accountant.Result{Ops: []accountant.PricedOp{{Operator: "context-guru.mask", ChangeID: "a"}}}).Estimated { + t.Fatal("exact figure marked estimated") + } + if !figOf(accountant.Result{Ops: []accountant.PricedOp{{Operator: "context-guru.mask", ChangeID: "a", Estimated: true}}}).Estimated { + t.Fatal("estimated op not flagged") + } + if !figOf(accountant.Result{CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectRescue, Estimated: true}}}).Estimated { + t.Fatal("estimated effect not flagged") + } +} + +func TestFigure_NoResultIsZeroAndNilSafe(t *testing.T) { + f := (&Event{}).AccountantFigure(accTestPrice) + if f.DeltaUSD != 0 || len(f.ByComponent) != 0 || f.Estimated || len(f.Unpriced) != 0 { + t.Fatalf("%+v", f) + } +} + +func TestFigure_ComponentNameTrimsOnlyTheOperatorPrefix(t *testing.T) { + f := figOf(accountant.Result{Ops: []accountant.PricedOp{ + {Operator: "context-guru.mask", ChangeID: "a", Saved: accountant.TierTokens{Input: 10}}, + {Operator: "headttl", ChangeID: "b", Saved: accountant.TierTokens{Input: 10}}, + }}) + if _, ok := f.ByComponent["mask"]; !ok { + t.Fatalf("%+v", f.ByComponent) + } + if _, ok := f.ByComponent["headttl"]; !ok { + t.Fatalf("%+v", f.ByComponent) + } +} + +func TestFigure_InteractionIsAttributedToNobody(t *testing.T) { + f := figOf(accountant.Result{Interaction: accountant.TierTokens{Input: 5000}}) + if f.DeltaUSD != 0 || len(f.ByComponent) != 0 { + t.Fatalf("%+v", f) + } +} + +func TestFigure_UnpricedOpInOpsIsNotAZeroSaving(t *testing.T) { + f := figOf(accountant.Result{ + Ops: []accountant.PricedOp{ + {OpID: "bad", Operator: "context-guru.mask", ChangeID: "c1"}, + {OpID: "ok", Operator: "context-guru.dedupe", ChangeID: "c2", Saved: accountant.TierTokens{Input: 100}}, + }, + Unpriced: []accountant.Unpriced{{OpID: "bad", Reason: "no_usage"}}, + }) + if _, in := f.ByComponent["mask"]; in { + t.Fatalf("unpriced op must be absent: %+v", f.ByComponent) + } + if !accNear(f.ByComponent["dedupe"], 100*3e-6) || !accNear(f.DeltaUSD, 100*3e-6) { + t.Fatalf("%+v", f) + } + if !reflect.DeepEqual(f.Unpriced, []string{"bad:no_usage"}) { + t.Fatalf("%v", f.Unpriced) + } +} + +func TestFigure_EmptyOpIDUnpricedEntryNeverDropsRealTokens(t *testing.T) { + f := figOf(accountant.Result{ + Ops: []accountant.PricedOp{{Operator: "context-guru.mask", ChangeID: "c1", Saved: accountant.TierTokens{Input: 100}}}, + Unpriced: []accountant.Unpriced{{Reason: "end_request_missing"}}, + }) + if !accNear(f.ByComponent["mask"], 100*3e-6) { + t.Fatalf("%+v", f) + } +} + +// A break with no ChangeID that the client body does not have is one an ended change left behind +// (accountant design §5.2 row 1, Q12): it counts in DeltaUSD, and no component owns it. +func TestFigure_EndedChangeBreakCountsAndIsUnowned(t *testing.T) { + f := figOf(accountant.Result{ + Ops: []accountant.PricedOp{{Operator: "context-guru.mask", Saved: accountant.TierTokens{Input: 100}}}, + CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectBreak, + Caused: accountant.TierTokens{CacheWrite5m: 100}, Saved: accountant.TierTokens{CacheRead: 100}}}, + }) + if !accNear(f.ByComponent["mask"], 100*3e-6) { + t.Fatalf("component touched: %+v", f) + } + brk := 100*0.3e-6 - 100*3.75e-6 + if !accNear(f.EndedBreakUSD, brk) || !accNear(f.DeltaUSD, 100*3e-6+brk) || f.UnattributedBreakUSD != 0 { + t.Fatalf("%+v", f) + } +} + +// A break the client body has too (InBaseline: a client edit, an expiry) is shown and never counted. +func TestFigure_InBaselineBreakIsShownNotCounted(t *testing.T) { + f := figOf(accountant.Result{ + Ops: []accountant.PricedOp{{Operator: "context-guru.mask", Saved: accountant.TierTokens{Input: 100}}}, + CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectBreak, InBaseline: true, + Caused: accountant.TierTokens{CacheWrite5m: 100}, Saved: accountant.TierTokens{CacheRead: 100}}}, + }) + if !accNear(f.DeltaUSD, 100*3e-6) || f.EndedBreakUSD != 0 { + t.Fatalf("an in-baseline break was counted: %+v", f) + } + if !accNear(f.UnattributedBreakUSD, 100*3.75e-6-100*0.3e-6) { + t.Fatalf("%+v", f) + } +} + +// An unpriced setting (no position: penalty_unknown) still owns the break the library priced for it. +func TestFigure_UnpricedSettingOwnsItsBreak(t *testing.T) { + f := figOf(accountant.Result{ + Ops: []accountant.PricedOp{{OpID: "op1", Operator: "inference-router", ChangeID: "rt:1"}}, + CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectBreak, ChangeID: "rt:1", + Caused: accountant.TierTokens{CacheWrite5m: 100}, Saved: accountant.TierTokens{CacheRead: 100}}}, + Unpriced: []accountant.Unpriced{{OpID: "op1", Reason: "penalty_unknown"}}, + }) + if brk := 100*0.3e-6 - 100*3.75e-6; !accNear(f.ByComponent["inference-router"], brk) || f.UnattributedBreakUSD != 0 { + t.Fatalf("%+v", f) + } +} + +func TestFigure_DuplicateChangeIDKeepsFirstOwner(t *testing.T) { + f := figOf(accountant.Result{ + Ops: []accountant.PricedOp{ + {Operator: "context-guru.a", ChangeID: "c1", Saved: accountant.TierTokens{Input: 100}}, + {Operator: "context-guru.b", ChangeID: "c1", Saved: accountant.TierTokens{Input: 100}}, + }, + CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectBreak, ChangeID: "c1", Caused: accountant.TierTokens{Input: 10}}}, + }) + if !accNear(f.ByComponent["a"], 100*3e-6-10*3e-6) || !accNear(f.ByComponent["b"], 100*3e-6) { + t.Fatalf("%+v", f.ByComponent) + } +} + +func TestFigure_BreakCausedAsOneHourWrite(t *testing.T) { + f := figOf(accountant.Result{ + Ops: []accountant.PricedOp{{Operator: "context-guru.mask", ChangeID: "c1", Saved: accountant.TierTokens{Input: 1000}}}, + CacheEffects: []accountant.CacheEffect{{Kind: accountant.EffectBreak, ChangeID: "c1", + Caused: accountant.TierTokens{CacheWrite1h: 200}, Saved: accountant.TierTokens{CacheRead: 200}}}, + }) + want := 1000*3e-6 - (200*6e-6 - 200*0.3e-6) + if !accNear(f.ByComponent["mask"], want) { + t.Fatalf("got %v want %v", f.ByComponent["mask"], want) + } +} diff --git a/dash/accountant_switch_test.go b/dash/accountant_switch_test.go new file mode 100644 index 00000000..801d652d --- /dev/null +++ b/dash/accountant_switch_test.go @@ -0,0 +1,340 @@ +package dash + +import ( + "path/filepath" + "reflect" + "testing" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +// switchEvent is a priced-looking event with a mask component that acted and a library Result +// that saved 1000 cache-read tokens for it. +func switchEvent() *Event { + e := &Event{Model: "m", FreshInput: 100, CacheRead: 1000, OutputTokens: 10, TokensBefore: 5000, + Components: []CompRow{ + {Component: "mask", Acted: true, Mutated: true, SavedGross: 9999, SavedUnique: 9999}, + {Component: "idle"}, + }} + e.SetAccountant(accountant.Result{Ops: []accountant.PricedOp{{Operator: "context-guru.mask", ChangeID: "c", + Saved: accountant.TierTokens{CacheRead: 1000}}}}) + return e +} + +// costOnlyCopy is a copy priced with Price alone: its cost and no saving, which is what an event the +// accountant library could not price carries. +func costOnlyCopy(e *Event) *Event { + c := *e + c.Components = append([]CompRow(nil), e.Components...) + c.Price(accTestPrice, true) + return &c +} + +func TestPriceMode_AccountantOverridesComponentsAndBaseline(t *testing.T) { + e := switchEvent() + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb != "" { + t.Fatalf("a qualifying event fell back: %q", fb) + } + want := 1000 * 0.3e-6 + if !accNear(e.Components[0].SavedUSD, want) || e.Components[1].SavedUSD != 0 { + t.Fatalf("components = %+v, want mask %v", e.Components, want) + } + if !accNear(e.BaselineCostUSD-e.CostUSD, want) { + t.Fatalf("baseline delta = %v want %v", e.BaselineCostUSD-e.CostUSD, want) + } + var sum float64 + for _, c := range e.Components { + sum += c.SavedUSD + } + if d := sum - (e.BaselineCostUSD - e.CostUSD); d > 1e-9 || d < -1e-9 { + t.Fatalf("components sum %v != baseline delta %v", sum, e.BaselineCostUSD-e.CostUSD) + } + l := costOnlyCopy(switchEvent()) + if e.CacheSavedUSD != l.CacheSavedUSD || e.CostUSD != l.CostUSD { + t.Fatal("cache-saved or cost changed by the override") + } +} + +// The keep-alive rescue and the split saving come from the library, in their own columns. +func TestPriceMode_KeepAliveAndSplitComeFromTheLibrary(t *testing.T) { + e := switchEvent() + e.KeepAlivePings, e.SinceLastMs, e.SplitStableTokens = 1, 1<<40, 500 // legacy would credit both + e.accRes.CacheEffects = []accountant.CacheEffect{ + {Kind: accountant.EffectRescue, ChangeID: "ka:main:w", Saved: accountant.TierTokens{CacheWrite5m: 200}, Caused: accountant.TierTokens{CacheRead: 200}}, + {Kind: accountant.EffectSplit, ChangeID: "cg:cachesplit:x", Saved: accountant.TierTokens{CacheWrite5m: 100}, Caused: accountant.TierTokens{CacheRead: 100}}, + } + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb != "" { + t.Fatalf("fell back: %q", fb) + } + if !accNear(e.KeepAliveSavedUSD, 200*(3.75e-6-0.3e-6)) || !accNear(e.CachesplitSavedUSD, 100*(3.75e-6-0.3e-6)) { + t.Fatalf("keep-alive %v split %v", e.KeepAliveSavedUSD, e.CachesplitSavedUSD) + } + if !accNear(e.BaselineCostUSD-e.CostUSD, 1000*0.3e-6) { + t.Fatalf("the rescue or the split leaked into the baseline: delta %v", e.BaselineCostUSD-e.CostUSD) + } +} + +// With no rescue from the library, the keep-alive column is 0, whatever the gap and the pings. +func TestPriceMode_NoRescueMeansNoKeepAliveCredit(t *testing.T) { + e := switchEvent() + e.KeepAlivePings, e.SinceLastMs, e.KeepAliveRefreshed = 1, 1<<40, 1000 // the gaps the legacy rule credited on + e.PriceMode(accTestPrice, true) + if e.KeepAliveSavedUSD != 0 { + t.Fatalf("keep-alive %v, want 0: the library found no rescue", e.KeepAliveSavedUSD) + } +} + +// The head-TTL legs go on the headttl row, inside the baseline delta. +func TestPriceMode_HeadTTLIsAComponentFigure(t *testing.T) { + e := switchEvent() + e.accRes.CacheEffects = []accountant.CacheEffect{{Kind: accountant.EffectTTLPremium, ChangeID: "headttl:main:x", + Saved: accountant.TierTokens{CacheWrite5m: 100}, Caused: accountant.TierTokens{CacheWrite1h: 100}}} + e.PriceMode(accTestPrice, true) + var got float64 + var sum float64 + for _, c := range e.Components { + sum += c.SavedUSD + if c.Component == "headttl" { + got = c.SavedUSD + } + } + want := 100*3.75e-6 - 100*2*3e-6 + if !accNear(got, want) || !accNear(sum, e.BaselineCostUSD-e.CostUSD) { + t.Fatalf("headttl %v want %v; rows %v delta %v", got, want, sum, e.BaselineCostUSD-e.CostUSD) + } +} + +func TestPriceMode_FallsBackToNoSaving(t *testing.T) { + cases := map[string]struct { + mut func(*Event) + reason string + }{ + "no result": {func(e *Event) { e.accRes = nil }, FallbackNoResult}, + "ops dropped": {func(e *Event) { e.SetAccountantContext(1, false) }, FallbackOpsDropped}, + "observe": {func(e *Event) { e.SetAccountantContext(0, true) }, FallbackObserve}, + "unpriced": {func(e *Event) { + e.accRes.Unpriced = []accountant.Unpriced{{OpID: "x", Reason: "r"}} + }, FallbackUnpriced}, + "acted without figure": {func(e *Event) { e.Components[1].Acted = true }, FallbackActedWithoutFigure}, + "figure without row": {func(e *Event) { e.Components = e.Components[1:] }, FallbackFigureWithoutRow}, + "saved above size": {func(e *Event) { e.TokensBefore = 999 }, FallbackImplausibleSize}, + "size unknown": {func(e *Event) { e.TokensBefore = 0 }, FallbackSizeUnknown}, + "duplicate names": {func(e *Event) { + e.Components = append(e.Components, CompRow{Component: "mask"}) + }, FallbackDuplicateName}, + "check sum": {func(e *Event) { + e.accRes.Checks = []accountant.CheckFailure{{Check: "sum", Detail: "section=messages ops=1800 bodies=900 bytes"}} + }, FallbackCheckSum}, + "check unique_change": {func(e *Event) { + e.accRes.Checks = []accountant.CheckFailure{{Check: "unique_change", Operator: "context-guru.mask"}} + }, FallbackCheckUniqueChange}, + "check spans": {func(e *Event) { + e.accRes.Checks = []accountant.CheckFailure{{Check: "spans", Operator: "context-guru.mask"}} + }, FallbackCheckSpans}, + } + for name, c := range cases { + e := switchEvent() + c.mut(e) + want := costOnlyCopy(e) + e.PriceMode(accTestPrice, true) + if got := e.AccountantFallback(); got != c.reason { + t.Errorf("%s: fallback reason %q, want %q", name, got, c.reason) + } + e.accFallback = "" // the copy carries no reason; every other field must match + // A fallback row is marked, so the read views do not value it with the legacy rule. + for i := range e.Components { + if e.Components[i].PricedBy != PricedByNone { + t.Errorf("%s: component %q PricedBy %q, want %q", name, e.Components[i].Component, e.Components[i].PricedBy, PricedByNone) + } + e.Components[i].PricedBy = "" + } + if !reflect.DeepEqual(e, want) { + t.Errorf("%s: event differs from the cost-only copy:\n got %+v\nwant %+v", name, e, want) + } + } +} + +// conservation is informational: it fails when the usage does not bill the forwarded body, which +// the plausibility bound covers where it matters. It must not gate the override. +func TestPriceMode_ConservationCheckDoesNotGate(t *testing.T) { + e := switchEvent() + e.accRes.Checks = []accountant.CheckFailure{{Check: "conservation", Detail: "bill difference=10 ops=5 tokens"}} + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb != "" || !accNear(e.Components[0].SavedUSD, 1000*0.3e-6) { + t.Fatalf("conservation gated the override: reason %q components %+v", fb, e.Components) + } +} + +// An envelope operator (an injected tool declaration) has a figure and no pipeline row: the override +// adds an envelope row for it, so the rows still sum to the baseline delta. +func TestPriceMode_EnvelopeOperatorGetsARow(t *testing.T) { + e := switchEvent() + e.accRes.Ops = append(e.accRes.Ops, accountant.PricedOp{Operator: "context-guru.expand", ChangeID: "cg:expand:x", + Caused: accountant.TierTokens{CacheRead: 100}}) + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb != "" { + t.Fatalf("fell back: %q", fb) + } + var row *CompRow + var sum float64 + for i := range e.Components { + sum += e.Components[i].SavedUSD + if e.Components[i].Component == "expand" { + row = &e.Components[i] + } + } + if row == nil || row.Kind != KindEnvelope || !accNear(row.SavedUSD, -100*0.3e-6) { + t.Fatalf("expand row %+v", row) + } + if !accNear(sum, e.BaselineCostUSD-e.CostUSD) { + t.Fatalf("rows sum %v, baseline delta %v", sum, e.BaselineCostUSD-e.CostUSD) + } +} + +// A figure for a name that is neither a row nor an envelope operator still falls back. +func TestPriceMode_UnknownOperatorStillFallsBack(t *testing.T) { + e := switchEvent() + e.accRes.Ops = append(e.accRes.Ops, accountant.PricedOp{Operator: "context-guru.nosuch", ChangeID: "n", + Saved: accountant.TierTokens{CacheRead: 1}}) + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb != FallbackFigureWithoutRow { + t.Fatalf("fallback %q, want %q", fb, FallbackFigureWithoutRow) + } +} + +// The break an ended change left behind is counted on a row of its own. +func TestPriceMode_EndedChangeBreakHasARow(t *testing.T) { + e := switchEvent() + e.accRes.CacheEffects = []accountant.CacheEffect{{Kind: accountant.EffectBreak, + Saved: accountant.TierTokens{CacheRead: 100}, Caused: accountant.TierTokens{CacheWrite5m: 100}}} + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb != "" { + t.Fatalf("fell back: %q", fb) + } + brk := 100*0.3e-6 - 100*3.75e-6 + var got float64 + found := false + for _, c := range e.Components { + if c.Component == EndedChangeComponent { + got, found = c.SavedUSD, true + } + } + if !found || !accNear(got, brk) || !accNear(e.BaselineCostUSD-e.CostUSD, 1000*0.3e-6+brk) { + t.Fatalf("ended_change %v (found %v), delta %v", got, found, e.BaselineCostUSD-e.CostUSD) + } +} + +// penalty_unknown (a thread's first request) does not gate: the figure applies and is an estimate +// (accountant design §6.4). Any other unpriced reason still gates (TestPriceMode_FallsBackToNoSaving). +func TestPriceMode_PenaltyUnknownIsAnEstimate(t *testing.T) { + e := switchEvent() + e.accRes.Unpriced = []accountant.Unpriced{{Reason: accountant.ReasonPenaltyUnknown}} + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb != "" || !e.AccountantEstimated() { + t.Fatalf("fallback %q estimated %v, want the override applied as an estimate", fb, e.AccountantEstimated()) + } + e.Price(accTestPrice, true) + if e.AccountantEstimated() { + t.Fatal("Price left the estimate flag set") + } +} + +// A request the library could not price books no saving, and the read views must agree: the +// component tab must not value its row with the legacy rule (it looks like a row from before the +// switch otherwise: saved_gross > 0, saved_usd = 0). +func TestAFallbackRowIsNotEstimatedAtReadTime(t *testing.T) { + db := openTestDB(t) + e := mkEvent(DayMs+1, "s", "m1", 5000, 4000) + e.TenantID = "t1" + e.FreshInput, e.CacheRead, e.CacheWrite, e.OutputTokens = 100, 1000, 0, 10 + e.Components = []CompRow{{Component: "mask", Acted: true, Mutated: true, SavedGross: 9999, SavedUnique: 9999}} + e.SetAccountant(accountant.Result{Unpriced: []accountant.Unpriced{{OpID: "x", Reason: "r"}}}) + e.PriceMode(accTestPrice, true) + if e.AccountantFallback() != FallbackUnpriced { + t.Fatalf("fallback %q, want unpriced", e.AccountantFallback()) + } + if got := e.Components[0].PricedBy; got != PricedByNone { + t.Fatalf("PricedBy %q, want %q", got, PricedByNone) + } + if err := db.insertBatch([]*Event{e}); err != nil { + t.Fatal(err) + } + rows, err := db.Components(Filter{TenantAll: true}) + if err != nil { + t.Fatal(err) + } + // The two read-time passes that value a row with the legacy rule when it has no stored figure. + if err := db.EstimateComponentSavedUSD(Filter{TenantAll: true}, handPrice, rows); err != nil { + t.Fatal(err) + } + if err := db.DecomposeComponentSavedUSD(Filter{TenantAll: true}, handPrice, rows); err != nil { + t.Fatal(err) + } + for _, r := range rows { + if r.Component != "mask" { + continue + } + if r.SavedUSD != 0 || r.SavedUSDEstimated != 0 || r.SavedUSDEstimatedRows != 0 || r.NetUSDWithEstimate != r.NetUSD || + r.SavedUSDFirstRemoval != 0 || r.SavedUSDReplay != 0 { + t.Fatalf("a fallback row was valued at read time: saved %v estimated %v (%d rows) net %v with-estimate %v", + r.SavedUSD, r.SavedUSDEstimated, r.SavedUSDEstimatedRows, r.NetUSD, r.NetUSDWithEstimate) + } + return + } + t.Fatal("no mask row") +} + +// An event with no usage or no price books no saving, and says why: an empty reason would read as +// "the override applied". +func TestPriceMode_AnUnpricedEventSaysWhyItBooksNoSaving(t *testing.T) { + for name, tc := range map[string]struct { + price modelinfo.Price + complete bool + }{ + "no usage": {accTestPrice, false}, + "no price": {modelinfo.Price{}, true}, + } { + e := switchEvent() + e.PriceMode(tc.price, tc.complete) + if fb := e.AccountantFallback(); fb != FallbackUnpriced { + t.Errorf("%s: fallback = %q, want %q", name, fb, FallbackUnpriced) + } + } +} + +// The estimate flag outlives the process: it is written with the request and read back, so the +// dashboard marks the figure after a restart, and an exact figure stays unmarked. +func TestPriceEstimatedIsStoredAndReadBack(t *testing.T) { + db, err := Open(filepath.Join(t.TempDir(), "e.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + est, exact := switchEvent(), switchEvent() + est.TS, est.SessionID = 1, "est" + exact.TS, exact.SessionID = 2, "exact" + est.accRes.Unpriced = []accountant.Unpriced{{Reason: accountant.ReasonPenaltyUnknown}} + est.PriceMode(accTestPrice, true) + exact.PriceMode(accTestPrice, true) + if !est.PriceEstimated || exact.PriceEstimated { + t.Fatalf("PriceMode set estimated=%v on the penalty_unknown event and %v on the exact one (fallbacks %q, %q)", + est.PriceEstimated, exact.PriceEstimated, est.AccountantFallback(), exact.AccountantFallback()) + } + if err := db.insertBatch([]*Event{est, exact}); err != nil { + t.Fatal(err) + } + page, err := db.Requests(Filter{}, 0, 10) + if err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, e := range page.Requests { + got[e.SessionID] = e.PriceEstimated + } + if len(got) != 2 || !got["est"] || got["exact"] { + t.Fatalf("stored price_estimated = %v, want est true and exact false", got) + } +} diff --git a/dash/accounting_test.go b/dash/accounting_test.go index 9dc8a47d..435f1baf 100644 --- a/dash/accounting_test.go +++ b/dash/accounting_test.go @@ -164,9 +164,9 @@ func TestDecompositionReconcilesWithTheStoredFigure(t *testing.T) { Components: []CompRow{{Component: "extract", Acted: true, Mutated: true, SavedGross: tr.gross, SavedUnique: tr.unique}}, } - // Price at write time, exactly as the proxy does. + // Price at write time, as the proxy did before the accountant library (priced_by ''). p, _ := handPrice.Price(context.Background(), "m1") - e.Price(p, true) + legacyPrice(e, p) insertReq(t, db, e) } rows, err := db.Components(Filter{TenantAll: true}) @@ -813,7 +813,7 @@ func TestCrossCheckCoversOnlyStoredRows(t *testing.T) { Components: []CompRow{{Component: "extract", Kind: "offload", Acted: true, Mutated: true, SavedGross: 1000, SavedUnique: 1000}}, } - stored.Price(p, true) + legacyPrice(stored, p) insertReq(t, db, stored) insertReq(t, db, &Event{ TS: 2000, SessionID: "s1", Model: "m1", TenantID: "t1", @@ -1018,3 +1018,46 @@ func TestDirectoryScopedSkillIsNotAPlugin(t *testing.T) { } } } + +// legacyPrice writes a row the way the proxy did before the accountant library priced savings +// (priced_by ”): the cost, then each component's SavedUSD from the legacy rule, ComponentSavedUSD, +// and the baseline as their sum. The read-time passes still value such rows, so their tests need one. +func legacyPrice(e *Event, p modelinfo.Price) { + e.Price(p, true) + for i := range e.Components { + c := &e.Components[i] + c.SavedUSD = e.ComponentSavedUSD(p, c.Component, c.SavedGross, c.SavedUnique) + e.BaselineCostUSD += c.SavedUSD + } +} + +// A row the accountant library priced carries its split: the decomposition sums it as stored and +// the cross-check agrees exactly, with the costs as the third term. +func TestDecompositionSumsTheAccountantSplitAsStored(t *testing.T) { + db := openTestDB(t) + e := &Event{TS: 1000, SessionID: "s1", Model: "m1", TenantID: "t1", + TokensBefore: 20000, TokensAfter: 19000, FreshInput: 5, CacheRead: 40000, OutputTokens: 50, + TokenAccounting: AccountingComplete, + Components: []CompRow{{Component: "extract", Acted: true, Mutated: true, SavedGross: 1000, SavedUnique: 1000, + SavedUSD: 0.009, SavedFirstUSD: 0.006, SavedReplayUSD: 0.004, PricedBy: PricedByAccountant}}} + insertReq(t, db, e) + rows, err := db.Components(Filter{TenantAll: true}) + if err != nil { + t.Fatal(err) + } + if err := db.DecomposeComponentSavedUSD(Filter{TenantAll: true}, handPrice, rows); err != nil { + t.Fatal(err) + } + if err := db.EstimateComponentSavedUSD(Filter{TenantAll: true}, handPrice, rows); err != nil { + t.Fatal(err) + } + c := rows[0] + nearEq(t, "first removal", c.SavedUSDFirstRemoval, 0.006) + nearEq(t, "replay", c.SavedUSDReplay, 0.004) + nearEq(t, "costs", c.SavedUSDCosts, -0.001) + nearEq(t, "decomposed", c.SavedUSDDecomposed, 0.009) + nearEq(t, "stored cross-check", c.SavedUSDDecomposedStored, c.SavedUSD) + if c.SavedUSDEstimated != 0 { + t.Fatalf("an accountant row was estimated again: %v", c.SavedUSDEstimated) + } +} diff --git a/dash/adaptcompaction.go b/dash/adaptcompaction.go index 57972b05..4d234fad 100644 --- a/dash/adaptcompaction.go +++ b/dash/adaptcompaction.go @@ -850,29 +850,46 @@ func (d *DB) adaptDataset(ctx context.Context, f Filter, p modelinfo.Pricer, cov // tokenizer correction carry no BilledDeltaFactor, so that column is on a different ruler // from one deployment to the next. MAX per (request, component): request_components has // no unique key and older rows carry duplicates. + // + // A row the accountant library priced (priced_by) carries the figure it booked, in dollars, on one + // ruler; that figure is the observation, and is not re-priced. A fallback row ('none') booked + // nothing. Only a row from before the switch (priced_by '') is re-priced from its tokens. orow, err := d.sql.QueryContext(ctx, `SELECT rc.request_id, rc.component, MAX(rc.saved_gross), - MAX(rc.saved_unique) FROM request_components rc JOIN requests r ON r.id = rc.request_id - WHERE `+cond+` AND rc.saved_gross > 0 GROUP BY rc.request_id, rc.component`, args...) + MAX(rc.saved_unique), MAX(rc.saved_usd), MAX(rc.priced_by) + FROM request_components rc JOIN requests r ON r.id = rc.request_id + WHERE `+cond+` AND (rc.saved_gross > 0 OR rc.priced_by = 'accountant') GROUP BY rc.request_id, rc.component`, args...) if err != nil { return nil, err } defer orow.Close() for orow.Next() { var id, gross, unique int64 - var comp string - if err := orow.Scan(&id, &comp, &gross, &unique); err != nil { + var comp, pricedBy string + var storedUSD float64 + if err := orow.Scan(&id, &comp, &gross, &unique, &storedUSD, &pricedBy); err != nil { return nil, err } - // Savings to date: every priced request in the window, whether or not its content is stored. - if ar := all[id]; ar != nil { + if pricedBy == PricedByNone { + continue // the library could not price this request: it booked no saving + } + // observedUSD is what the row booked, in dollars: the stored figure for a row the library + // priced, else its tokens re-priced at the replay's rates. + observedUSD := func(rq *adaptReq) float64 { + if pricedBy == PricedByAccountant { + return storedUSD + } u := unique if u < 0 { u = 0 } else if u > gross { - u = gross + u = gross // shared content keys can over-attribute, as in Event.Price } - tail, repeat, _ := adaptRates(ar) - v := (float64(u)*tail + float64(gross-u)*repeat) * ar.factor + tail, repeat, _ := adaptRates(rq) + return (float64(u)*tail + float64(gross-u)*repeat) * rq.factor + } + // Savings to date: every priced request in the window, whether or not its content is stored. + if ar := all[id]; ar != nil { + v := observedUSD(ar) cov.ObservedUSD += v if cov.ObservedByComponent == nil { cov.ObservedByComponent = map[string]float64{} @@ -883,13 +900,7 @@ func (d *DB) adaptDataset(ctx context.Context, f Filter, p modelinfo.Pricer, cov if r.observed == nil { r.observed = map[string]float64{} } - if unique < 0 { - unique = 0 - } else if unique > gross { - unique = gross // shared content keys can over-attribute, as in Event.Price - } - tail, repeat, _ := adaptRates(r) - r.observed[comp] += (float64(unique)*tail + float64(gross-unique)*repeat) * r.factor + r.observed[comp] += observedUSD(r) } } if err := orow.Err(); err != nil { diff --git a/dash/adaptcompaction_test.go b/dash/adaptcompaction_test.go index 809a65be..7707389c 100644 --- a/dash/adaptcompaction_test.go +++ b/dash/adaptcompaction_test.go @@ -829,3 +829,38 @@ func TestSaveMoreOffersThePresetWithoutTheAccountsOverrides(t *testing.T) { } } } + +// The adapt report's "savings to date" is what the rows booked. A row the library priced contributes +// its stored dollars, not a re-pricing of its tokens; a fallback row booked nothing and contributes +// nothing; a row from before the switch is re-priced from its tokens. +func TestAdaptObservedFollowsWhatTheRowsBooked(t *testing.T) { + text := prettyJSON() + blocks := map[string]string{"messages.1.content.0.content": text} + db := adaptFixtureDB(t, "acct", map[string][]adaptFixtureReq{"s1": { + {ts: 1_000_000, fresh: 10, write: 5000, blocks: blocks}, + {ts: 1_060_000, fresh: 10, read: 5000, write: 50, blocks: blocks}, + {ts: 1_120_000, fresh: 10, read: 5050, write: 50, blocks: blocks}, + }}) + // Request 1 was priced by the library at $0.5 (its tokens would re-price to something else), + // request 2 fell back, request 3 predates the switch. + for _, r := range []struct { + ts int64 + usd float64 + pricedBy string + }{{1_000_000, 0.5, PricedByAccountant}, {1_060_000, 0, PricedByNone}, {1_120_000, 0, ""}} { + if _, err := db.sql.Exec(`INSERT INTO request_components(request_id, component, saved_gross, saved_unique, saved_usd, priced_by) + SELECT id, 'format', 4000, 4000, ?, ? FROM requests WHERE ts = ?`, r.usd, r.pricedBy, r.ts); err != nil { + t.Fatal(err) + } + } + cov := &AdaptCompactionCoverage{} + if _, err := db.adaptDataset(context.Background(), Filter{Tenant: "acct"}, staticTable{"m": adaptTestPrice}, cov); err != nil { + t.Fatal(err) + } + // The pre-switch row re-prices 4000 tokens at the rate that turn paid: strictly positive, and far + // below the $0.5 the library booked, so the sum tells the three rows apart. + legacy := cov.ObservedUSD - 0.5 + if legacy <= 0 || legacy >= 0.5 { + t.Fatalf("ObservedUSD %v: want 0.5 (library row) + a small re-priced legacy row + 0 (fallback row)", cov.ObservedUSD) + } +} diff --git a/dash/billedfactor_test.go b/dash/billedfactor_test.go deleted file mode 100644 index 7e88011f..00000000 --- a/dash/billedfactor_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package dash - -import ( - "math" - "testing" - - "github.com/rossoctl/context-guru/internal/tokens" -) - -// TestBilledFactorReachesEverySavingsFigure is the regression guard for #240. The defect was -// that our own token counts were priced at the provider's rates with no reconciliation, and -// the fix is a measured per-family factor applied to the COUNTERFACTUAL dollars. -// -// It pins the factor at every site it has to reach, because the arithmetic is implemented more -// than once: Event.Price writes baseline_cost_usd and each component's saved_usd, and -// dash/readvalue.go re-derives the same formula at READ time. A fix applied to one while the -// other stayed put is the failure this asserts against. -func TestBilledFactorReachesEverySavingsFigure(t *testing.T) { - const removed = 10_000 - - mk := func(model string) *Event { - e := &Event{ - TS: 1, SessionID: "s", Model: model, - TokensBefore: 100_000, TokensAfter: 100_000 - removed, - SavedUnique: removed, // all of it new content: the whole delta at the write rate - FreshInput: 10, CacheRead: 0, CacheWrite: 80_000, OutputTokens: 5, - Components: []CompRow{{Component: "searchfold", SavedGross: removed, SavedUnique: removed}}, - } - e.Price(ibmSonnet, true) - return e - } - - // A measured family: the delta must be the raw arithmetic TIMES the factor, and the factor - // must be on the row so a reader can divide it back out. - son := mk("aws/claude-sonnet-5") - wantF, measured := tokens.BilledDeltaFactor("aws/claude-sonnet-5") - if !measured { - t.Fatal("aws/claude-sonnet-5 should be a measured family") - } - if son.BilledTokenFactor != wantF || !son.BilledTokenFactorMeasured { - t.Errorf("factor on the row = %v (measured=%v), want %v (true)", - son.BilledTokenFactor, son.BilledTokenFactorMeasured, wantF) - } - raw := float64(removed) * ibmSonnet.CacheWrite - if got, want := son.BaselineCostUSD-son.CostUSD, raw*wantF; math.Abs(got-want) > 1e-12 { - t.Errorf("baseline delta = %.12f, want %.12f (raw %.12f x %.4f)", got, want, raw, wantF) - } - if got, want := son.Components[0].SavedUSD, raw*wantF; math.Abs(got-want) > 1e-12 { - t.Errorf("component saved_usd = %.12f, want %.12f", got, want) - } - // The direction of the whole issue: the corrected figure is LARGER, because the provider - // counts more tokens in the same removed text than our o200k estimate does. - if son.BaselineCostUSD-son.CostUSD <= raw { - t.Errorf("the correction must raise the delta on a Claude model: %.12f vs raw %.12f", - son.BaselineCostUSD-son.CostUSD, raw) - } - - // An UNMEASURED family gets exactly 1.0 and is flagged as unmeasured — no other family's - // constant is borrowed, and no correction is invented. - other := mk("some-model-nobody-measured") - if other.BilledTokenFactor != 1.0 || other.BilledTokenFactorMeasured { - t.Errorf("unmeasured family: factor %v measured %v, want 1.0 false", - other.BilledTokenFactor, other.BilledTokenFactorMeasured) - } - if got := other.BaselineCostUSD - other.CostUSD; math.Abs(got-raw) > 1e-12 { - t.Errorf("unmeasured family delta = %.12f, want the raw %.12f", got, raw) - } - - // CostUSD is provider-reported and must NOT move: the factor corrects counterfactuals only. - // If this ever fires, a measured bill is being scaled by an estimate. - if math.Abs(son.CostUSD-other.CostUSD) > 1e-15 { - t.Errorf("cost_usd differs by model factor (%.12f vs %.12f) — the factor leaked into a billed figure", - son.CostUSD, other.CostUSD) - } - // And the haiku family must differ from the sonnet/opus one: they were measured as - // different tokenizers, and collapsing them to one constant is the mistake this prevents. - hf, _ := tokens.BilledDeltaFactor("claude-haiku-4-5") - if hf == wantF { - t.Errorf("haiku and sonnet/opus factors are identical (%v); they were measured apart", hf) - } -} diff --git a/dash/cachecost_test.go b/dash/cachecost_test.go index 3e099c81..b8a72bc7 100644 --- a/dash/cachecost_test.go +++ b/dash/cachecost_test.go @@ -17,51 +17,6 @@ import ( // public API's, which is why the operator price list exists at all. var ibmSonnet = modelinfo.Price{Input: 1.52e-6, Output: 7.6e-6, CacheRead: 1.52e-7, CacheWrite: 1.9e-6} -// The re-sent part of a removal is worth the cache-READ rate on a turn whose cache hit, -// and the cache-WRITE rate on a turn whose cache missed — because on a miss the provider -// re-billed the whole prompt, so the removed content would have been re-billed too. -// -// This is the correction with the largest effect on the real service: TTL-expired turns -// were 4% of production requests and 31% of spend, every token of it cache_creation, and -// their savings were being valued at a rate 12.5x too low. -func TestRepeatSavingsAreValuedByTheCacheOutcome(t *testing.T) { - const removed, unique = 10_000, 0 // all of it re-sent history, none of it new - - mk := func(read, write int64) *Event { - e := &Event{ - TS: 1000, SessionID: "s", Model: "aws/claude-sonnet-5", - TokensBefore: 90_000, TokensAfter: 90_000 - removed, - SavedUnique: unique, - FreshInput: 10, CacheRead: read, CacheWrite: write, OutputTokens: 50, - } - e.Price(ibmSonnet, true) - return e - } - hit, miss := mk(80_000, 0), mk(0, 80_000) - - // Both scaled by the tokenizer correction (#240), taken from the event itself so the - // assertions stay about WHICH TIER is used — the thing this test exists to pin. - wantHit := float64(removed) * ibmSonnet.CacheRead * hit.BilledTokenFactor - wantMiss := float64(removed) * ibmSonnet.CacheWrite * miss.BilledTokenFactor - if got := hit.BaselineCostUSD - hit.CostUSD; math.Abs(got-wantHit) > 1e-12 { - t.Errorf("cache HIT: removal valued at %.10f, want the read rate %.10f", got, wantHit) - } - if got := miss.BaselineCostUSD - miss.CostUSD; math.Abs(got-wantMiss) > 1e-12 { - t.Errorf("cache MISS: removal valued at %.10f, want the write rate %.10f", got, wantMiss) - } - // The point of the change, stated as a ratio so a partial revert also fails. - ratio := (miss.BaselineCostUSD - miss.CostUSD) / (hit.BaselineCostUSD - hit.CostUSD) - if ratio < 10 { - t.Errorf("a cold turn's removal is only %.1fx a warm turn's; the write/read spread is 12.5x", ratio) - } - // A turn that neither read nor wrote cache (a provider with no cache at all) values - // the removal at the fresh rate, never at zero. - none := mk(0, 0) - if got := none.BaselineCostUSD - none.CostUSD; math.Abs(got-float64(removed)*ibmSonnet.Input*none.BilledTokenFactor) > 1e-12 { - t.Errorf("no cache at all: removal valued at %.10f, want the fresh rate", got) - } -} - // The prompt cache is a saving of its own, and the dashboard reported none of it. func TestCacheSavedIsCacheReadsAgainstTheFreshRate(t *testing.T) { e := &Event{TS: 1, SessionID: "s", Model: "m", TokensBefore: 100, TokensAfter: 100, @@ -84,69 +39,6 @@ func TestCacheSavedIsCacheReadsAgainstTheFreshRate(t *testing.T) { } } -// The three conditions, one test each, because each one rules out a different way of -// over-claiming — and the figure this replaces failed all three. -func TestCachesplitSavingNeedsTheSplit_TheHit_AndAMovedTail(t *testing.T) { - const stable = 8_478 // what the split measured as the stable half on a real session - split := []CompRow{{Component: "cachesplit", Kind: "reformat", Acted: true, Mutated: true}} - mk := func(comps []CompRow, read int64, tailMoved bool, stableTok int) *Event { - e := &Event{TS: 1, SessionID: "s", Model: "aws/claude-sonnet-5", - TokensBefore: 50_000, TokensAfter: 50_000, - FreshInput: 10, CacheRead: read, OutputTokens: 20, - Components: comps, TailChanged: tailMoved, SplitStableTokens: stableTok} - e.Price(ibmSonnet, true) - return e - } - // The counterfactual is a MISS, which bills those tokens as cache creation, not as - // fresh input: they carry cache_control. So the spread is write minus read. And the - // amount is the STABLE HALF, not the request's whole cache_read — the control arm still - // hit on 45,805 tokens with cachesplit off, so the rest was never ours to claim. - want := float64(stable) * (ibmSonnet.CacheWrite - ibmSonnet.CacheRead) - if got := mk(split, 54_304, true, stable).CachesplitSavedUSD; math.Abs(got-want) > 1e-12 { - t.Errorf("all three conditions met: %.10f, want %.10f", got, want) - } - for name, e := range map[string]*Event{ - "nothing split": mk(nil, 54_304, true, 0), - "component ran but skipped": mk([]CompRow{{Component: "cachesplit", Skipped: true}}, 54_304, true, 0), - "the split produced no hit": mk(split, 0, true, stable), - // A turn whose snapshot did NOT move: the stable half would have been served from - // cache either way, split or not, so there is nothing of ours in it. - "the tail did not change": mk(split, 54_304, false, stable), - // A read too small to have contained our half. Refused rather than clamped: a hit on - // some other, smaller prefix says nothing about ours. - "read smaller than the half we split": mk(split, 3_000, true, stable), - } { - if e.CachesplitSavedUSD != 0 { - t.Errorf("%s: claimed %.10f", name, e.CachesplitSavedUSD) - } - } - // The subtle over-claim, and the reason a hit alone is not enough: the session's first - // request AFTER the stable half itself changed (someone edited CLAUDE.md, the tool list - // moved). An earlier agent breakpoint still hits, so cache_read is large — but our half is - // being billed as CREATION on this very request, so there is nothing to credit. - changed := &Event{TS: 1, SessionID: "s", Model: "aws/claude-sonnet-5", - TokensBefore: 50_000, TokensAfter: 50_000, FreshInput: 10, - CacheRead: 20_000, CacheWrite: 12_000, OutputTokens: 20, - Components: split, TailChanged: true, SplitStableTokens: stable} - changed.Price(ibmSonnet, true) - if changed.CachesplitSavedUSD != 0 { - t.Errorf("credited a request that re-created the stable half: %.10f", changed.CachesplitSavedUSD) - } - // The provider's own cache saving is still measured on every one of them — it is a - // diagnostic, and losing it would hide a pipeline destroying a prefix. - if mk(nil, 54_304, false, 0).CacheSavedUSD <= 0 { - t.Error("the provider-cache diagnostic stopped being measured") - } - // And ours is a small fraction of it, which is the whole correction: 5,654 tokens at - // 1.15x versus 54,304 at 0.9x. Pinned as a bound so a revert to "credit the whole - // cache_read" fails here rather than in production. - e := mk(split, 54_304, true, stable) - if e.CachesplitSavedUSD > e.CacheSavedUSD/3 { - t.Errorf("ours %.10f is too large a share of the provider figure %.10f", - e.CachesplitSavedUSD, e.CacheSavedUSD) - } -} - // The aggregate: our two savings add, and the provider's does not join them. func TestOverviewClaimsOnlyTheTurnsWhoseTailMoved(t *testing.T) { const stable = 8_478 @@ -166,6 +58,11 @@ func TestOverviewClaimsOnlyTheTurnsWhoseTailMoved(t *testing.T) { TailChanged: i == 0, } e.Price(ibmSonnet, true) + // The split's credit is the accountant library's (a split effect, on the turn whose tail + // moved): stored as written. The Overview sums it. + if e.TailChanged { + e.CachesplitSavedUSD = float64(stable) * (ibmSonnet.CacheWrite - ibmSonnet.CacheRead) + } evs = append(evs, e) } if err := db.insertBatch(evs); err != nil { @@ -233,7 +130,7 @@ func TestAdditiveColumnKeepsExistingRows(t *testing.T) { if err != nil { t.Fatal(err) } - added := []string{"cache_saved_usd", "cachesplit_saved_usd", "split_stable_tokens", "project"} + added := []string{"cache_saved_usd", "cachesplit_saved_usd", "split_stable_tokens", "project", "price_estimated"} // An old database has no project index, and SQLite refuses to drop an indexed column. if _, err := raw.Exec(`DROP INDEX IF EXISTS idx_requests_project`); err != nil { t.Fatal(err) @@ -265,6 +162,11 @@ func TestAdditiveColumnKeepsExistingRows(t *testing.T) { t.Fatalf("%s was not added back: %v", col, err) } } + // A pre-migration row reads as an exact figure: that is the claim every such row always made. + var est int + if err := db2.sql.QueryRow(`SELECT price_estimated FROM requests LIMIT 1`).Scan(&est); err != nil || est != 0 { + t.Errorf("a row written before price_estimated reads %d (err %v), want 0", est, err) + } // A pre-migration row reads as the default, which for `project` means "unattributed" rather // than any real folder — the distinction the whole dimension depends on. var proj string diff --git a/dash/capture.go b/dash/capture.go index b7cc3f1a..d6bfc6c0 100644 --- a/dash/capture.go +++ b/dash/capture.go @@ -14,6 +14,13 @@ import ( // database, content capture on, 7-day / 512 MiB retention, loopback-only access // to per-request content and effective config. type Options struct { + // Tap, when non-nil, is called on the writer goroutine with each event once it is stored, + // before the live feed gets its summary copy. It must not block or keep the event after + // it returns. It is the SOLE test seam for the in-memory fields the store does not keep + // (the accountant Result, read through Event.AccountantResult): the database drops them + // and the Hub's subscribe is unexported, so no other route reaches them. No production + // code sets it. + Tap func(*Event) // DBPath is the SQLite file. "" or ":memory:" keeps everything in RAM (the // no-persistence mode), which is also the automatic fallback when the path // cannot be opened — the proxy must never fail to start over a dashboard. @@ -523,6 +530,9 @@ func (r *Recorder) run() { slog.Debug("dash: wrote a batch of captured requests", "n", len(batch), "written_total", r.written.Load(), "dropped_total", r.dropped.Load()) for _, e := range batch { + if r.opts.Tap != nil { + r.opts.Tap(e) + } r.hub.Publish(e) r.enqueueCount(e) } @@ -576,7 +586,7 @@ func (r *Recorder) Observe(tenant, session, model string, now int64) (seenSessio // ObserveSplit is Observe plus the volatile-tail comparison: tailChanged reports whether this // request's tail differs from the previous request's in the same session, which is the turn on -// which the split is worth money (see dash.Event.cachesplitSavedUSD). A session's FIRST request +// which the split is worth money (the accountant library prices it). A session's FIRST request // counts as changed — there was nothing there to match. // // tailHash 0 means nothing split, and then tailChanged is false: there is no split to credit. @@ -627,9 +637,9 @@ func (r *Recorder) observe(tenant, session, threadID, fromThread string, hasFrom // // By AGE, not by clearing the map. The old code emptied lastSeen entirely at 20,000 // entries, which made every live session's next turn report a cold start at once — - // and `seenSession` is no longer only a label: it decides whether a cache hit is - // credited as ours (see Event.cachesplitSavedUSD), so a wholesale reset was a - // wholesale over-claim. Dropping entries by age cannot do that: a session with no + // and under the legacy rule `seenSession` decided whether a cache hit was credited + // as ours, so a wholesale reset was a wholesale over-claim. It is still the cache + // verdict's input. Dropping entries by age cannot do that: a session with no // request for staleSession really has lost its provider cache, so treating its next // turn as a cold start is not a guess, it is correct. if len(r.lastSeen) > 20000 || len(r.graceSeen) > 20000 { diff --git a/dash/capture_test.go b/dash/capture_test.go index d86ff389..3081a2fd 100644 --- a/dash/capture_test.go +++ b/dash/capture_test.go @@ -279,10 +279,8 @@ func TestUncompressedReasonBuckets(t *testing.T) { func TestPriceNeverReportsUnknownCostAsFree(t *testing.T) { p := modelinfo.Price{Input: 2e-06, Output: 1e-05, CacheRead: 2e-07, CacheWrite: 2.5e-06} - // Complete accounting: cost, plus a baseline that prices the UNIQUE removed - // tokens at the cache-WRITE rate they would have entered as and the re-sent - // remainder at the cache-READ rate the provider would have served it from. Here - // every removed token is unique, so the whole 200 gets the write rate. + // Complete accounting: the cost. Price books no saving of ours (the accountant library does, in + // PriceMode), so the baseline is the cost. e := &Event{TokensBefore: 1000, TokensAfter: 800, SavedUnique: 200, FreshInput: 10, CacheRead: 5000, CacheWrite: 500, OutputTokens: 100} e.Price(p, true) @@ -303,11 +301,8 @@ func TestPriceNeverReportsUnknownCostAsFree(t *testing.T) { if math.Abs(e.CostUSD-wantCost) > eps { t.Errorf("cost = %v; want %v", e.CostUSD, wantCost) } - if math.Abs(e.BaselineCostUSD-(wantCost+200*p.CacheWrite)) > eps { - t.Errorf("baseline = %v; want cost + 200 unique removed tokens at the cache-write rate", e.BaselineCostUSD) - } - if e.BaselineCostUSD <= e.CostUSD { - t.Error("baseline must exceed actual when tokens were removed") + if e.BaselineCostUSD != e.CostUSD { + t.Errorf("baseline = %v; want the cost %v: Price books no saving", e.BaselineCostUSD, e.CostUSD) } // No usage data: partial, and NOT priced — a cost we cannot compute must read as @@ -329,97 +324,6 @@ func TestPriceNeverReportsUnknownCostAsFree(t *testing.T) { } } -// TestDollarsDeriveFromUniqueNotGrossSavings is the regression test for the defect -// this dashboard shipped with: `net_dollars_saved` was priced off GROSS savings, so a -// single compaction re-sent on every later turn was paid for once per turn. The tile -// read $7.00 beside an `overcount_ratio` of 13.1x — the dashboard displayed the -// correction factor for its own headline and did not apply it. -// -// The fixture is DELIBERATELY overcounted: one 1,000-token compaction, unique on turn -// 1, re-sent unchanged on nine further turns. Gross savings are 10,000 tokens; -// genuinely-never-sent content is 1,000. Both pricing bugs are pinned: -// -// - the denominator (unique, not gross), and -// - the tier (the re-sent remainder is a cache READ, not a cache WRITE; on this -// price table a write is 12.5x a read, so mispricing it inflates on top of the -// overcount). -// -// A gross+write implementation reports ~11.4x the correct figure here and fails. -func TestDollarsDeriveFromUniqueNotGrossSavings(t *testing.T) { - p := modelinfo.Price{Input: 2e-06, Output: 1e-05, CacheRead: 2e-07, CacheWrite: 2.5e-06} - const turns, saved, uniqueTurn = 10, 1000, 0 - - db := openTestDB(t) - var events []*Event - for i := range turns { - e := &Event{ - TS: int64(1000 + i), SessionID: "s1", Model: "m", - TokensBefore: 50_000, TokensAfter: 50_000 - saved, - // Unique only on the first turn: every later turn re-sends the same content, - // which is exactly what MarkUnique's dedup reports. - FreshInput: 10, CacheRead: 49_000, CacheWrite: 100, OutputTokens: 50, - } - if i == uniqueTurn { - e.SavedUnique = saved - } - e.Price(p, true) - events = append(events, e) - } - if err := db.insertBatch(events); err != nil { - t.Fatal(err) - } - - o, err := db.Overview(Filter{}) - if err != nil { - t.Fatal(err) - } - - // The fixture's own overcount factor, as the dashboard computes and displays it. - if o.SavedGross != turns*saved || o.SavedUnique != saved { - t.Fatalf("fixture: gross=%d unique=%d; want %d/%d", o.SavedGross, o.SavedUnique, turns*saved, saved) - } - if o.OvercountRatio != float64(turns) { - t.Fatalf("overcount_ratio = %v; want %v", o.OvercountRatio, float64(turns)) - } - - // The dollar figure the tile renders, from first principles: the unique 1,000 - // tokens would have been new input (cache-write), the 9,000 re-sent ones would - // have been served as cache reads. - wantDelta := float64(saved)*p.CacheWrite + float64((turns-1)*saved)*p.CacheRead - if got := o.BaselineCostUSD - o.CostUSD; math.Abs(got-wantDelta) > 1e-12 { - t.Errorf("baseline − actual = %.10f; want %.10f", got, wantDelta) - } - if math.Abs(o.NetSavedUSD-wantDelta) > 1e-12 { - t.Errorf("net_saved_usd = %.10f; want %.10f", o.NetSavedUSD, wantDelta) - } - - // And the explicit anti-regression: the figure the old code produced. Pricing all - // 10,000 gross tokens at the cache-write rate is 11.36x the honest number, so a - // reversion cannot slip through as a rounding difference. - grossWritePriced := float64(turns*saved) * p.CacheWrite - if math.Abs(o.NetSavedUSD-grossWritePriced) < 1e-9 { - t.Fatal("net_saved_usd still prices GROSS savings at the cache-write rate") - } - if o.NetSavedUSD >= grossWritePriced { - t.Errorf("net_saved_usd %.10f should be far below the gross-priced %.10f", - o.NetSavedUSD, grossWritePriced) - } -} - -// TestBaselineNeverPricesMoreThanTheRequestSaved guards the clamp: SavedUnique is -// attributed per component and can exceed a single request's own gross saving when -// several components stash the same content key. Without the clamp the "re-sent -// remainder" term goes negative and the baseline inflates. -func TestBaselineNeverPricesMoreThanTheRequestSaved(t *testing.T) { - p := modelinfo.Price{CacheRead: 2e-07, CacheWrite: 2.5e-06} - e := &Event{TokensBefore: 1000, TokensAfter: 900, SavedUnique: 5000} - e.Price(p, true) - if want := 100 * p.CacheWrite; math.Abs(e.BaselineCostUSD-e.CostUSD-want) > 1e-12 { - t.Errorf("baseline delta = %.12f; want %.12f (clamped to the 100 tokens actually removed)", - e.BaselineCostUSD-e.CostUSD, want) - } -} - func TestAgentClassification(t *testing.T) { cases := map[string]string{ "claude-cli/2.0.14 (external, cli)": "claude-cli", diff --git a/dash/compactepisode.go b/dash/compactepisode.go index 0767ca6e..83ab18de 100644 --- a/dash/compactepisode.go +++ b/dash/compactepisode.go @@ -30,7 +30,8 @@ import ( // // # Why almost nothing is computed here // -// The economics already exist and are already right. Event.baselineDeltaUSD prices a removal as +// The economics already exist. The legacy rule (Event.ComponentSavedUSD, which still values rows +// from before the accountant library) prices a removal as // `SavedUnique × cacheWriteRate + (Saved − SavedUnique) × repeatRate`, and repeatRate prices the // re-sent remainder at the cache-READ rate on a turn whose cache hit, the cache-CREATION rate on a // turn whose cache missed, and fresh otherwise — with three documented guards against inflation. @@ -97,8 +98,8 @@ type compactRow struct { // Conservative for a savings figure, and stated in the panel's assumptions. SavedGross int64 // SavedUSD is summarize's share of this request's baseline delta: the credit this turn earns. - // The dataset loader prices it through Event.ComponentSavedUSD, the same path Event.Price - // stores, so a row written before the first-removal rule changed is valued like a new one. + // The dataset loader prices it through Event.ComponentSavedUSD, the path the legacy + // write-time rules stored it with, so a row written before the first-removal rule changed is valued like a new one. // Zero means summarize did nothing priceable on this turn (or the row predates the column). SavedUSD float64 // FirstUSD is the part of SavedUSD that is a FIRST removal: the unique tokens, priced at the rate @@ -117,6 +118,10 @@ type compactRow struct { Events string // Acted is the stored flag: summarize removed tokens on this turn. True on a replay too. Acted bool + // Accountant is true when the accountant library priced the row: SavedUSD is its figure and + // storedFirstUSD its first-removal part, and no read-time formula re-prices it (rePriceRow). + Accountant bool + storedFirstUSD float64 } // Episode states. A closed episode is the only one that contributes to a total. @@ -1165,7 +1170,7 @@ func compactAssumptions(spanFrac, fillFrac float64) CompactionAssumptions { const compactEpisodeSelectTmpl = `SELECT r.tenant_id, r.session_id, r.thread_id, r.model, r.ts, r.tokens_before, r.fresh_input + r.cache_read + r.cache_write AS billed, r.cache_miss_reason, r.cache_write, r.cache_write_1h, r.cache_read, r.cg_llm_cost_usd, - %[1]s, r.fresh_input, %[2]s + %[1]s, r.fresh_input, %[2]s, %[4]s, %[5]s FROM requests r %[3]s WHERE %%s @@ -1176,7 +1181,7 @@ const compactEpisodeSelectTmpl = `SELECT r.tenant_id, r.session_id, r.thread_id, // the sum across the components (a request normally has one). Names come from SummarizerComponents // only; anything else is refused. func compactEpisodeSelectFor(components []string) string { - var joins, gross, usd, events, acted, unique []string + var joins, gross, usd, events, acted, unique, first, acc []string for i, name := range components { if !IsSummarizer(name) { continue @@ -1188,6 +1193,8 @@ func compactEpisodeSelectFor(components []string) string { events = append(events, al+".events") acted = append(acted, "COALESCE("+al+".acted, 0)") unique = append(unique, "COALESCE("+al+".saved_unique, 0)") + first = append(first, "COALESCE("+al+".saved_first_usd, 0)") + acc = append(acc, "COALESCE("+al+".priced_by <> '', 0)") } if len(joins) == 0 { return "" // no valid component: the caller reads nothing @@ -1199,7 +1206,8 @@ func compactEpisodeSelectFor(components []string) string { cols = strings.Join(gross, " + ") + ", " + strings.Join(usd, " + ") + ", COALESCE(" + strings.Join(events, ", ") + ", ''), " + acted[0] } - return fmt.Sprintf(compactEpisodeSelectTmpl, cols, strings.Join(unique, " + "), strings.Join(joins, "\n\t")) + return fmt.Sprintf(compactEpisodeSelectTmpl, cols, strings.Join(unique, " + "), strings.Join(joins, "\n\t"), + strings.Join(first, " + "), strings.Join(acc, " OR ")) } // summarizeSessionsOnly narrows the dataset to sessions in which a summarizer ever acted. The @@ -1245,7 +1253,7 @@ func (d *DB) compactEpisodeDataset(f Filter, price rePriceFn, summarizeOnly bool if err := rows.Scan(&r.Tenant, &r.Session, &r.Thread, &r.Model, &r.TS, &r.TokensBefore, &r.Billed, &r.MissReason, &r.CacheWrite, &r.CacheWrite1h, &r.CacheRead, &r.CGLLMCostUSD, &r.SavedGross, &r.SavedUSD, &r.Events, &acted, - &r.FreshInput, &r.SavedUnique); err != nil { + &r.FreshInput, &r.SavedUnique, &r.storedFirstUSD, &r.Accountant); err != nil { return nil, err } r.Acted = acted != 0 diff --git a/dash/event.go b/dash/event.go index 416cf40a..c028cc87 100644 --- a/dash/event.go +++ b/dash/event.go @@ -3,7 +3,7 @@ package dash import ( "strings" - bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/accountant" "github.com/rossoctl/context-guru/apply" "github.com/rossoctl/context-guru/internal/modelinfo" "github.com/rossoctl/context-guru/internal/tokens" @@ -120,6 +120,14 @@ type WaitCauseResult struct { // token counting, no extra allocation of the transcript) and is then owned // entirely by the writer goroutine. type Event struct { + // accRes is the accountant Result (see accountant_event.go). Unexported: not serialized. + accRes *accountant.Result + // accOpsDropped and accObserve qualify accRes (see PriceMode). Unexported: not serialized. + accOpsDropped int + accObserve bool + // accFallback is why PriceMode booked no saving (see AccountantFallback). + accFallback string + ID int64 `json:"id"` TS int64 `json:"ts"` // epoch ms // TenantID owns this row; "" in single-tenant deployments. @@ -360,9 +368,8 @@ type Event struct { // distinguishes them. KeepAlivePings int `json:"keepalive_pings"` // KeepAliveRefreshed is what the last ping on this session actually read from cache — - // the tokens our ping demonstrably refreshed. Not persisted: it is the numerator's - // CEILING, and what gets stored is the credited dollars beside the cache_read they were - // computed from. See keepaliveSavedUSD for why the credit is the smaller of the two. + // the tokens our ping demonstrably refreshed. Not persisted. The legacy credit rule capped + // the credit with it; the credit is now the accountant library's rescue. KeepAliveRefreshed int64 `json:"-"` // KeepAliveSavedUSD is the cache re-creation this request avoided because a ping kept // its prefix alive. @@ -430,7 +437,12 @@ type Event struct { DeferGrossUSD float64 `json:"defer_gross_usd,omitempty"` DeferNetUSD float64 `json:"defer_net_usd,omitempty"` - TokenAccounting string `json:"token_accounting"` + TokenAccounting string `json:"token_accounting"` + // PriceEstimated is true when the accountant library priced this request without knowing whether a + // change broke the cache (penalty_unknown): the gross saving is right and the net may be too + // high. It is stored, so the dashboard marks such a figure after a restart. False on an exact + // figure, on an event that books no saving, and on a row from before the column existed. + PriceEstimated bool `json:"price_estimated,omitempty"` CacheMissReason string `json:"cache_miss_reason"` UncompressedReason string `json:"uncompressed_reason"` // CacheMissCause says what changed on a miss that is not hit, cold_start or ttl_expiry. @@ -606,14 +618,21 @@ type CompRow struct { SavedUnique int `json:"saved_unique"` // SavedUSD is this component's share of the request's baseline delta, in dollars, // priced at write time from the request's OWN model and the tier the request itself - // paid — the same rule and the same rates as baselineDeltaUSD, so the per-component - // figures sum to the request-level saving. It exists because the components view had + // paid. Set by the accountant library (PriceMode); the per-component figures sum to the + // request-level saving. It exists because the components view had // only a bare COST for the components that spend, and a dollar value improvised // client-side from hardcoded rates is wrong by whatever the deployment's model // actually charges. - SavedUSD float64 `json:"saved_usd"` - DurationMs float64 `json:"duration_ms"` - Err string `json:"err,omitempty"` + SavedUSD float64 `json:"saved_usd"` + // SavedFirstUSD and SavedReplayUSD split SavedUSD by the tier the accountant library priced the + // saved tokens at: the write or input tier (a first removal) and the read tier (a replay of + // content in the cached prefix). SavedUSD minus both is the costs. PricedBy is "accountant" when + // the library priced the row; "" on a row priced before it (see the request_components schema). + SavedFirstUSD float64 `json:"saved_first_usd"` + SavedReplayUSD float64 `json:"saved_replay_usd"` + PricedBy string `json:"priced_by,omitempty"` + DurationMs float64 `json:"duration_ms"` + Err string `json:"err,omitempty"` // Gates counts, per named gate, the candidates this component turned away. It is the // only answer to "why did nothing happen?", and it never left the pipeline before: // /stats had it service-wide, the log line had it per request, and the dashboard — @@ -798,50 +817,38 @@ func uncompressedReason(e *Event, tr apply.Trace) string { return ReasonNoSavings } -// Price fills the cost columns AT WRITE TIME, from this request's four billed -// token tiers plus a baseline counterfactual. -// -// The baseline is what the SAME request would have cost had context-guru not -// removed anything. Getting this right is the whole point of the dashboard, and -// there are two ways to get it wrong, both of which inflate it: -// -// - Pricing GROSS savings. `Saved()` is tokens_before − tokens_after for THIS -// turn, and the agent re-sends its whole transcript every turn, so the same -// compaction is re-counted once per remaining turn. On a real 63-request -// window that is a 13.1x overcount — a factor this dashboard computes and -// displays as `overcount_ratio` right beside the dollar figure. Only -// SavedUnique is content that genuinely never reached the provider. -// - Pricing everything at the cache-WRITE rate. That rate (12.5x a read) is -// right for content entering the prompt for the first time. The re-sent -// remainder, on a turn whose cache HIT, would have been served from the -// provider's cache, so the most it could have been billed at is the -// cache-READ rate. Pricing it as a write multiplies the overcount by ~12.5. -// -// So: unique savings at the write rate, and the re-sent remainder at the rate the -// request itself actually paid — see repeatRate, which is where "whose cache hit" -// stopped being an assumption. Restored (expanded) content is content we removed -// and then had to serve back, so it is added to the ACTUAL cost side, never -// subtracted from baseline. +// Price fills the cost columns AT WRITE TIME, from this request's four billed token tiers: CostUSD, +// the provider's own cache saving (CacheSavedUSD, a measurement, never ours), and the accounting +// fields. It books NO saving of ours: BaselineCostUSD equals CostUSD, and every component's SavedUSD, +// CachesplitSavedUSD and KeepAliveSavedUSD are 0. The savings are the accountant library's, applied +// by PriceMode. An event the library cannot price (AccountantFallback) keeps these zeros: a saving +// nothing measured is not shown. // -// accountingComplete=false leaves every cost at zero and the row is marked -// partial/missing: a cost we cannot compute must read as unknown, not as free. +// accountingComplete=false leaves every cost at zero and the row is marked partial/missing: a cost we +// cannot compute must read as unknown, not as free. func (e *Event) Price(p modelinfo.Price, accountingComplete bool) { + e.accFallback, e.PriceEstimated = "", false + e.priceCost(p, accountingComplete) +} + +// priceCost is Price without clearing the accountant state. It reports whether the cost was +// computed (accounting complete and a price known). +func (e *Event) priceCost(p modelinfo.Price, accountingComplete bool) bool { if !accountingComplete || p.Zero() { if e.TokensBefore > 0 { e.TokenAccounting = AccountingPartial } else { e.TokenAccounting = AccountingMissing } - return + return false } e.TokenAccounting = AccountingComplete - // Resolve the tokenizer correction ONCE per request, before anything prices a saving. - // It multiplies counterfactual token counts only — never a provider-reported count, so - // CostUSD below is untouched. + // Resolve the tokenizer correction ONCE per request. It is kept for the read-time passes that + // price a stored row's tokens (ComponentSavedUSD); CostUSD below is untouched by it. e.BilledTokenFactor, e.BilledTokenFactorMeasured = tokens.BilledDeltaFactor(e.Model) // See Price.CostWithCacheWrite1h for the one-hour write premium this folds in. e.CostUSD = p.CostWithCacheWrite1h(e.FreshInput, e.CacheRead, e.CacheWrite, e.OutputTokens, e.CacheWrite1h) - e.BaselineCostUSD = e.CostUSD + e.baselineDeltaUSD(p) + e.BaselineCostUSD = e.CostUSD // What the PROVIDER's prompt cache saved on this request: every cache-read token was // billed at the read rate instead of the fresh rate it would have cost with no cache at // all. Kept as a measurement, reported as one, and never called a saving of ours — it is @@ -852,31 +859,27 @@ func (e *Event) Price(p modelinfo.Price, accountingComplete bool) { // NOT on a keep-alive ping. A ping's whole job is to read the prefix, so it reports a large // cache_read and would book ~$0.13 of "provider cache saving" per 48k ping against $0.0073 // of real cost — a fabricated saving on a figure PR #83 puts on the page, and at production - // ping volumes it is on the order of a thousand dollars. The counterfactual is the tell: - // without the keep-alive that ping does not exist, so there is no unbatched-fresh-input - // alternative for it to have saved anything against. + // ping volumes it is on the order of a thousand dollars. Without the keep-alive that ping + // does not exist, so there is no unbatched-fresh-input alternative for it to have saved + // anything against. if !e.KeepAlive { e.CacheSavedUSD = float64(e.CacheRead) * (p.Input - p.CacheRead) if e.CacheSavedUSD < 0 { e.CacheSavedUSD = 0 // a provider whose cache reads cost MORE than fresh input saved nothing } } - e.CachesplitSavedUSD = e.cachesplitSavedUSD(p) - e.KeepAliveSavedUSD = e.keepaliveSavedUSD(p) - // Per-component dollars, same rule and same rates as baselineDeltaUSD above: the unique - // part at the write rate it would have entered as, the re-sent remainder at the tier this - // request actually paid. Summed over a component's turns this IS the amortization — value - // realized turn by turn as the frozen reduction replays, not a projection. + e.CachesplitSavedUSD, e.KeepAliveSavedUSD = 0, 0 for i := range e.Components { - c := &e.Components[i] - c.SavedUSD = e.ComponentSavedUSD(p, c.Component, c.SavedGross, c.SavedUnique) + e.Components[i].SavedUSD = 0 } + return true } -// ComponentSavedUSD is ONE component's share of the baseline delta on this request: the -// single pricing path for a removal. Price stores it at write time; the read-time passes -// (dash/summarizeattr.go) call it again on a stored row so an old row and a new row are valued -// by the same arithmetic and cannot drift apart. +// ComponentSavedUSD is the LEGACY per-component rule: a removal priced from a row's own token +// counts. Nothing stores it any more; the accountant library prices every new row (PriceMode). It +// stays for rows written before the switch (priced_by ”), which the read-time passes value with it +// (readvalue.go, summarizeattr.go, replaycap.go), because a stored row has no thread history for +// the library to price from. A row the library priced is never re-priced with it. // // The first (unique) removal is priced by uniqueRate, the re-sent remainder by repeatRate. func (e *Event) ComponentSavedUSD(p modelinfo.Price, component string, gross, unique int) float64 { @@ -884,7 +887,7 @@ func (e *Event) ComponentSavedUSD(p modelinfo.Price, component string, gross, un gross = 0 } if unique > gross { - unique = gross // same clamp as baselineDeltaUSD: shared content keys can over-attribute + unique = gross // shared content keys can over-attribute } if unique < 0 { unique = 0 @@ -935,213 +938,6 @@ func (e *Event) uniqueRate(p modelinfo.Price, component string) float64 { return p.CacheWrite } -// cachesplitSavedUSD is the cache saving this project is willing to sign its name to. -// -// The mechanism: Claude Code appends a live environment snapshot (branch, git status, recent -// commits) to the END of its big system block, and that block is ONE cacheable unit whose -// breakpoint sits after the churn. So the provider's hash covers the volatile tail and the -// block re-bills as cache CREATION every time the snapshot changes. cachesplit splits the -// block in two and moves the breakpoint onto the stable half — same bytes to the model, a -// hash boundary that excludes the snapshot. -// -// Three conditions, each ruling out a way of being wrong: -// -// - SplitStableTokens > 0, i.e. the split actually happened on this request. It is set -// only where splitVolatileTail rewrote the block, which is exactly when cachesplit -// reports `mutated`, so this is the component test as well as the size. -// -// - the provider READ from cache, and read AT LEAST as much as the half we split off while -// WRITING less than that. A hit alone is not enough: on the first request after the -// stable half itself changed — someone edited CLAUDE.md, the tool list changed — an -// earlier agent breakpoint still hits, so cache_read > 0 while OUR half is being billed -// as creation on that very request. The write test is what excludes it. Neither test can -// isolate our breakpoint from the usage block, so both are deliberately blunt and both -// err towards refusing the credit. -// -// - the volatile TAIL CHANGED since this session's previous request (a session's first -// request counts as changed — there was nothing there to match). This is the condition -// that makes the hit ours: with the block unsplit, a moved snapshot re-creates the whole -// thing, while a tail that did not move would have been served from cache either way. -// -// It replaced "the session's first request", which sounded conservative and was simply -// wrong about how this traffic behaves. Measured over 1,127 stored sessions: 1,105 of -// the first requests were COLD — the previous session's cache had expired before the -// next one began — so that test reported ~$0 while the component was demonstrably -// serving the system prompt from cache on hundreds of mid-session turns. The turn that -// matters is the one where the agent commits or edits and the snapshot moves, which is -// mid-session, and which the SWE-bench A/B (0% -> 96.7% hit) was measuring all along. -// -// The comparison survives a restart: the tail hash is stored per request and the map is -// seeded from it (dash.Recorder.SeedSessions), so the first turn after a restart is not -// mistaken for a change. -// -// And the amount is the STABLE HALF, not the request's whole cache_read. That distinction was -// found by running the control arm rather than by reasoning about it. Real Claude Code -// sessions against litellm-gateway, each started after a fresh commit so the environment snapshot -// genuinely differed, first request of each session: -// -// first-request cache_read cache_write -// cachesplit on 54,304 1,030-1,065 -// cachesplit off (control) 45,805 9,555-9,564 -// -// The control still HIT. Claude Code sets several breakpoints and the ones before this block -// match whatever we do — so crediting the whole 54,304 would have booked the agent's own cache -// placement as ours. What the split moved is the difference the provider reports: 8,499 -// tokens, from the write tier to the read tier. -// -// We claim less than that. The split's own measurement of the half it moved is 5,654 tokens on -// that traffic, and the two disagree because they count different things — the provider reports -// block-granular usage over a prefix that also gained a block boundary, while this counts BPE -// tokens over the text. Both are evidence; the smaller one is what gets billed, because -// under-crediting is the only safe direction. -// -// In dollars on that request: $0.0099 ours against $0.0743 for the provider's whole cache read. -// 7.5x, and that 7.5x is the claim this replaced. -// -// The counterfactual is a cache MISS, not fresh input: those tokens carry cache_control, so -// a miss bills them as creation at 1.25x fresh, not at 1x. Hence CacheWrite - CacheRead, a -// 1.15x-fresh spread rather than 0.9x. The max(CacheWrite, Input) floor covers a provider that -// charges no write premium, where a miss still costs at least the fresh rate. -// -// It is a FLOOR: a stable prefix serves a whole session while this counts one request of it, -// and a session resumed after the TTL expires starts another first-request hit this cannot -// see. Under-crediting is the only direction a savings figure is allowed to be wrong in. -func (e *Event) cachesplitSavedUSD(p modelinfo.Price) float64 { - if !e.TailChanged || e.SplitStableTokens <= 0 { - return 0 - } - // The read has to be big enough to have included our half, and the write small enough - // not to have re-created it. Either failing means the stable half was not what got served - // from cache on this request, whatever else was. - n := int64(e.SplitStableTokens) - if e.CacheRead < n || e.CacheWrite >= n { - return 0 - } - miss := p.CacheWrite - if miss < p.Input { - miss = p.Input - } - delta := miss - p.CacheRead - if delta <= 0 { - return 0 - } - return float64(n) * delta -} - -// keepaliveSavedUSD is the cache re-creation a keep-alive ping avoided on THIS request. -// -// The mechanism, and why it is worth this much: the provider's default cache lifetime is -// five minutes, "measured from the start of the request that writes or reads the cache -// entry, not from the end of its response". A session idle longer than that loses its whole -// cached prefix, and the next turn re-bills every token of it at the creation rate. Measured -// over 19,805 production requests: 742 such requests cost $741.07 — 23.6% of all spend — at -// $0.9987 each against $0.1178 for a request that hit. Comparing each one against its own -// counterfactual (the same tokens billed as a read) puts the penalty at 11.35x, of which -// 91.2% is pure re-write. -// -// A cache READ refreshes the lifetime for free — "the cache is refreshed for no additional -// cost each time the cached content is used" — so a ping that re-reads the prefix at 0.1x -// buys back a re-creation at 1.25x. That 11.5:1 ratio is the whole economics, and it is why -// this is the one cache lever worth more than everything the transformation pipeline -// delivers. -// -// Four conditions, and each rules out a specific way of claiming money that is not ours: -// -// - A ping actually happened on this session during the idle span this request ended -// (KeepAlivePings > 0). Without it, every long-gap hit in the deployment would be -// credited to a mechanism that was switched off. -// - The gap exceeded the provider's lifetime. Inside the lifetime the entry would have -// survived on its own and the ping changed nothing — those pings are the measured waste -// the policy accepts (53 of 912 at the shipped X=280s, K=2, gated), and they must not also be -// credited. -// - The provider read from cache and read MORE than it wrote. A request that re-created -// most of its prefix did not get rescued, whatever else it read. -// - The credit is the smaller of what the ping refreshed and what this request read. The -// ping's own response says exactly how many tokens were in the entry it touched, so -// crediting more than that would claim tokens our ping never kept alive. -// -// One confound remains and is not removable from a single row: the provider's cache is keyed -// on CONTENT, so another session sending a byte-identical prefix inside the window would -// have refreshed the same entry for free. This is therefore the CEILING of what the ping -// earned on this row, and the counts beside it (pings sent, pings that read nothing) are what -// an operator checks it against. -// -// Priced against a cache MISS rather than fresh input, for the same reason -// cachesplitSavedUSD is: these tokens carry cache_control, so a miss bills them as creation -// at 1.25x, not at 1.0x. -func (e *Event) keepaliveSavedUSD(p modelinfo.Price) float64 { - if e.KeepAlive || e.KeepAlivePings <= 0 || e.CacheRead <= 0 { - return 0 // a ping never credits itself - } - if e.SinceLastMs <= keepaliveCreditFloorMs(e.Provider, e.Model, e.CacheTTL) { - return 0 - } - if e.CacheWrite >= e.CacheRead { - return 0 - } - n := e.CacheRead - if e.KeepAliveRefreshed > 0 && e.KeepAliveRefreshed < n { - n = e.KeepAliveRefreshed - } - miss := p.CacheWrite - if miss < p.Input { - miss = p.Input - } - delta := miss - p.CacheRead - if delta <= 0 { - return 0 - } - return float64(n) * delta -} - -// keepaliveCreditFloorMs is how long a gap must be before a ping can take credit for the return: -// the cache lifetime of the thread. cacheTTL is the tier the returning request ASKED for, which -// for a given client is constant across a session (Claude Code asks 1h on a subscription for every -// request). A 1-hour thread is pinged at 59 min, so crediting any gap over 5 minutes would claim -// returns the entry survived on its own. -func keepaliveCreditFloorMs(provider, model, cacheTTL string) int64 { - if provider == string(bschemas.OpenAI) { - if !modelinfo.GPT56OrLater(model) { - return 1<<63 - 1 // no known lifetime means no defensible credit - } - } - return modelinfo.CacheLifetime(provider, model, cacheTTL == apply.TTLEphemeral1h).Milliseconds() -} - -// baselineDeltaUSD is what the removed content would have cost had it been sent: -// the unique part as new input (cache-write rate), the re-sent remainder as a -// cache read — with the token counts corrected to the provider's own count. -// -// The correction (BilledTokenFactor) is issue #240: the counts here are -// schema.MessagesTokens, an o200k_base estimate, while the RATE is the provider's. Pricing -// one at the other under-reported this delta on every Claude row. See -// tokens.BilledDeltaFactor for how the factor was measured and what it does not claim. -func (e *Event) baselineDeltaUSD(p modelinfo.Price) float64 { - unique := e.SavedUnique - gross := e.Saved() - // SavedUnique is attributed per component and can exceed the request's own - // gross saving when several components stash the same content key; clamp it, so - // the repeat term can never go negative and inflate the baseline. - if unique > gross { - unique = gross - } - if unique < 0 { - unique = 0 - } - // The part of the unique removal that belongs to a summarizer is priced by its own rate (see - // uniqueRate); every other component's unique part stays at the write rate. - sumUnique := 0 - for _, c := range e.Components { - if IsSummarizer(c.Component) && c.SavedUnique > 0 { - sumUnique += min(c.SavedUnique, max(c.SavedGross, 0)) - } - } - sumUnique = min(sumUnique, unique) - raw := float64(unique-sumUnique)*p.CacheWrite + float64(sumUnique)*e.uniqueRate(p, SummarizerComponents[0]) + - float64(gross-unique)*e.repeatRate(p) - return raw * e.BilledTokenFactor -} - // repeatRate is what the RE-SENT part of the removed content would have been billed // at on this request. // diff --git a/dash/honesty_test.go b/dash/honesty_test.go index 6121c00e..abeef525 100644 --- a/dash/honesty_test.go +++ b/dash/honesty_test.go @@ -304,75 +304,6 @@ var tierCases = []struct { {"non_caching", 0, 0, 80_000, func(p modelinfo.Price) float64 { return p.Input }}, } -// TestPerComponentSavedUSDReconcilesWithTheBaseline is the identity that makes the -// per-component dollars trustworthy: they are a PARTITION of the request's own saving, not -// a second, independently-computed estimate that can drift from the headline. Reconciled -// against production data at 0.9%; on a fixture whose component rows sum exactly to the -// request's, it has to be exact to float noise. -func TestPerComponentSavedUSDReconcilesWithTheBaseline(t *testing.T) { - for _, tc := range tierCases { - t.Run(tc.name, func(t *testing.T) { - e := perComponentEvent(tc.read, tc.write, tc.fresh) - e.Price(ibmOpus, true) - var sum float64 - for _, c := range e.Components { - sum += c.SavedUSD - } - want := e.BaselineCostUSD - e.CostUSD - if want <= 0 { - t.Fatalf("fixture removed nothing: baseline %.10f cost %.10f", e.BaselineCostUSD, e.CostUSD) - } - if math.Abs(sum-want) > 1e-12 { - t.Errorf("Σ per-component saved_usd = %.10f, request-level saving = %.10f (error %.3g); "+ - "the components view and the headline would disagree", - sum, want, math.Abs(sum-want)/want) - } - }) - } -} - -// TestComponentSavingUsesTheTierTheRequestPaid: the same removals are worth different money -// depending on how the provider billed THIS request, and a component whose saving is pure -// replay earns that tier — not the creation rate. Pricing replay as creation is the ~12.5x -// overstatement the request-level figure already avoids; this pins the per-component path -// against the same mistake. -func TestComponentSavingUsesTheTierTheRequestPaid(t *testing.T) { - for _, tc := range tierCases { - t.Run(tc.name, func(t *testing.T) { - e := perComponentEvent(tc.read, tc.write, tc.fresh) - e.Price(ibmOpus, true) - tier := tc.tier(ibmOpus) - for _, want := range []struct { - comp string - unique, replay float64 - }{ - {"mask", 900, 11_100}, - {"extract_llm", 1_100, 4_900}, - {"collapse", 0, 2_000}, // pure replay - {"cacheinject", 0, 0}, - } { - got := compByName(t, e, want.comp).SavedUSD - // Scaled by the row's own tokenizer correction (#240) rather than a literal, - // so this test keeps pinning the TIER RULE and does not have to be edited - // every time the measured factor is refreshed. - exp := (want.unique*ibmOpus.CacheWrite + want.replay*tier) * e.BilledTokenFactor - if math.Abs(got-exp) > 1e-12 { - t.Errorf("%s saved_usd = %.10f, want %.10f (unique at the write rate, "+ - "replay at this request's %s tier)", want.comp, got, exp, tc.name) - } - } - // And on a warm turn the replay term must NOT be the creation rate, which is the - // specific inflation the tier rule exists to prevent. - if tc.name == "warm" { - if got, ceiling := compByName(t, e, "collapse").SavedUSD, 2_000*ibmOpus.CacheWrite*e.BilledTokenFactor; got >= ceiling { - t.Errorf("a pure-replay component on a warm turn is valued at %.10f, at or above "+ - "the creation rate %.10f — replay sits in the cached prefix", got, ceiling) - } - } - }) - } -} - func compByName(t *testing.T, e *Event, name string) CompRow { t.Helper() for _, c := range e.Components { @@ -621,7 +552,7 @@ func TestPreColumnComponentRowsAreValuedNotZeroed(t *testing.T) { t.Run(tc.name, func(t *testing.T) { db := openTestDB(t) e := perComponentEvent(tc.read, tc.write, tc.fresh) - e.Price(ibmOpus, true) + legacyPrice(e, ibmOpus) // history: written by the legacy rule, before the accountant library if err := db.insertBatch([]*Event{e}); err != nil { t.Fatal(err) } diff --git a/dash/keepalive.go b/dash/keepalive.go index cb60d0b6..d0ef81bb 100644 --- a/dash/keepalive.go +++ b/dash/keepalive.go @@ -99,7 +99,7 @@ func kaSavedSince(alias string, since int64) string { // not make, applied on read. pingFloor is one extra predicate on the ping row (kp), "" for // none — see kaSavedSince for the only caller that needs one. // -// The credit written at capture time asks four questions (Event.keepaliveSavedUSD) and not one +// The credit the legacy rule wrote at capture time asked four questions and not one // of them is an upper bound on the idle gap: it requires that the gap EXCEEDED the provider's // lifetime — the floor — and then credits the row whatever the gap was, on the strength of // keepalive_pings alone. That column is the whole of the write path's evidence, and on real rows @@ -187,10 +187,10 @@ func kaSavedGate(alias, pingFloor string) string { // // So no page treats a pre-GPT-5.6 OpenAI model as 30 minutes through this CASE: the outer // `keepalive_saved_usd > 0` is true only for a row the write path credited, and the write path - // never credits one (keepaliveSavedUSD gates on GPT56OrLater). The provider-only CASE sees + // never credited one (the legacy rule gated on GPT56OrLater). The provider-only CASE sees // only rows that already passed the model gate. // - // This is a reachability ceiling for rows ALREADY credited by keepaliveSavedUSD, + // This is a reachability ceiling for rows ALREADY credited at write time, // not a second model-eligibility rule. The write path uses GPT56OrLater and // never credits older OpenAI models; SQL needs only the provider's coverage // window for a successful ping. Duplicating Go's routed-model parser here @@ -231,7 +231,7 @@ type KeepAliveLedger struct { PingUSD float64 `json:"ping_usd"` MissesAvoided int64 `json:"misses_avoided"` // SavedUSD is a CEILING, and every tile that shows it says so. The credit itself is exact - // arithmetic — see Event.keepaliveSavedUSD — but the provider's cache is keyed on CONTENT, + // arithmetic (the accountant library's rescue) — but the provider's cache is keyed on CONTENT, // so another session sending a byte-identical prefix would have refreshed the same entry // for nothing. That confound cannot be measured from our side; it can only be declared. SavedUSD float64 `json:"saved_usd"` diff --git a/dash/keepalive_ttl_test.go b/dash/keepalive_ttl_test.go index 4a7bde49..75c3b4fb 100644 --- a/dash/keepalive_ttl_test.go +++ b/dash/keepalive_ttl_test.go @@ -2,29 +2,6 @@ package dash import "testing" -// A 1-hour thread is pinged at 59 minutes, so the credit floor is an hour, not five minutes. -func TestKeepAliveCreditFloorFollowsTheRequestsTier(t *testing.T) { - const min = 60 * 1000 - for _, tc := range []struct { - name string - ttl string - gap int64 - want bool - }{ - {"5m tier, 20 min gap: the entry had expired", "ephemeral_5m", 20 * min, true}, - {"1h tier, 20 min gap: the entry survived alone", "ephemeral_1h", 20 * min, false}, - {"1h tier, 59 min gap: still inside the hour", "ephemeral_1h", 59 * min, false}, - {"1h tier, 80 min gap: the 59-minute ping rescued it", "ephemeral_1h", 80 * min, true}, - } { - e := keepaliveEvent(tc.gap, 1, 48_576, 48_576, 0) - e.CacheTTL = tc.ttl - e.Price(ibmSonnet, true) - if got := e.KeepAliveSavedUSD > 0; got != tc.want { - t.Errorf("%s: credited=%v, want %v", tc.name, got, tc.want) - } - } -} - // kaSaved is the read-side reachability check. With the 5-minute window every rescue of a 1-hour // thread was zeroed (ping at 59 min, return at 80 min: 21 min > 5 min) while the ping's cost still // counted, so the ledger went negative and insights told the user to switch keep-alive off. diff --git a/dash/keepalivecost_test.go b/dash/keepalivecost_test.go index 87c90a88..1351c6a8 100644 --- a/dash/keepalivecost_test.go +++ b/dash/keepalivecost_test.go @@ -19,81 +19,6 @@ func keepaliveEvent(gapMs int64, pings int, refreshed, read, write int64) *Event return e } -// The credit exists at all: a request that came back after twenty minutes and was served from -// cache anyway did not pay the 11.35x re-creation penalty, and a ping is why. -func TestKeepAliveCreditsARescuedRequest(t *testing.T) { - e := keepaliveEvent(20*60*1000, 1, 48_576, 48_576, 0) - // Priced against a cache MISS, not fresh input: these tokens carry cache_control, so a - // miss bills them as creation at 1.25x rather than at 1x. - want := 48_576 * (ibmSonnet.CacheWrite - ibmSonnet.CacheRead) - if got := e.KeepAliveSavedUSD; got < want*0.999 || got > want*1.001 { - t.Errorf("keepalive_saved_usd = %.6f, want %.6f (cache_read x (write - read))", got, want) - } -} - -// Each gate, and the specific over-claim it prevents. Every one of these would otherwise -// credit this mechanism with money it did not earn. -func TestKeepAliveCreditGates(t *testing.T) { - for _, tc := range []struct { - name string - e *Event - why string - }{ - {"no ping was sent", keepaliveEvent(20*60*1000, 0, 0, 48_576, 0), - "every long-gap hit in the deployment would be credited to a mechanism that was off"}, - {"the gap was inside the provider lifetime", keepaliveEvent(120*1000, 2, 48_576, 48_576, 0), - "the entry would have survived on its own; those pings are the measured waste, not a saving"}, - {"the request missed anyway", keepaliveEvent(20*60*1000, 1, 48_576, 0, 48_576), - "nothing was rescued, so there is nothing to credit"}, - {"it re-created more than it read", keepaliveEvent(20*60*1000, 1, 48_576, 1_000, 48_576), - "a request that re-wrote most of its prefix was not kept warm"}, - {"the row IS a ping", func() *Event { - e := keepaliveEvent(20*60*1000, 1, 48_576, 48_576, 0) - e.KeepAlive, e.KeepAliveSavedUSD = true, 0 - e.Price(ibmSonnet, true) - return e - }(), "a ping crediting itself is circular"}, - } { - if got := tc.e.KeepAliveSavedUSD; got != 0 { - t.Errorf("%s: credited $%.6f — %s", tc.name, got, tc.why) - } - } -} - -func TestOpenAIKeepAliveCreditWaitsForTheThirtyMinuteGuarantee(t *testing.T) { - for _, tc := range []struct { - gap int64 - want bool - }{ - {29 * 60 * 1000, false}, - {31 * 60 * 1000, true}, - } { - e := keepaliveEvent(tc.gap, 1, 8000, 8000, 0) - e.Provider, e.Model = "openai", "azure/gpt-5.6-luna" - e.Price(ibmSonnet, true) - if got := e.KeepAliveSavedUSD > 0; got != tc.want { - t.Errorf("gap %d ms: credited=%v, want %v", tc.gap, got, tc.want) - } - } -} - -// The credit is capped by what the ping actually refreshed. The ping's own response says how -// many tokens were in the entry it touched, and claiming more than that would claim tokens our -// ping never kept alive — for instance a prefix that grew between the ping and the next turn. -func TestKeepAliveCreditIsCappedByWhatThePingRefreshed(t *testing.T) { - e := keepaliveEvent(20*60*1000, 1, 10_000, 48_576, 0) - want := 10_000 * (ibmSonnet.CacheWrite - ibmSonnet.CacheRead) - if got := e.KeepAliveSavedUSD; got < want*0.999 || got > want*1.001 { - t.Errorf("keepalive_saved_usd = %.6f, want %.6f (capped at the ping's own cache_read)", - got, want) - } - // And a ping that read nothing (the entry was already gone) caps at nothing extra: the - // request's own read is then the only evidence, which is the pre-cap behaviour. - if e2 := keepaliveEvent(20*60*1000, 1, 0, 48_576, 0); e2.KeepAliveSavedUSD <= 0 { - t.Error("a ping with no recorded refresh blocked the credit entirely") - } -} - // A ping row must be priced like any other request — its cost is the whole point of the // audit trail — while contributing nothing to the saving side of the ledger. func TestKeepAlivePingRowIsPricedAndNotCredited(t *testing.T) { diff --git a/dash/openreadonly_test.go b/dash/openreadonly_test.go index 1b846ff2..776578bc 100644 --- a/dash/openreadonly_test.go +++ b/dash/openreadonly_test.go @@ -157,3 +157,37 @@ func TestKVCacheSuggestRunsOnAReadOnlyHandle(t *testing.T) { t.Fatalf("KVCacheSuggest on a read-only handle: %v", err) } } + +// A file written before the split columns existed lacks request_components.priced_by and its two +// siblings. A read-only handle cannot ALTER, so the shim must supply them, or a reader of an older +// file fails with "no such column". +func TestOpenReadOnlyShimsAnOlderRequestComponentsTable(t *testing.T) { + path := seededFile(t) + rw, err := Open(path) + if err != nil { + t.Fatal(err) + } + for _, c := range []string{"priced_by", "saved_first_usd", "saved_replay_usd"} { + if _, err := rw.sql.Exec(`ALTER TABLE request_components DROP COLUMN ` + c); err != nil { + t.Fatalf("drop %s: %v", c, err) + } + } + if err := rw.Close(); err != nil { + t.Fatal(err) + } + db, err := OpenReadOnly(path) + if err != nil { + t.Fatalf("OpenReadOnly: %v", err) + } + defer db.Close() + var n int + var by string + var first float64 + if err := db.sql.QueryRow(`SELECT COUNT(*), COALESCE(MAX(priced_by), ''), COALESCE(MAX(saved_first_usd), 0) + FROM request_components`).Scan(&n, &by, &first); err != nil { + t.Fatalf("reading the missing columns: %v", err) + } + if by != "" || first != 0 { + t.Fatalf("missing columns read as %q, %v; want their defaults", by, first) + } +} diff --git a/dash/overview.go b/dash/overview.go index 71ae8252..bd635e7f 100644 --- a/dash/overview.go +++ b/dash/overview.go @@ -167,7 +167,7 @@ type Overview struct { // CachesplitSavedUSD is the cache saving that IS ours: summed over requests where the // volatile-tail split ran, the snapshot had MOVED since the session's previous request, // and the provider then read at least the stable half from cache while writing less than - // it. Priced against a cache miss. A floor. See Event.cachesplitSavedUSD. + // it. Priced against a cache miss. A floor. The accountant library's split effect. CachesplitSavedUSD float64 `json:"cachesplit_saved_usd"` // DeferNetUSD is tool deferral's NET saving, OURS and MEASURED: the sum of the per-request // `defer_net_usd` the proxy wrote (gross minus the names list and the searches). It is in @@ -177,7 +177,7 @@ type Overview struct { // mysterious. This mattered: when the credit was gated on the session's FIRST REQUEST the // figure read ~$0 on real traffic, and the reason was visible only in these counts — // 1,105 of 1,127 session starts had no cache to hit, because the previous session's had - // expired. That gate is gone (it is TailChanged now, see Event.cachesplitSavedUSD); the + // expired. That gate is gone (the split is priced by the accountant library now); the // counts stay, because the replacement gate also fires rarely and for its own reason. // // SplitRequests is requests the split ran on; SplitTailMoved is the subset whose snapshot @@ -385,8 +385,8 @@ type Overview struct { // // DOLLARS ARE NOT UNAFFECTED. This comment used to end "Dollars are unaffected: they come // from the provider's own reported usage", which is true of cost_usd and false of every - // savings figure beside it: baseline_cost_usd is cost_usd + baselineDeltaUSD, and that - // delta prices OUR token counts (Saved(), SavedUnique) at the provider's rate. A reader + // savings figure beside it: baseline_cost_usd is cost_usd plus a delta that prices OUR + // token counts at the provider's rate (the accountant library's bytes-to-tokens ratio). A reader // auditing exactly this was told not to look, which is how #240 survived. The correction // factor is NOT this ratio — see tokens.BilledDeltaFactor, which is a factor on a // DIFFERENCE, measured against real bills, where the fixed overhead above cancels. @@ -581,7 +581,7 @@ func (d *DB) Overview(f Filter) (*Overview, error) { COALESCE(SUM(r.cache_saved_usd),0), -- Our own cache saving. A plain SUM of a per-request column, because all three -- conditions that qualify a request were settled at write time where the model's - -- rates and the session's history were both in hand (Event.cachesplitSavedUSD). + -- rates and the session's history were both in hand (the accountant library). -- The predecessor did the component test here as a correlated EXISTS over -- request_components, which was both slower and wrong: it credited every cache read -- in a session, including the later turns that would have hit anyway. diff --git a/dash/query.go b/dash/query.go index f867441a..016c791f 100644 --- a/dash/query.go +++ b/dash/query.go @@ -193,13 +193,13 @@ var requestCols = `r.id, r.ts, r.tenant_id, r.session_id, r.model, r.provider, r r.agent_compaction, r.cache_miss_cause, r.manual_ping, r.manual_saved_usd, r.project, r.source, r.sweep_expands, r.sweep_expand_tokens, r.sweep_false_expands, r.sweep_answer_usd, - r.session_alias, + r.session_alias, r.price_estimated, r.defer_tokens, r.defer_list_tokens, r.defer_searches, r.defer_loaded, r.defer_wrong_guesses, r.defer_gross_usd, r.defer_net_usd, r.defer_dropped_searches` func scanRequest(rows interface{ Scan(...any) error }) (*Event, error) { var e Event - var byp, ca, stream, keepAlive, agentComp, manual int + var byp, ca, stream, keepAlive, agentComp, manual, estimated int var temp, topP sql.NullFloat64 var waitingFor, waitTool, wakeCause sql.NullString var openBackground, openTeammate, monitorExpiryMs sql.NullInt64 @@ -219,9 +219,10 @@ func scanRequest(rows interface{ Scan(...any) error }) (*Event, error) { &agentComp, &e.CacheMissCause, &manual, &e.ManualSavedUSD, &e.Project, &e.Source, &e.SweepExpands, &e.SweepExpandTokens, &e.SweepFalseExpands, &e.SweepAnswerUSD, - &e.SessionAlias, + &e.SessionAlias, &estimated, &e.DeferTokens, &e.DeferListTokens, &e.DeferSearches, &e.DeferLoaded, &e.DeferWrongGuesses, &e.DeferGrossUSD, &e.DeferNetUSD, &e.DeferDroppedSearches) + e.PriceEstimated = estimated != 0 e.WaitingFor, e.WaitTool, e.WakeCause = waitingFor.String, waitTool.String, wakeCause.String e.OpenBackground, e.OpenTeammate = int(openBackground.Int64), int(openTeammate.Int64) e.MonitorExpiryMs = monitorExpiryMs.Int64 @@ -308,7 +309,8 @@ func (d *DB) Request(id int64, withContent bool) (*Event, error) { return nil, err } crows, err := d.sql.QueryContext(d.readCtx(), `SELECT component, kind, acted, mutated, reverted, skipped, - saved_gross, saved_unique, saved_usd, duration_ms, err, gates, events FROM request_components + saved_gross, saved_unique, saved_usd, duration_ms, err, gates, events, + saved_first_usd, saved_replay_usd, priced_by FROM request_components WHERE request_id = ? ORDER BY rowid`, id) if err != nil { return nil, err @@ -320,7 +322,7 @@ func (d *DB) Request(id int64, withContent bool) (*Event, error) { var gates, events string if err := crows.Scan(&c.Component, &c.Kind, &a, &m, &rv, &sk, &c.SavedGross, &c.SavedUnique, &c.SavedUSD, &c.DurationMs, &c.Err, - &gates, &events); err != nil { + &gates, &events, &c.SavedFirstUSD, &c.SavedReplayUSD, &c.PricedBy); err != nil { return nil, err } c.Acted, c.Mutated, c.Reverted, c.Skipped = a != 0, m != 0, rv != 0, sk != 0 @@ -806,6 +808,10 @@ type ComponentRow struct { // runs the identical formula, so agreement is arithmetic. The UI prints the covered // fraction rather than presenting the whole comparison as evidence. SavedUSDDecomposedStored float64 `json:"saved_usd_decomposed_stored"` + // SavedUSDCosts is the costs the accountant library netted into saved_usd on the rows it + // priced: added tokens and the breaks a component caused. Negative. It is the third term of + // SavedUSDDecomposed on those rows (first removal + replay + costs = saved_usd); 0 before them. + SavedUSDCosts float64 `json:"saved_usd_costs,omitempty"` // UniqueUnkeyed marks a component whose saved_unique is not a dedup measurement, because // it reports no content keys and MarkUnique therefore returns its saving in full every // turn. True for every reformatter. See DecomposeComponentSavedUSD for why this is @@ -1145,6 +1151,9 @@ func (d *DB) components(f Filter) ([]*ComponentRow, map[compGroupKey]*compGroup, type compGroupKey struct { comp, model, tier string usd, gross, complete bool + // acc is a group of rows the accountant library priced (priced_by): their dollar split is + // stored (firstUSD, replayUSD), and no read-time formula re-prices them. + acc bool } // compGroup is one cell of that pass. uClamp and rClamp are the decomposition's unique and replay @@ -1155,6 +1164,7 @@ type compGroup struct { runs, acted, mutated, reverted, skipped, gross, unique int64 errs, uClamp, rClamp, nDiff int64 usd, dur float64 + firstUSD, replayUSD float64 // the stored split, on an acc group } func sortedCompKeys(groups map[compGroupKey]*compGroup) []compGroupKey { @@ -1194,13 +1204,14 @@ func (d *DB) componentGroups(cond string, args []any) (map[compGroupKey]*compGro CASE WHEN r.cache_read > 0 THEN 'read' WHEN r.cache_write > 0 AND r.cache_write >= r.fresh_input THEN 'write' ELSE 'fresh' END, - c.saved_usd <> 0, c.saved_gross > 0, r.token_accounting = 'complete', + c.saved_usd <> 0, c.saved_gross > 0, r.token_accounting = 'complete', c.priced_by <> '', MAX(c.kind), COUNT(*), SUM(c.acted), SUM(c.mutated), SUM(c.reverted), SUM(c.skipped), SUM(c.saved_gross), SUM(c.saved_unique), SUM(c.saved_usd), SUM(c.duration_ms), SUM(CASE WHEN c.err <> '' THEN 1 ELSE 0 END), - SUM(`+uniq+`), SUM(`+gross+` - `+uniq+`), SUM(CASE WHEN c.saved_gross <> c.saved_unique THEN 1 ELSE 0 END) + SUM(`+uniq+`), SUM(`+gross+` - `+uniq+`), SUM(CASE WHEN c.saved_gross <> c.saved_unique THEN 1 ELSE 0 END), + SUM(c.saved_first_usd), SUM(c.saved_replay_usd) FROM request_components c JOIN requests r ON r.id = c.request_id - WHERE `+cond+pred+` GROUP BY 1, 2, 3, 4, 5, 6`, + WHERE `+cond+pred+` GROUP BY 1, 2, 3, 4, 5, 6, 7`, append(append([]any(nil), args...), bounds...)...) if err != nil { return err @@ -1210,9 +1221,9 @@ func (d *DB) componentGroups(cond string, args []any) (map[compGroupKey]*compGro var k compGroupKey var g compGroup var usd sql.NullFloat64 - if err := rows.Scan(&k.comp, &k.model, &k.tier, &k.usd, &k.gross, &k.complete, + if err := rows.Scan(&k.comp, &k.model, &k.tier, &k.usd, &k.gross, &k.complete, &k.acc, &g.kind, &g.runs, &g.acted, &g.mutated, &g.reverted, &g.skipped, &g.gross, &g.unique, - &usd, &g.dur, &g.errs, &g.uClamp, &g.rClamp, &g.nDiff); err != nil { + &usd, &g.dur, &g.errs, &g.uClamp, &g.rClamp, &g.nDiff, &g.firstUSD, &g.replayUSD); err != nil { return err } g.usd = usd.Float64 @@ -1236,6 +1247,8 @@ func (d *DB) componentGroups(cond string, args []any) (map[compGroupKey]*compGro m.nDiff += g.nDiff m.usd += g.usd m.dur += g.dur + m.firstUSD += g.firstUSD + m.replayUSD += g.replayUSD } mu.Unlock() } diff --git a/dash/readvalue.go b/dash/readvalue.go index 329788fd..16aed16c 100644 --- a/dash/readvalue.go +++ b/dash/readvalue.go @@ -191,8 +191,28 @@ func decomposeFromGroups(cgroups map[compGroupKey]*compGroup, p modelinfo.Pricer } type dgVal struct{ unique, replay, nRows, nDiff int64 } groups := map[dgKey]*dgVal{} + // Rows the accountant library priced carry their split already (saved_first_usd and + // saved_replay_usd): it is summed as stored, never re-priced, because the library priced from + // the thread's history, which the row does not keep. The formula below values only the rows + // written before the switch. for k, g := range cgroups { - if !k.gross || !k.complete { + if !k.acc { + continue + } + seen[k.comp] += g.runs + differ[k.comp] += g.nDiff + if c, ok := by[k.comp]; ok { + c.SavedUSDFirstRemoval += g.firstUSD + c.SavedUSDReplay += g.replayUSD + // The stored side of the cross-check is saved_usd, which nets the costs (added tokens, + // breaks): they are part of this side too, so the two agree on an accountant row. + costs := g.usd - g.firstUSD - g.replayUSD + c.SavedUSDDecomposedStored += g.usd + c.SavedUSDCosts += costs + } + } + for k, g := range cgroups { + if k.acc || !k.gross || !k.complete { continue } stored := 0 @@ -268,10 +288,10 @@ func decomposeFromGroups(cgroups map[compGroupKey]*compGroup, p modelinfo.Pricer } } for _, c := range out { - c.SavedUSDDecomposed = c.SavedUSDFirstRemoval + c.SavedUSDReplay + c.SavedUSDDecomposed = c.SavedUSDFirstRemoval + c.SavedUSDReplay + c.SavedUSDCosts c.NetUSDFirstRemoval = c.SavedUSDFirstRemoval - c.LLMCostUSD if c.SavedUSDFirstRemoval > 0 { - c.ReplayMultiple = c.SavedUSDDecomposed / c.SavedUSDFirstRemoval + c.ReplayMultiple = (c.SavedUSDFirstRemoval + c.SavedUSDReplay) / c.SavedUSDFirstRemoval } // Whether this component's `unique` is a DEDUP MEASUREMENT at all. // @@ -352,7 +372,8 @@ func estimateFromGroups(cgroups map[compGroupKey]*compGroup, p modelinfo.Pricer, // cannot say" and "it was worth nothing" are different answers. unpriced := map[string]int64{} for k, g := range cgroups { - if k.usd || !k.gross { + // A row the accountant priced is never estimated: its saved_usd is a figure, also when 0. + if k.acc || k.usd || !k.gross { continue } if !k.complete { diff --git a/dash/replaycap.go b/dash/replaycap.go index 6e0fa1da..99544dff 100644 --- a/dash/replaycap.go +++ b/dash/replaycap.go @@ -135,6 +135,10 @@ type compCredit struct { Unique int64 USD float64 FirstUSD float64 + // Accountant marks a credit the accountant library priced: storedFirst is its first-removal + // part as stored, and no read-time formula splits it again (firstShare). + Accountant bool + storedFirst float64 } // capRow is one request as the cap walk needs it. @@ -300,7 +304,7 @@ func walkCapConversation(conv []capRow, window windowFn, compactionTS []int64, o const capSelect = `SELECT r.id, r.tenant_id, r.session_id, r.model, r.ts, r.bypassed, r.agent_compaction, r.tokens_before, r.tokens_after, r.fresh_input + r.cache_read + r.cache_write, r.cache_miss_reason, r.fresh_input, r.cache_read, r.cache_write, - c.component, c.saved_gross, c.saved_unique, c.saved_usd + c.component, c.saved_gross, c.saved_unique, c.saved_usd, c.saved_first_usd, c.priced_by <> '' FROM requests r LEFT JOIN request_components c ON c.request_id = r.id AND %c @@ -380,6 +384,10 @@ func (d *DB) ReplayLedgerForCtx(ctx context.Context, f Filter, windows modelinfo } for j := range r.Credits { c := &r.Credits[j] + if c.Accountant { + c.FirstUSD = c.storedFirst + continue + } c.FirstUSD = c.USD * firstShare(&ev, pr, priced, c.Component, c.Gross, c.Unique) } } @@ -392,10 +400,11 @@ func (d *DB) ReplayLedgerForCtx(ctx context.Context, f Filter, windows modelinfo var bypassed, agentComp int var comp sql.NullString var gross, unique sql.NullInt64 - var usd sql.NullFloat64 + var usd, first sql.NullFloat64 + var acc sql.NullBool if err := rs.Scan(&id, &r.Tenant, &r.Session, &r.Model, &r.TS, &bypassed, &agentComp, &r.TokensBefore, &r.TokensAfter, &r.Billed, &r.reason, &r.fresh, &r.read, &r.write, - &comp, &gross, &unique, &usd); err != nil { + &comp, &gross, &unique, &usd, &first, &acc); err != nil { return nil, err } if id != lastID { @@ -422,7 +431,8 @@ func (d *DB) ReplayLedgerForCtx(ctx context.Context, f Filter, windows modelinfo continue } cur := &buf[len(buf)-1] - c := compCredit{Component: comp.String, Gross: gross.Int64, Unique: unique.Int64, USD: usd.Float64} + c := compCredit{Component: comp.String, Gross: gross.Int64, Unique: unique.Int64, USD: usd.Float64, + Accountant: acc.Bool, storedFirst: first.Float64} dup := false for i := range cur.Credits { // request_components has no UNIQUE: keep the larger of duplicates if cur.Credits[i].Component == c.Component { diff --git a/dash/schema.go b/dash/schema.go index 9809f7b9..c1e4d5fc 100644 --- a/dash/schema.go +++ b/dash/schema.go @@ -101,6 +101,10 @@ CREATE TABLE IF NOT EXISTS requests ( -- exactly this id. '' when there is no separate alias, and on every row written before this -- column existed. A hash, not content: the same kind of id session_id itself is. session_alias TEXT NOT NULL DEFAULT '', + -- 1 when the accountant library could not rule out a cache break (penalty_unknown), so the booked + -- net saving may be too high: the gross is right, the penalty is not known. 0 on an exact figure + -- and on every row written before this column existed, which is the claim those rows always made. + price_estimated INTEGER NOT NULL DEFAULT 0, preset TEXT NOT NULL DEFAULT '', mode TEXT NOT NULL DEFAULT '', -- operating mode (active|bypass|observe) route TEXT NOT NULL DEFAULT '', @@ -129,8 +133,8 @@ CREATE TABLE IF NOT EXISTS requests ( -- provider read at least that much from cache while writing less, and the snapshot had MOVED -- since this session's previous request (Event.TailChanged) — the turn that would otherwise -- have missed. Priced against a cache MISS (creation rate), not fresh input. - -- See Event.cachesplitSavedUSD, which is the authority on the condition; the - -- session's-first-request test it once used was REPLACED because it reported ~$0, and three + -- Written from the accountant library's split effect. The legacy rule's + -- session's-first-request test was REPLACED because it reported ~$0, and three -- descriptions of this column went on stating it for a while after. cachesplit_saved_usd REAL NOT NULL DEFAULT 0, -- The size of the prefix half cachesplit moved the breakpoint onto: the numerator of the @@ -336,6 +340,10 @@ CREATE TABLE IF NOT EXISTS request_components ( -- that was in force, and because the UI was otherwise left improvising a dollar value -- from hardcoded rates. saved_usd REAL NOT NULL DEFAULT 0, + -- The accountant library's split of saved_usd and who priced the row (additiveColumns). + saved_first_usd REAL NOT NULL DEFAULT 0, + saved_replay_usd REAL NOT NULL DEFAULT 0, + priced_by TEXT NOT NULL DEFAULT '', duration_ms REAL NOT NULL DEFAULT 0, err TEXT NOT NULL DEFAULT '', -- Why this component turned candidates away, as {"gate_name":count} — the only answer @@ -756,6 +764,7 @@ var additiveColumns = []struct{ table, column, ddl string }{ {"requests", "filtered_decl_tokens", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "decl_digest", "TEXT NOT NULL DEFAULT ''"}, {"requests", "session_alias", "TEXT NOT NULL DEFAULT ''"}, + {"requests", "price_estimated", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "cache_ttl", "TEXT NOT NULL DEFAULT ''"}, {"requests", "sse_buffered", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "ttfb_ms", "REAL NOT NULL DEFAULT 0"}, @@ -804,6 +813,16 @@ var additiveColumns = []struct{ table, column, ddl string }{ // records having got wrong once already. {"request_components", "events", "TEXT NOT NULL DEFAULT ''"}, {"request_components", "saved_usd", "REAL NOT NULL DEFAULT 0"}, + // The accountant library's split of saved_usd, stored at write time because the library prices + // from the thread's history, which a stored row does not keep, so a read cannot re-price it. + // saved_first_usd is the saved tokens it priced at the write or input tier (content the provider + // had not cached), saved_replay_usd those at the read tier (content in the cached prefix), and + // saved_usd - first - replay is the costs (added tokens, breaks, the head-TTL legs). priced_by is + // 'accountant' on such a row; '' on a row from before the switch, which the read views still + // value with the legacy estimate (readvalue.go). + {"request_components", "saved_first_usd", "REAL NOT NULL DEFAULT 0"}, + {"request_components", "saved_replay_usd", "REAL NOT NULL DEFAULT 0"}, + {"request_components", "priced_by", "TEXT NOT NULL DEFAULT ''"}, // extract_llm_sweep's quality signals and answer cost (#404 PR C). Additive with a default of 0: // a row from before the sweep applied anything reads as "no sweep activity", which is true. // sweep_expands / sweep_expand_tokens are the part of expands / expand_tokens that restored a diff --git a/dash/store.go b/dash/store.go index f88df169..0ff1a859 100644 --- a/dash/store.go +++ b/dash/store.go @@ -337,8 +337,8 @@ func OpenReadOnly(path string) (*DB, error) { // file written by an older proxy may lack (the Ping button's manual_ping among them). Failing each // query with "no such column" would break a reader (the cache-campaign tool opens live databases // this way, and operators mix tool and proxy versions), so each read-only connection gets a TEMP -// view named `requests` that adds every missing column as its default. Temp objects are searched -// before main, so every unqualified `requests` resolves to the shim; the view is per connection and +// view named after each table with additive columns (`requests`, `request_components`) that adds every missing column as its default. Temp objects are searched +// before main, so every unqualified table name resolves to the shim; the view is per connection and // writes nothing to the file. A file with all its columns gets no shim. query_only is switched on // last, because it forbids creating even a temp object. // @@ -349,30 +349,37 @@ func init() { return nil } ctx := context.Background() - rows, err := conn.QueryContext(ctx, `SELECT name FROM pragma_table_info('requests')`, nil) - if err != nil { - return err - } - have := map[string]bool{} - vals := make([]driver.Value, 1) - for rows.Next(vals) == nil { - if n, ok := vals[0].(string); ok { - have[n] = true - } - } - rows.Close() - var extra string + // One shim per table that has additive columns (requests, request_components, ...). + tables := map[string]bool{} for _, c := range additiveColumns { - if c.table == "requests" && !have[c.column] { - extra += ", " + additiveDefault(c.ddl) + " AS " + c.column - } + tables[c.table] = true } - if extra != "" && len(have) > 0 { - if _, err := conn.ExecContext(ctx, `CREATE TEMP VIEW requests AS SELECT *`+extra+` FROM main.requests`, nil); err != nil { + for table := range tables { + rows, err := conn.QueryContext(ctx, `SELECT name FROM pragma_table_info('`+table+`')`, nil) + if err != nil { return err } + have := map[string]bool{} + vals := make([]driver.Value, 1) + for rows.Next(vals) == nil { + if n, ok := vals[0].(string); ok { + have[n] = true + } + } + rows.Close() + var extra string + for _, c := range additiveColumns { + if c.table == table && !have[c.column] { + extra += ", " + additiveDefault(c.ddl) + " AS " + c.column + } + } + if extra != "" && len(have) > 0 { + if _, err := conn.ExecContext(ctx, `CREATE TEMP VIEW `+table+` AS SELECT *`+extra+` FROM main.`+table, nil); err != nil { + return err + } + } } - _, err = conn.ExecContext(ctx, `PRAGMA query_only = 1`, nil) + _, err := conn.ExecContext(ctx, `PRAGMA query_only = 1`, nil) return err }) } @@ -586,19 +593,20 @@ func (d *DB) insertBatch(evs []*Event) error { waiting_for, wait_tool, wake_cause, open_background, open_teammate, monitor_expiry_ms, agent_compaction, cache_miss_cause, manual_ping, manual_saved_usd, project, source, sweep_expands, sweep_expand_tokens, sweep_false_expands, sweep_answer_usd, session_alias, + price_estimated, defer_tokens, defer_list_tokens, defer_searches, defer_loaded, defer_wrong_guesses, defer_gross_usd, defer_net_usd, defer_dropped_searches ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, - ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) + ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { return err } defer reqStmt.Close() compStmt, err := tx.Prepare(`INSERT INTO request_components( request_id, component, kind, acted, mutated, reverted, skipped, saved_gross, saved_unique, - saved_usd, duration_ms, err, gates, events - ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) + saved_usd, duration_ms, err, gates, events, saved_first_usd, saved_replay_usd, priced_by + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { return err } @@ -665,7 +673,7 @@ func (d *DB) insertBatch(evs []*Event) error { * safe direction — it claims LESS about the row. */ sourceOf(e.Source), e.SweepExpands, e.SweepExpandTokens, e.SweepFalseExpands, e.SweepAnswerUSD, - e.SessionAlias, + e.SessionAlias, boolInt(e.PriceEstimated), e.DeferTokens, e.DeferListTokens, e.DeferSearches, e.DeferLoaded, e.DeferWrongGuesses, e.DeferGrossUSD, e.DeferNetUSD, e.DeferDroppedSearches, ) @@ -687,7 +695,7 @@ func (d *DB) insertBatch(evs []*Event) error { if _, err := compStmt.Exec(id, c.Component, c.Kind, boolInt(c.Acted), boolInt(c.Mutated), boolInt(c.Reverted), boolInt(c.Skipped), c.SavedGross, c.SavedUnique, c.SavedUSD, c.DurationMs, c.Err, - gatesJSON(c.Gates), gatesJSON(c.Events)); err != nil { + gatesJSON(c.Gates), gatesJSON(c.Events), c.SavedFirstUSD, c.SavedReplayUSD, c.PricedBy); err != nil { return err } } diff --git a/dash/summarizeattr.go b/dash/summarizeattr.go index 4772e0e3..a5a26075 100644 --- a/dash/summarizeattr.go +++ b/dash/summarizeattr.go @@ -35,12 +35,18 @@ func pricerFn(ctx context.Context, p modelinfo.Pricer) rePriceFn { } } -// rePriceRow values a stored summarize row through Event.ComponentSavedUSD, the same path -// Event.Price used to store it, so a row written before the first-removal rule changed reads -// like a new one. Only a row with a unique (first) part changes; a pure replay keeps its stored +// rePriceRow values a stored summarize row from before the accountant library through +// Event.ComponentSavedUSD, the path the legacy rules stored it with, so a row written before the +// first-removal rule changed reads like a new one. A row the library priced is not re-priced. Only a row with a unique (first) part changes; a pure replay keeps its stored // figure, which keeps the rates it was written with. A row with no stored figure is left at zero: // it predates the column, and the estimate path owns it. func rePriceRow(r *compactRow, price rePriceFn) { + if r.Accountant { + // The library priced it from the thread's history, which the row does not keep: its + // figure and its first-removal part are stored and stand as written. + r.FirstUSD = r.storedFirstUSD + return + } if price == nil || r.StoredUSD == 0 || r.SavedUnique <= 0 || r.SavedGross <= 0 { return } @@ -60,7 +66,7 @@ func rePriceRow(r *compactRow, price rePriceFn) { // apportionFirst gives a row that was not re-priced (no rates available) the unique share of its // stored figure as its first removal, so the ledger can still tell a first removal from a replay. func apportionFirst(r *compactRow) { - if r.FirstUSD != 0 || r.SavedUnique <= 0 || r.SavedGross <= 0 { + if r.Accountant || r.FirstUSD != 0 || r.SavedUnique <= 0 || r.SavedGross <= 0 { return } share := float64(min(int(r.SavedUnique), int(r.SavedGross))) / float64(r.SavedGross) diff --git a/dash/summarizeattr_test.go b/dash/summarizeattr_test.go index b74533df..c2f505c1 100644 --- a/dash/summarizeattr_test.go +++ b/dash/summarizeattr_test.go @@ -9,41 +9,8 @@ import ( "github.com/rossoctl/context-guru/internal/tokens" ) -// The first removal of a summary is priced at the rate the previous turn paid for that content. -// A live cache means the removed tokens, left in, would have been READ. A lapsed one means they -// would have been re-created. Every other component keeps the write rate: its content is new. -func TestSummarizeFirstRemovalIsPricedAtTheRateThePreviousTurnPaid(t *testing.T) { - price := handPrice["m1"] - for _, tc := range []struct { - name, component, reason string - want float64 - }{ - {"hit reads", SummarizerComponents[0], CacheHit, price.CacheRead}, - {"our own rewrite of a live entry reads", SummarizerComponents[0], CachePrefixChange, price.CacheRead}, - {"a lapsed entry re-creates", SummarizerComponents[0], CacheTTLExpiry, price.CacheWrite}, - {"a cold start re-creates", SummarizerComponents[0], CacheColdStart, price.CacheWrite}, - {"another component's new content is a write", "extract", CacheHit, price.CacheWrite}, - } { - t.Run(tc.name, func(t *testing.T) { - e := &Event{ - Model: "m1", TS: 1, TokensBefore: 10_000, TokensAfter: 9_000, SavedUnique: 1000, - FreshInput: 10, CacheRead: 50_000, TokenAccounting: AccountingComplete, - CacheMissReason: tc.reason, - Components: []CompRow{{Component: tc.component, Acted: true, Mutated: true, - SavedGross: 1000, SavedUnique: 1000}}, - } - e.Price(price, true) - want := 1000 * tc.want * e.BilledTokenFactor - nearEq(t, "component saved_usd", e.Components[0].SavedUSD, want) - // The request-level baseline must agree with the component figure, or the headline and - // the Components tab part ways on the same turn. - nearEq(t, "baseline delta", e.BaselineCostUSD-e.CostUSD, want) - }) - } -} - -// rePriceRow is the read-time twin of Event.Price: a row stored under the old rule reads like a -// row written under the new one. +// rePriceRow is the read-time twin of the legacy write path (legacyPrice): a row stored under the +// old rule reads like a row written under the newer one. An accountant row is never re-priced. func TestRePriceRowMatchesWhatPriceWouldHaveStored(t *testing.T) { price := handPrice["m1"] e := &Event{ @@ -51,7 +18,7 @@ func TestRePriceRowMatchesWhatPriceWouldHaveStored(t *testing.T) { CacheMissReason: CacheHit, TokenAccounting: AccountingComplete, Components: []CompRow{{Component: SummarizerComponents[0], SavedGross: 589_000, SavedUnique: 589_000}}, } - e.Price(price, true) + legacyPrice(e, price) r := compactRow{ Model: "m1", MissReason: CacheHit, FreshInput: 10, CacheRead: 49_000, CacheWrite: 7_400, SavedGross: 589_000, SavedUnique: 589_000, @@ -70,6 +37,14 @@ func TestRePriceRowMatchesWhatPriceWouldHaveStored(t *testing.T) { if replay.SavedUSD != 0.5 { t.Errorf("a replay row was re-priced: %v", replay.SavedUSD) } + + // A row the accountant library priced keeps its figure and its stored first-removal part. + acc := compactRow{Model: "m1", MissReason: CacheHit, SavedGross: 1000, SavedUnique: 1000, SavedUSD: 0.5, + StoredUSD: 0.5, Accountant: true, storedFirstUSD: 0.2} + rePriceRow(&acc, func(string) (modelinfo.Price, bool) { return price, true }) + if acc.SavedUSD != 0.5 || acc.FirstUSD != 0.2 { + t.Errorf("an accountant row was re-priced: saved %v first %v", acc.SavedUSD, acc.FirstUSD) + } } type testWindows struct{} diff --git a/dash/summarizercomponents_test.go b/dash/summarizercomponents_test.go index d4488560..be7ce9f0 100644 --- a/dash/summarizercomponents_test.go +++ b/dash/summarizercomponents_test.go @@ -43,26 +43,6 @@ func TestSummarizerComponentListIsOneSafeList(t *testing.T) { // would shrink with it and still pass when a name is dropped from the list. var namedSummarizers = []string{"summarize", "cache_aware_summarizer"} -func TestEveryNamedSummarizerPricesItsFirstRemovalAsARead(t *testing.T) { - price := handPrice["m1"] - for _, name := range namedSummarizers { - for _, tc := range []struct { - reason string - want float64 - }{{CacheHit, price.CacheRead}, {CachePrefixChange, price.CacheRead}, {CacheTTLExpiry, price.CacheWrite}} { - e := &Event{ - Model: "m1", FreshInput: 10, CacheRead: 50_000, TokenAccounting: AccountingComplete, - CacheMissReason: tc.reason, SavedUnique: 1000, TokensBefore: 10_000, TokensAfter: 9_000, - Components: []CompRow{{Component: name, SavedGross: 1000, SavedUnique: 1000}}, - } - e.Price(price, true) - want := 1000 * tc.want * e.BilledTokenFactor - nearEq(t, name+"/"+tc.reason+" component", e.Components[0].SavedUSD, want) - nearEq(t, name+"/"+tc.reason+" baseline", e.BaselineCostUSD-e.CostUSD, want) - } - } -} - func asComponent(e *Event, name string) *Event { e.Components[0].Component = name return e diff --git a/dash/sweepattr_test.go b/dash/sweepattr_test.go index 3fcf6eeb..3a188307 100644 --- a/dash/sweepattr_test.go +++ b/dash/sweepattr_test.go @@ -28,14 +28,15 @@ func sweepEvent(ts int64, session string, miss string, cacheRead, cacheWrite, fr Components: []CompRow{{Component: SweepComponent, Kind: "offload", Acted: removed > 0, Mutated: true, SavedGross: removed, SavedUnique: unique}}, } - e.Price(sweepPrice, true) + legacyPrice(e, sweepPrice) // the rows #404 was measured on: the legacy rule's figures return e } -// ⭐ M1. The credit on the COLD request is removed tokens x the cache-write rate (1.25 x input): the -// removed tokens are new content on that request. On each LATER request it is removed tokens x the rate -// that request paid. Both come out of the one pricing path every component uses, so this asserts the -// path gives the numbers point 9 states. +// ⭐ M1. Under the legacy rule (the rows #404 was measured on, still valued by the read passes) the +// credit on the COLD request is removed tokens x the cache-write rate (1.25 x input): the removed +// tokens are new content on that request. On each LATER request it is removed tokens x the rate that +// request paid. New rows are priced by the accountant library, whose shared cases cover the same +// rule by position (rewrite/deep-on-cold/then-hot-replay/5m). func TestM1TheCreditIsTheWriteRateOnTheColdRequestAndTheRequestsRateAfter(t *testing.T) { const removed = 10_000 factor, _ := tokens.BilledDeltaFactor("claude-sonnet-5") @@ -202,7 +203,7 @@ func TestM2ReplayOfTheTwoLosingSweepsShowsNoLossAndNoFalseCredit(t *testing.T) { CacheMissReason: CachePrefixChange, Components: []CompRow{{Component: SweepComponent, Kind: "offload", Acted: true, Mutated: true, SavedGross: s.removedLocal, SavedUnique: s.removedLocal}}} - old.Price(p, true) + legacyPrice(old, p) if got := old.Components[0].SavedUSD; math.Abs(got-s.issueCredit) > 0.003 { t.Fatalf("the pricing path gives a credit of %.4f; the issue's was %.3f. The fixture's price is not the issue's", got, s.issueCredit) } @@ -221,7 +222,7 @@ func TestM2ReplayOfTheTwoLosingSweepsShowsNoLossAndNoFalseCredit(t *testing.T) { FreshInput: 50, CacheWrite: s.after, OutputTokens: 400, CacheMissReason: CacheTTLExpiry, Components: []CompRow{{Component: SweepComponent, Kind: "offload", Acted: true, Mutated: true, SavedGross: s.removedLocal, SavedUnique: s.removedLocal}}} - coldWith.Price(p, true) + legacyPrice(coldWith, p) coldWithout := &Event{TS: 2, TenantID: "t", SessionID: s.name, Model: "claude-sonnet-5", Status: 200, FreshInput: 50, CacheWrite: s.before, OutputTokens: 400, CacheMissReason: CacheTTLExpiry} coldWithout.Price(p, true) diff --git a/dash/testdata/accountant/README.md b/dash/testdata/accountant/README.md new file mode 100644 index 00000000..b545a5d9 --- /dev/null +++ b/dash/testdata/accountant/README.md @@ -0,0 +1,25 @@ +# Test data for the accountant comparison + +`known-differences.json` lists the cases where cg's saving logic (`dash.Event.Price`) is **meant** to +give a different figure from the shared test cases of +[`github.com/rossoctl/accountant`](https://github.com/rossoctl/accountant). + +`dash/accountant_play_test.go` plays every accountant case through cg and compares the result: + +- A case that is **not** in this file must agree with cg to within 1 µ$. +- A case that **is** in this file must really differ. If cg starts to agree, the test fails, so remove + the entry. +- An entry whose case no longer exists in accountant also fails the test. + +One entry: + +| Key | Meaning | +|---|---| +| `case` | The id of an accountant case. | +| `kind` | `differs`: same question, different answer on purpose. `notCovered`: cg prices something the model does not, or the reverse. | +| `reason` | Why they differ. Cites a spec section and the cg function. | +| `allow` | The exact places (`step`, and `change` when one change differs) where cg may differ. A difference anywhere else fails. | +| `total` | Whether the thread total may differ. It follows from the steps. | + +The file stays in context-guru because it compares the cases with cg code. The figures and the root +causes are in [`docs/accountant/`](../../../docs/accountant/cg-differences.md). diff --git a/dash/ui/app.js b/dash/ui/app.js index 0da66af7..5190411e 100644 --- a/dash/ui/app.js +++ b/dash/ui/app.js @@ -3696,8 +3696,12 @@ async function openRequest(id, fromURL) { // Directly beneath the cost, because the cost on its own is the figure that makes // readers conclude the product is worthless: baseline − actual − our own spend is // what this one turn was actually worth. - kv(term('Net saving', 'Baseline cost minus actual cost minus context-guru model cost.'), - priced ? usd(e.baseline_cost_usd - e.cost_usd - e.cg_llm_cost_usd) : '—'), + // price_estimated: the accountant could not rule out a cache break (no history for this + // thread yet), so the gross saving is right and the net may be too high. Marked "≈", the + // same way this page marks every other estimate. + kv(term('Net saving', 'Baseline cost minus actual cost minus context-guru model cost.' + + (e.price_estimated ? ' An estimate: there was no history to tell whether a change broke the cache, so the net may be too high.' : '')), + priced ? (e.price_estimated ? '≈ ' : '') + usd(e.baseline_cost_usd - e.cost_usd - e.cg_llm_cost_usd) : '—'), // On a turn whose cache HIT this is usually the largest money figure on the row, // and it was not reported anywhere: the cache reads this request was billed for, // against the fresh rate they would have cost without the cache. diff --git a/docs/accountant/cg-differences-root-causes.md b/docs/accountant/cg-differences-root-causes.md new file mode 100644 index 00000000..d9d404e1 --- /dev/null +++ b/docs/accountant/cg-differences-root-causes.md @@ -0,0 +1,194 @@ +# Root causes of the differences between cg and the spec + +**Status:** companion to [cg-differences.md](cg-differences.md), regenerated from the final commit of +branch `docs/context-operations-spec`. The cases, the spec and the design are in +`github.com/rossoctl/accountant` (v0.1.0). +**Reproduce:** `go test ./dash/ -run TestCasesAgreeWithCurrentLogic -v` from the repo root. + +**Update, accountant `c628f22` (2026-11-02).** The tables and figures below come from the 63-case run on +accountant v0.1.0 and are not recalculated. Accountant has since added R54 and three cases, so the +harness now plays **66 cases, and 30 differ** (20 `differs`, 10 `notCovered`). The three new cases +are in section "Cause 9" at the end. The branch named above was closed without merging. + +This file explains each of the root causes in cg's logic (`dash.Event.Price`). The case list, +the per-case figures and the limits of the method are in `cg-differences.md`. All figures here come +from the same run (63 cases, 27 differ). + +**Total gap.** The sum of the thread-total gaps over the 27 differing cases is 1,273,201 µ$. One of them, +`rewrite/four-operations/hot/5m`, differs only in one line and has no total gap. + +| # | Root cause | Cases | Gap (µ$) | Kind | +|---|---|---:|---:|---| +| 1 | No cache-break charge | 9 | 644,501 | cg defect | +| 2 | The rate follows novelty, not position | 3 | 29,475 | cg defect | +| 3 | Growth is clamped; an ended change is still credited | 3 | 38,400 | cg defect | +| 4 | No concept of a setting change | 4 | 378,350 | Different question | +| 5 | Keep-alive credit floor | 0 | 0 | Resolved: both cases agree | +| 6 | Credits booked outside the per-component figure | 4 | 27,450 | Adapter, not evidence | +| 7 | No check for an unreported edit | 1 | 1,500 | Missing check | +| 8 | The unique-versus-replay split has no session or thread | 3 | 153,525 | cg defect or open question | + +## Cause 1: no cache-break charge + +**What cg does.** `baselineDeltaUSD` builds the "bill without the change". It takes the bill the +provider sent and adds the price of the removed tokens. It does not re-bill the client body. + +**Why this is wrong.** An edit inside the cached prefix changes the prompt above the cached tokens. +The provider cannot read them any more, so it writes them again. That extra write is already inside +the billed amount. So the no-change bill is too high, and the edit gets credit instead of blame. + +**Example, `summary/break-then-payback`, request A.** + +1. The true no-change bill is 8,175 µ$. +2. The bill with the summary is 17,400 µ$. +3. cg computes 17,400 + 2,850 = 20,250 µ$ as the no-change bill. +4. The gap is 12,075 µ$. This is 3,500 tokens × ($3.75 − $0.30). + +**Example, `coref`.** The spec books +5,985 µ$ for the saving and −65,584.5 µ$ for the break. The net +is −59,599.5 µ$. cg books 19,950 × $3.75 = +74,813 µ$. The sign is wrong. + +**cg already knows the cost.** `components/offload/prefix_econ.go` uses an 11.5 factor to decide +whether to break the prefix. The reported saving does not use it. + +**A fix has two parts.** Each part is needed. + +- Charge moved tokens × (write − read) to the edit that caused the move. +- Price the removed tokens by position (see cause 2). + +Fixing only one still leaves a wrong figure. + +**Unverified.** If the live pipeline never picks `coref` on such a prompt, six of the nine cases are +theoretical. This was not tested. + +## Cause 2: the rate follows novelty, not position + +**What cg does.** `uniqueRate` returns the write rate for the first removal. The only exception is a +summarizer on a live cache, which gets the read rate. + +**What the spec does.** It prices a token by where it sits: + +| Position | Tier | +|---|---| +| Before B\* | read | +| B\* to the last breakpoint | write | +| After the last breakpoint | input | +| No breakpoint in the body | input | + +**Cases.** + +- `after-last-breakpoint` and `no-breakpoint`: 7,000 tokens × ($3.75 − $3.00) = 5,250 µ$. +- `state-lost-reads-cache`: the usage shows 18,500 tokens read. So the 5,500 replaced tokens are + reads. cg prices them as writes. The gap is 5,500 × $3.45 = 18,975 µ$. + +**Note.** The summarizer exception is the same fix, applied to one component by name. The general fix +is to give `uniqueRate` the position or the usage evidence. + +**Open point.** The spec's rule for B\* is not settled. A literal reading puts B\* at the end of +`system` in the `coref` case. Then cg's write-rate answer would be right. The earlier model review +read B\* as "the end of the longest prefix the request could have hit". The spec text should say +this. + +## Cause 3: growth is clamped, and an ended change is still credited + +**What cg does.** `ComponentSavedUSD` sets a negative gross to zero. A change that adds tokens, such +as `expand` or a restore, has no cost line. The request total is still right, because +`Event.Saved()` nets the whole request. + +**Two separate failures.** + +1. **Addition.** The spec books `expand` as −90 µ$ per hot turn. cg books 0. In `fixed-restore-cost`, + the restore costs nothing and the `mask` is credited in full every turn. The gap is 17,700 µ$. +2. **Ended change.** In `restore-in-place` the original goes back and the mask ends. The provider + holds the marker, so 6,000 tokens move from read to write. cg has no "end" event. It keeps + crediting +1,350 µ$ and books no break. The gap is 20,700 µ$. + +**Unverified.** It is not known whether the live pipeline reports `expand` as a component. If it does +not, cg has no row, which is different from a row of zero. The adapter passes `max(delta, 0)`, so the +test cannot tell the two apart. + +## Cause 4: no concept of a setting change + +**What cg does.** `AttributeCache` sends a model it has not seen to `cold_start`. Its comment says a +first use of a model must never be reported as a bust. cg does not read `effort`. It prices only its +own edits. + +**Why it is not a defect.** A router or a client changed the setting. cg did not. cg's answer ("I +saved nothing") is correct for the question it asks. The spec asks a different question: "what did +the cache cost the user?" + +**Decision needed.** If the product must show users that a setting change cost them a cache, cg needs +new logic. If not, mark these cases out of scope. + +**Open and unverified.** + +- Which rates price a model-change break. The case uses the small model (−36,800 µ$). At the large + model's rates it is −138,000 µ$. +- That a real model renders `effort` ahead of the messages. This comes from `cached-prefix.md`, not + from a test. + +## Cause 5: keep-alive credit floor (resolved) + +**What cg does.** `keepaliveSavedUSD` credits a ping only if the gap is above +`keepaliveCreditFloorMs`. + +**What changed.** The earlier report listed two cases here (1-hour entry, and minimum lifetime). The +adapter did not pass `CacheTTL` or `Provider`, so the test took the 5-minute path and did not run the +branches. `Play` now sets both. Both cases agree with cg now, and neither is in `known-differences.json`. +The suspected inconsistency between `providerCacheTTLMs` and `keepaliveCreditFloorMs` is **not +confirmed** by them. It is still unmeasured on real traffic. + +## Cause 6: credits outside the per-component figure + +cg books the cachesplit credit and the head-TTL credit in their own columns. `Play` does not fill +them, so cg shows 0. These four cases do not test cg's logic. They show a gap in the adapter. + +**Next step.** Feed the two credits in `Play`, then re-check. + +## Cause 7: no check for an unreported edit + +The spec refuses to price a body edit that no operation reports. cg has no such check. It books +1,500 µ$ for the size gap between the client body and the forwarded body. The mechanism was not +traced. The money is small. The risk is that an unexplained size change gets counted as a saving. + +## Cause 8: the unique-versus-replay split has no session or thread + +**What cg does.** `Recorder.MarkUnique` (`dash/capture.go`) decides whether a removal is new. Its key +is tenant, component and content key. It has no session and no thread. + +**What the spec does.** Each thread has its own state, so a change reported on another thread, or in +another session, is new there. Three cases show it: + +- `threads/change-stays-on-its-thread` and `threads/two-sessions-same-change/main-and-subagent`: the + spec books 7,000 tokens as a write (26,250 µ$) per thread. cg books the repeat as a read (2,100 + µ$). Which is right depends on whether the thread has its own cache entry. +- `threads/fork-first-request/summary`: the spec keeps no history for a fork, so it prices the first + request as a write. A fork reads its parent's entry by content, so here cg's read is the truer tier + and the spec is the one that is off. + +**Open question.** Should the library read the parent's state for a fork (`Thread.Parent`)? + +## Priority + +1. Fix cause 1 together with cause 2. They hold 673,976 µ$ of the 1,273,201 µ$ total gap and share + one design flaw: the additive baseline. +2. Fix the adapter for cause 6 before judging it. Cause 5 is resolved. +3. Decide cause 4 as a product question, not a bug. +4. Decide cause 8 together with the fork rule in the accountant library. + +## Cause 9: no sibling-thread input to the keep-alive rescue + +Added after the run above. Accountant's R54 says a sibling thread of the session that read part of what +a ping read, inside the TTL, kept that part warm without the ping, so the rescue leaves it out. The +host reports it in `EntryFacts.SharedWarm`. `dash.Event.keepAliveSavedUSD` caps the credit by what the +last ping read (`KeepAliveRefreshed`) and has no sibling-thread input, so it credits the rescue in full. + +| Case | Accountant | cg | Gap (µ$) | +|---|---|---|---:| +| `keepalive/shared-head-sibling/exact/5m` | credits 8,000 tokens | credits 10,000 | 6,900 | +| `keepalive/shared-past-ping-read/exact/5m` | credits nothing | credits in full | see the test output | +| `keepalive/shared-prefix-all-sibling/exact/5m` | no rescue, the ping stays a cost | credits in full | 34,500 | + +They are listed in `dash/testdata/accountant/known-differences.json` as `notCovered`: cg prices +something the model does not. Whether cg should learn about sibling threads is open. The proxy path does +not report `SharedWarm` either, so `TestProxyPathMatchesTheAccountantCases` passes on these cases only +because it plays the case's own facts. diff --git a/docs/accountant/cg-differences.md b/docs/accountant/cg-differences.md new file mode 100644 index 00000000..defdb637 --- /dev/null +++ b/docs/accountant/cg-differences.md @@ -0,0 +1,343 @@ +# Where cg's saving differs from the expected figure + +**Status:** report, regenerated from the final commit of branch `docs/context-operations-spec`, against `github.com/rossoctl/accountant` v0.1.0. +**Reproduce:** `go test ./dash/ -run TestCasesAgreeWithCurrentLogic -v` from the repo root. The test +imports `github.com/rossoctl/accountant` (v0.1.0). +The cases, the spec and the design are in the `rossoctl/accountant` repository. + +## 1. What this report compares + +context-guru (**cg**) edits LLM requests. It shortens tool outputs, removes unused tool +declarations and replaces old messages with a summary. Each edit makes the request cheaper, and cg +reports how much money that saved. The figure comes from `dash.Event.Price` in `dash/event.go`. + +The `accountant` library (the "spec") computes the same saving by one shared rule. The rule is +the **counterfactual**: *if this change had never been made, how would the provider have billed these +tokens on this request?* Where the spec and cg disagree, one of them is wrong, or they answer +different questions. This report groups the disagreements by the cause in cg's logic. + +**How the two sides are produced** + +- **Expected figure.** Each case in `test/cases/` of the accountant repository is a sequence of requests. The + expected tokens were written by hand from the spec's rules. Each case passes two checks: billed + bill minus no-change bill equals tokens saved minus tokens caused, and each dollar total matches + the document it came from. This run did **not** call `accountant.Compute`. So "expected" means + "what the spec's rules give", not a measured result. +- **cg's figure.** The test calls the real `dash.Event.Price` for every request of every case. The + only code written on that side is the adapter `Play` in `dash/accountant_play_test.go`. It + turns a case into a `dash.Event`. It treats the first request of a change as a "unique" removal + and later requests as "replays". + +**Result.** There are **63 cases**. **36 agree** with cg to within 1 µ$ (rounding). **27 differ**. +The test passes because each known difference is listed in +`dash/testdata/accountant/known-differences.json`: 20 as `differs` and 7 as `notCovered`. A listed case that +starts to agree fails the test, so the list cannot go stale. The first report had 9 cases and 5 +differences. The set grew because the case library grew, not because cg changed. + +## 2. Terms + +| Term | Meaning | +|---|---| +| Tier | The rate a token is billed at: **input** (fresh), **cache write** (1.25 times input on Anthropic), **cache read** (0.1 times input). | +| Hot / cold | Hot: the provider still holds the cached prefix, so it reads it. Cold: the entry expired, so the prefix is written again. | +| Prefix, breakpoint | The provider caches the start of the prompt (tools, then system, then messages) up to a `cache_control` breakpoint. | +| B\* | The end of the prefix the provider would have read on this request if no edit had happened. | +| Saved / caused | Tokens the provider did not bill because of an edit, and tokens it billed because of an edit. | +| Break | An edit changes the prompt inside the cached prefix. The provider cannot read from the first changed position onward and writes it again. It is a **cost** of the edit. | +| µ$ | Micro-dollar, $0.000001. | + +**Rates in most cases** (per million tokens): input $3.00, cache write $3.75, cache read $0.30, +output $15.00. Some cases use a second model with its own rates, given in the case file. These are +test rates, not production rates. + +**The two pricing rules.** The spec prices a removed token at the tier it would have had at its +*position*: before B\* a read, after B\* and up to the last breakpoint a write, after the last +breakpoint input. cg prices it by *novelty*. The first time content is removed it is a **write** +(`uniqueRate`, `dash/event.go:894`). On later requests it is the tier this request paid +(`repeatRate`, `dash/event.go:1163`). cg builds the no-change bill by adding the priced removed +tokens to the billed cost (`baselineDeltaUSD`, `dash/event.go:1102`). It never re-bills the client +body. Causes 1 to 3 all follow from this design. + +## 3. Summary of root causes + +| # | Root cause in cg | Cases | Spec total − cg total (µ$, sum of case gaps) | Kind | +|---|---|---:|---:|---| +| 1 | No cache-break charge: an edit inside the cached prefix is never billed to the edit | 9 | 644,501 | cg defect | +| 2 | The rate follows novelty, not position or usage | 3 | 29,475 | cg defect | +| 3 | Growth is clamped to zero and an ended change is still credited | 3 | 38,400 | cg defect | +| 4 | cg has no concept of a setting change | 4 | 378,350 | different question | +| 5 | The keep-alive credit floor (resolved: both cases now agree, see section 8) | 0 | 0 | none | +| 6 | Credits that cg books outside the per-component figure are not fed by the adapter | 4 | 27,450 | adapter, not evidence | +| 7 | No check for an edit that no operation reports | 1 | 1,500 | missing check | +| 8 | The unique-versus-replay split has no session or thread (and the library's fork rule is open) | 3 | 153,525 | cg defect or open question | + +Gaps are the absolute difference of each case's thread total (section 10 lists every case). Causes +1 and 2 overlap in six cases. Each case is counted once, under its main cause. + +--- + +## 4. Cause 1: no cache-break charge + +**The logic.** `Event.Price` has no term for tokens that moved from read to write because an edit +changed the prompt above them. The extra write cost is already inside the billed figure. cg counts +it in the bill and never charges it to the edit. So the edit looks cheaper than it is, and often +looks like a gain when it is a loss. + +cg knows this cost elsewhere. `components/offload/prefix_econ.go:14-20` states the rule +(`S × T > 11.5 × W`) that decides whether to break the prefix. The decision uses the 11.5 factor. +The saving reported afterward does not. + +| Case | Spec µ$ | cg µ$ | Gap µ$ | Note | +|---|---:|---:|---:|---| +| `rewrite+spend/summary/break-then-payback/5m` | 16,750 | 28,825 | 12,075 | Break only. 3,500 tokens move read to write (500 summary + 3,000 tail). cg already prices a summarizer's removal as a read, so the gap is exactly the break: 3,500 × ($3.75 − $0.30). | +| `rewrite/coref-break-inside-prefix/hot/5m` | 33,524 | 167,935 | 134,411 | Cause 2 adds to it. `coref` cuts a 20,000-token output inside the cached prefix. The spec books a loss, cg books +74,813 µ$ for that edit on turn 7. | +| `rewrite/break-then-payback-turns/hot/5m` | 76,920 | 214,920 | 138,000 | Same sign error as `coref`. Later hot turns and the cold turn agree. | +| `rewrite/searchfold-inside-prefix/hot/5m` | −49,275 | 9,375 | 58,650 | The 14,500-token break is not charged. Issue 496: the dashboard showed a hit. | +| `rewrite/replay-at-new-position/hot/5m` | 10,050 | 44,550 | 34,500 | A second occurrence is compacted at depth. | +| `rewrite/two-spans-two-tiers/hot/5m` | −43,275 | 32,625 | 75,900 | One read span and one write span. cg uses one rate and no break. | +| `rewrite/toolschema-mid-session/break-then-saving/5m` | −81,600 | 8,100 | 89,700 | Trimming a cached tool declaration breaks the whole prefix once. cg books +7,500 µ$ on that turn where the spec books −82,200 µ$. | +| `rewrite/decision-flips-inside-prefix/hot/5m` | −26,805 | 7,695 | 34,500 | Request 2 breaks the cache. Request 4 returns the original and the spec charges a 6,000-token break to the ended change (design open question 12). | +| `rewrite/expand-first-injection/break-once/5m` | −66,765 | 0 | 66,765 | An injected tool declaration changes `tools` and breaks the whole prefix once (−66,675 µ$), then costs −90 µ$ per turn. cg books neither. Also cause 3. | + +**Worked example, `summary/break-then-payback`.** Request A: the provider reads 8,000 tokens (tools ++ system) and writes 4,000. With no summary it would read 21,000 and write 500. The no-change bill +is 8,175 µ$. The bill with the summary is 17,400 µ$. cg computes the no-change bill as the billed +17,400 µ$ plus the 9,500 removed tokens at the read rate (2,850 µ$) = 20,250 µ$. The true +no-change bill is 8,175 µ$. The 12,075 µ$ gap is the break. + +**What was not verified.** `chargeRowCosts` and `causedWriteUSD` (`dash/compactepisode.go`, +`:727`, `:801`) charge the whole cache write of a compaction turn to the summary, as a stated upper +bound. This run did not call that code. Reading it, the charge would be 4,000 × $3.75 = 15,000 µ$ +for request A, against the spec's 12,075 µ$. So the episode view may be close but not equal. It is +unverified which view a user sees. It is also unverified whether the live pipeline would choose +`coref` on a prompt like this one. If it would not, the `coref` cases are theoretical. + +--- + +## 5. Cause 2: the rate follows novelty, not position or usage + +**The logic.** `uniqueRate` returns `CacheWrite` for any first removal that is not a summarizer. +It does not look at where the content sits (inside the cached prefix, after B\*, after the last +breakpoint, or with no breakpoint) or at what the usage block shows. + +| Case | Spec µ$ | cg µ$ | Gap µ$ | What the spec does | +|---|---:|---:|---:|---| +| `boundary/state-lost-reads-cache` | 1,650 | 20,625 | 18,975 | With no history, the usage still shows 18,500 tokens read. A read in the usage is evidence. The 5,500 replaced tokens are a read ($0.30), not a write ($3.75). | +| `rewrite/after-last-breakpoint/input/5m` | 21,000 | 26,250 | 5,250 | Tokens after the last client breakpoint are billed as input ($3.00), not write. | +| `rewrite/no-breakpoint/input/5m` | 21,000 | 26,250 | 5,250 | With no breakpoint in the body every token is input. | + +The six position cases in section 4 also carry this error. It is not separate money there. Fixing +the break without fixing the rate would still leave `coref` pricing 19,950 tokens at the write +rate. Fixing the rate alone would still leave the break uncharged. + +**Open point in the spec.** The rule for B\* (design §5.1) snaps to the client's own breakpoints. A +literal reading puts B\* at the end of `system` for the `coref` case, and then the removed tokens +are writes, which is cg's answer with no break. The earlier four-model review (section 9) concluded +that B\* is the end of the longest prefix the request's own breakpoints could have hit, which is +message 11. The spec text should say this. Until it does, the `coref` expected figure depends on an +undecided rule. + +--- + +## 6. Cause 3: growth is clamped to zero and an ended change is still credited + +**The logic.** `ComponentSavedUSD` clamps a negative gross to zero (`dash/event.go:849-852`). A +change that *adds* tokens has no cost in cg. `Event.Saved()` (`dash/event.go:557`) works on the +whole request, so the request total can still net correctly. The per-change line does not. Cg also +has no term for a change that ends: it keeps crediting the change that is no longer applied. + +| Case | Spec µ$ | cg µ$ | Gap µ$ | Note | +|---|---:|---:|---:|---| +| `rewrite/four-operations/hot/5m` | 81,879 | 81,879 | 0 | Totals agree. Only the `expand` line differs: the spec books −90 µ$ per hot turn, cg books 0. | +| `rewrite/fixed-restore-cost/hot/5m` | 525 | 18,225 | 17,700 | A restore grows the body. The spec books the growth as a cost. cg books 0 for it and credits the `mask` in full every turn (issue 481). | +| `rewrite/restore-in-place/hot/5m` | −2,475 | 18,225 | 20,700 | The original goes back and the mask ends. The provider holds the marker, so 6,000 tokens move read to write. cg keeps crediting the mask (+1,350 µ$) and books no break (issue 481). | + +**Unverified.** Whether the live pipeline reports `expand` as a component at all. If it does not, +cg has no row for it, and the difference is "no row", not "a row of zero". The adapter passes +`max(delta, 0)` the same way the clamp does, so the test cannot tell the two apart. + +--- + +## 7. Cause 4: cg has no concept of a setting change + +**The logic.** cg measures what *its own edits* saved. A router that changes the model, or a client +that changes `effort` or toggles web search, is not a cg edit. `AttributeCache` +(`dash/event.go:1181`) sends a model it has not seen to `cold_start` and never reports it as a bust. +cg does not read `effort`. `baselineDeltaUSD` adds only removed tokens. So cg books nothing. + +| Case | Spec µ$ | cg µ$ | Gap µ$ | +|---|---:|---:|---:| +| `setting/model-change/break` | −36,800 | 0 | 36,800 | +| `setting/effort-change/break-from-messages` | −110,400 | 0 | 110,400 | +| `setting/web-search-toggle/break-from-system` | −120,750 | 0 | 120,750 | +| `setting/nil-position/penalty-unknown` | −110,400 | 0 | 110,400 | + +Four related cases agree because they have no break to price: `setting/effort-change/read-intact-no-break` +and others. + +**This is not a defect in cg.** It is a different question. The difference matters only if the +product must show a user that a setting change cost them a cache. All four are marked `notCovered`. + +**Open and unverified.** + +- Which rates price a model-change break. The case uses the small model's rates (−36,800 µ$). At + the large model's rates it is −138,000 µ$. Against "stayed on the large model" the net is + −25,250 µ$, which mixes the break with the rate difference. This is spec open question 10. +- That a real model renders `effort` ahead of the messages only. This comes from + `cached-prefix.md`, not from a test. Public documentation establishes only that a change to + thinking parameters breaks the cache. +- `setting/nil-position/penalty-unknown` assumes a setting with no position falls back to the read + the usage shows, plus a `penalty_unknown` entry. cg has no unpriced list. + +--- + +## 8. Causes 5 to 8: smaller or weaker differences + +### Cause 5: keep-alive credit floor (resolved) + +The earlier report listed two keep-alive cases as differences: +`keepalive/ping-on-1h-entry/no-rescue/1h` and `keepalive/ping/minimum-lifetime/no-rescue`. The +adapter did not pass `CacheTTL` or `Provider`, so cg took its 5-minute default and the test did not +reach the branches that matter. + +`Play` now sets both (`dash/accountant_play_test.go`: an OpenAI model keeps its name so that cg picks +its own provider's floor, and a 1h case passes `CacheTTL`). The 1h case now agrees with cg (−3,045 +µ$ at step 1). The minimum-lifetime case is now `keepalive/ping/minimum-lifetime/rescue`, and it +agrees too. Neither is in `known-differences.json`. + +So the suspected inconsistency between `providerCacheTTLMs` and `keepaliveCreditFloorMs` is **not +confirmed** by these two cases. It stays unmeasured on real traffic. + +### Cause 6: credits booked outside the per-component figure (4 cases, adapter) + +cg books `CachesplitSavedUSD` and the head-TTL credit in their own columns. `Play` does not feed +them, so cg shows 0. + +| Case | Spec µ$ | cg µ$ | +|---|---:|---:| +| `split/delivered/stable-half-read` | 10,350 | 0 | +| `ttl/head-1h/never-read` | −6,750 | 0 | +| `ttl/head-1h/read-after-3-minutes` | −6,750 | 0 | +| `ttl/head-1h/read-after-20-minutes` | 3,600 | 0 | + +These cases are **not evidence about** cg's cachesplit or head-TTL logic. They show what the +adapter does not cover. + +### Cause 7: no check for an unreported edit (1 case) + +`unpriced/unknown-kind-and-unreported-edit`: the spec (§6.5 checks 1 and 6) refuses to price a body +edit that no operation reports, and an unknown kind is left unpriced. cg has no such check and +books 1,500 µ$ for the size gap between the client and the forwarded body. The mechanism was not +traced. + +--- + +### Cause 8: the unique-versus-replay split has no session or thread (3 cases) + +cg decides whether a removal is new with `Recorder.MarkUnique` (`dash/capture.go`). Its key is +tenant, component and content key. It has no session and no thread. The library gives each thread its +own state, so it prices a change reported on a new thread as new there. + +| Case | Spec µ$ | cg µ$ | Gap µ$ | +|---|---:|---:|---:| +| `threads/change-stays-on-its-thread` | 82,950 | 34,650 | 48,300 | +| `threads/fork-first-request/summary` | 23,125 | −9,650 | 32,775 | +| `threads/two-sessions-same-change/main-and-subagent` | 105,000 | 32,550 | 72,450 | + +- **Same change on another thread or session.** The library books 7,000 tokens as a write (26,250 + µ$) on each thread. cg books the repeat as a read (2,100 µ$), so it under-reports the saving by + 24,150 µ$ on each. Which one is right depends on whether the thread has its own cache entry (the + library's view) or reads main's (cg's). +- **A forked thread's first request.** The library keeps no history for a fork and does not read + `Thread.Parent`, so the first request is `unknown`. It prices the summary's 9,500 removed tokens as + a write. The fork inherits main's entry by content, so cg's read is the truer tier here, and the + library is the one that is off. **Open:** should the library read the parent's state for a fork? + +## 9. Cases that agree with cg + +36 of 63 cases agree. They are **not** evidence of a problem. For a plain removal of new content, +a removal before B\* read when hot and written when cold, or a keep-alive ping and its rescue, the +two rules give the same answer. + +Two of them differ by 1 µ$ and count as agreeing (rounding): +`keepalive/ping-timeout-unknown-result/5m` and +`rewrite+spend/extract-llm-sweep/fallback-ask-fresh-input/5m`. The two keep-alive cases of the +earlier report now agree outright (cause 5). + +Examples: single rewrite, hot (26,250 µ$); rewrite replayed hot, hot, cold (54,600 µ$); tool +removal with three spans (62,400 µ$); keep-alive ping and the request it rescues (31,455 µ$); +four operations at the request level (81,879 µ$); three refused-spend cases; five split cases that +have no effect; session isolation. + +## 10. Every case, thread total in µ$ + +Cases that agree are omitted. See section 9. + +| Case | Cause | Spec | cg | Gap | +|---|---:|---:|---:|---:| +| boundary/state-lost-reads-cache | 2 | 1,650 | 20,625 | 18,975 | +| rewrite/after-last-breakpoint/input/5m | 2 | 21,000 | 26,250 | 5,250 | +| rewrite/no-breakpoint/input/5m | 2 | 21,000 | 26,250 | 5,250 | +| rewrite+spend/summary/break-then-payback/5m | 1 | 16,750 | 28,825 | 12,075 | +| rewrite/break-then-payback-turns/hot/5m | 1 | 76,920 | 214,920 | 138,000 | +| rewrite/coref-break-inside-prefix/hot/5m | 1 | 33,524 | 167,935 | 134,411 | +| rewrite/decision-flips-inside-prefix/hot/5m | 1 | −26,805 | 7,695 | 34,500 | +| rewrite/expand-first-injection/break-once/5m | 1 | −66,765 | 0 | 66,765 | +| rewrite/replay-at-new-position/hot/5m | 1 | 10,050 | 44,550 | 34,500 | +| rewrite/searchfold-inside-prefix/hot/5m | 1 | −49,275 | 9,375 | 58,650 | +| rewrite/toolschema-mid-session/break-then-saving/5m | 1 | −81,600 | 8,100 | 89,700 | +| rewrite/two-spans-two-tiers/hot/5m | 1 | −43,275 | 32,625 | 75,900 | +| rewrite/four-operations/hot/5m | 3 | 81,879 | 81,879 | 0 (line only) | +| rewrite/fixed-restore-cost/hot/5m | 3 | 525 | 18,225 | 17,700 | +| rewrite/restore-in-place/hot/5m | 3 | −2,475 | 18,225 | 20,700 | +| setting/model-change/break | 4 | −36,800 | 0 | 36,800 | +| setting/effort-change/break-from-messages | 4 | −110,400 | 0 | 110,400 | +| setting/web-search-toggle/break-from-system | 4 | −120,750 | 0 | 120,750 | +| setting/nil-position/penalty-unknown | 4 | −110,400 | 0 | 110,400 | +| split/delivered/stable-half-read | 6 | 10,350 | 0 | −10,350 | +| ttl/head-1h/never-read | 6 | −6,750 | 0 | 6,750 | +| ttl/head-1h/read-after-3-minutes | 6 | −6,750 | 0 | 6,750 | +| ttl/head-1h/read-after-20-minutes | 6 | 3,600 | 0 | −3,600 | +| threads/change-stays-on-its-thread | 8 | 82,950 | 34,650 | 48,300 | +| threads/fork-first-request/summary | 8 | 23,125 | −9,650 | 32,775 | +| threads/two-sessions-same-change/main-and-subagent | 8 | 105,000 | 32,550 | 72,450 | +| unpriced/unknown-kind-and-unreported-edit | 7 | 0 | 1,500 | 1,500 | + +In 34 request steps the *baseline bill* also disagrees with the spec's. In 25 of them cg's no-change +bill is higher, for example 33,675 against 14,700 µ$ (`boundary/state-lost-reads-cache` step 0). In 9 +it is lower (`split` and `ttl` cases, cause 6, and the thread cases, cause 8). This is the additive baseline of section 2 made +visible. + +## 11. Limits of this report + +- **Expected figures are by hand.** This run did not execute `accountant.Compute`. A wrong + case file or a wrong rule is not caught by the conservation check, which catches arithmetic + errors only. +- **The tests use a flat 4 bytes per token** and a fixture model with no tokenizer correction. The + real cg factor (1.69 for Sonnet and Opus, `internal/tokens/billedfactor.go`) is not an input, and + no case tests another ratio. +- **The adapter `Play` is not the live pipeline.** It supplies the unique-versus-replay split that + `FromTrace` computes in production. A different split changes cg's figure for causes 1 and 2. It + also omits `Provider`, `CacheTTL`, the cachesplit credit and the head-TTL credit (causes 5 and 6). +- **B\* is not settled in the spec** (section 5). +- **No production traffic was used.** How often any of these situations occurs is not known. +- **The line numbers in this report** match `dash/event.go` on this branch's last commit. They move when `event.go` changes. + +## 12. Earlier independent review (not repeated) + +For the previous version of this report, four models on the LiteLLM proxy each checked the first +five cases (summary break, coref, added tool, model change, effort change) from general knowledge +and public prompt-caching documentation. They did not see the repo. The models were Opus 5.5, Sonnet +5.5, `azure/gpt-6.1-sol` (in place of a refused `aws/gpt-6-astra`) and `azure/FW-Kimi-K3` (in place +of an absent `aws/kimi-k2.5`). **They were not run again for this version.** Their verdicts are +opinions, not measurements. + +| Cause here | Earlier case | Consensus | +|---|---|---| +| 1 | Summary that breaks the cache; edit inside the cached prefix | Spec right, 4 of 4 for each. Cached content would have been read, so a write-rate price has the wrong sign. | +| 3 | Added tool declaration | Spec right, 3 of 4. Totals match and cg's clamp hides a real cost. | +| 4 | Model change; effort change | Different question, 4 of 4 and 3 of 4. The effort premise is unverified. | + +All four confirmed the arithmetic of the earlier totals. They also agreed on the B\* reading in +section 5. Two errors they found in the earlier text (the tail is 3,000 tokens, and `coref` repays in +about 10 more turns, not 11) are corrected here. diff --git a/docs/dashboard.md b/docs/dashboard.md index 3f8f60f2..1c965025 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -92,6 +92,84 @@ SQLite (pure Go, no C toolchain), in WAL mode. Retention is bounded by both **ag blow past the limit and a quiet week can't erase itself. `--dashboard-db :memory:` runs with no persistence at all. +## Savings source + +Every saving the dashboard stores comes from the accountant library +([rossoctl/accountant](https://github.com/rossoctl/accountant)). It needs `--dashboard`: the +library's result is attached to the recorded event, so without a recorder nothing runs. The proxy +itself books no saving: `Event.Price` stores the cost, the provider's own cache saving and the +accounting fields, and nothing else. + +What the library prices: + +- **Components.** Each component's saved dollars, net of the cache break it caused, and the baseline + cost. The component figures sum to baseline minus cost. Each row also stores the library's split: + `saved_first_usd` (saved tokens priced at the write or input tier: content the provider had not + cached), `saved_replay_usd` (at the read tier: content in the cached prefix), and the costs, which + are `saved_usd` minus both. `priced_by` is `accountant`. On a request the library could not price (a + fallback) it is `none`: the row books no saving, and the read views do not value it. +- **The envelope.** The proxy changes `tools` and `system` in seven places (toolschema, toolfilter, + cachesplit, headttl, the tail mark, expand and adjudicate). It reports each change to the library + from a diff of the two bodies. An injected tool declaration (expand, adjudicate) is a cost on a + row of its own. +- **Keep-alive.** `keepalive_saved_usd` is the library's rescue. The proxy passes each ping's usage, + and the library derives the extent the ping read from it. A prefix that another thread of the + session kept warm inside the TTL is not credited (`EntryFacts.SharedWarm`, accountant R54). +- **Cachesplit.** `cachesplit_saved_usd` is the library's split effect. +- **Head TTL.** The 1h premium and the rescue it earns are on the `headttl` row. +- **A break nobody owns.** A break that a change left behind when it ended is counted on the + `ended_change` row. A break the client body has too (a client edit, an expiry) is not counted. + +When the library cannot price an event, the event books **no saving**, and the proxy logs one +`acc_fallback` line with the reason: `no_result`, `ops_dropped`, `observe`, `unpriced`, +`check_sum`, `check_unique_change`, `check_spans`, `size_unknown`, `implausible_size`, +`figure_without_row`, `duplicate_name` or `acted_without_figure`. A saving nothing measured is not +shown. + +Limits: + +- A request with no history to classify a cache break against gets `penalty_unknown`. This is the + first request of a thread, and a request after a gap longer than the cache lifetime when the + provider's lifetime is unknown (no explicit breakpoints). The gross saving is right, and the net + can be too high if a component broke the cache on that request (accountant design §6.4). The + event carries an estimate flag, stored in `requests.price_estimated` and sent as `price_estimated` + in the request JSON. The request detail marks the net saving with "≈" and says why in its tooltip. + The totals and charts add such a figure like any other: they do not mark it. A row written before + the column existed reads as exact. +- A thread that continues another by content (a rewind, a fork) is priced as a first request. The + proxy passes the parent thread to the library, but the library does not read it, so the new thread + does not start from a copy of the parent's cache state (accountant design §4.1). +- The cache lifetime comes from the provider rule the cold gate uses (`apply.CacheLifetime`): an + exact expiry for Anthropic, a minimum for OpenAI's 30-minute entries. Where it is unknown, the + library does not credit a keep-alive rescue after a gap past the TTL. +- The proxy measures the operations against the body it forwards, and starts the request record from + the body it received. When a repair runs before the pipeline (`repairExpandErrors`, + `AnswerStrayCalls`, a forced model), the two differ, the sum check fails (`check_sum`) and the + event books no saving. A request with no usage, or a model with no price, books no saving and logs + `unpriced`. +- A change the pipeline made that no operation can describe drops every operation of the request + (`ops_dropped`): a message whose only change is a `cache_control` mark, a cache anchor written + after a count-changing splice, and a splice that moves the later messages in a request the equal-count + writeback handled. Such a request books no saving. +- A request whose response called `context_guru_expand` (more than one upstream round) books no + saving (`no_result`): the usage is billed on the expanded body, which the library did not see. +- The thread state lives in memory. After a restart, and for a thread dropped by the library's cap + of 10,000 threads (the oldest go first), the next request of the thread is a first request again. +- The figures are estimates. The library converts removed bytes to tokens with the request's own + billed-token ratio. A figure that implies more removed tokens than the request had is rejected + (`implausible_size`). +- The cost and the provider's cache saving are not the library's. +- Rows stored before the switch (`priced_by` empty) keep their legacy figures. The Components tab, + the summarize ledger and the replay cap value those rows with the legacy rule at read time, and + sum the stored split on the rows the library priced. A window that spans the switch mixes the two. +- The savings tracker runs only when a dashboard is on. The pipeline records what each component + changed on every request, with or without a dashboard (a scan of the changed messages). That cost + is accepted. +- A request that asks for the one-hour tier and is billed the five-minute tier (a gateway that + downgrades it) keeps the one-hour TTL in the facts, because the facts are written before the + response. The library can then credit a rescue after a gap of 5 to 60 minutes that the provider + did not honour. The keeper uses the billed tier. + ## Tool inventory panel *How much of what your agent carries in every request does it actually use?* diff --git a/docs/data-dependence.md b/docs/data-dependence.md index 3999913b..db682222 100644 --- a/docs/data-dependence.md +++ b/docs/data-dependence.md @@ -226,6 +226,7 @@ One row for each registered component. The test `components/all/datadep_test.go` | Thread tracker | metadata | 🟡 Works, metadata | 🟡 Works, metadata | 🟡 Works, metadata | It tells the main agent from its subagents. It uses hashes to check if a new request continues an earlier one. | | Session tracker | metadata | 🟡 Works, metadata | 🟡 Works, metadata | 🟡 Works, metadata | It needs a stable label for each conversation. When the client sends none, it hashes the start of the conversation. | | Cache tracker | metadata | 🟡 Works, metadata | 🟡 Works, metadata | 🟡 Works, metadata | It must know how much of the conversation the provider already cached, so components never edit that part. After a restart it asks the dashboard (when on) for the time each thread's cached prefix was last refreshed — by its own last request or a keep-alive ping that read the cache, under either session id the conversation uses (the row stores the content-derived alias id, a hash like the session id itself); timestamps and ids, metadata only — and treats a thread refreshed inside the request's cache lifetime as cached, not as new, until that thread's first request. Without the dashboard — and for a tenant running observe mode, whose rows carry no session id — it uses the proxy's start time instead. | +| Accountant savings (per-component figures from the accountant library) | metadata | ❌ Off. It needs the dashboard, because the figure is attached to a dashboard event. | 🟡 Works, metadata | 🟡 Works, metadata | To price a saving it must know which part of the conversation a rewrite changed and whether the provider had that part cached. It reads each request once and keeps only a hash and a size for each part, plus the cache breakpoints. The text is dropped as soon as it is hashed, and the hashes are held in memory only, one set per conversation thread, plus each thread's last set for the sibling check (SharedWarm). It reads the tools and system sections the same way, as names, hashes and sizes, to report the envelope changes. The result it writes to the dashboard is dollar amounts for each component, with their split by tier. No level changes what it does. | | Wait-cause and lifecycle signals | metadata | ❌ Off (it needs the dashboard) | 🟡 Works, metadata | 🟡 Works, metadata | To tell why the agent paused, it reads the request once. It keeps only counts and ids. Since #857 it also reads the **tool name** of the last tool call, to tell that an agent has finished (`done`). The keep-alive then sends no pings for that thread. A tool name is metadata, and the rule works at every storage level. Since #858 the keep-alive also uses the **wait cause** (from the stop reason, the tool names and the open-task counts), whether the thread id starts with `a:`, the token count of the prefix and the time of day, to pick the ping count, to ignore a strategy's time windows for a wait that is not for a person, and to skip the sweep question during a tool wait. It also keeps one flag per thread: "has a tool call been seen". All of it is metadata, and it works at every storage level. | | Costly settings check (`/context-guru:save-more settings`, `GET /api/adapt/settings`) | metadata | ❌ Off (the cost needs the dashboard). The plugin still finds the settings and says "cost not measured yet". | 🟡 Works, metadata | 🟡 Works, metadata | It counts and prices dashboard rows by request shape: model name, tool count, `max_tokens` and the stream flag. One check also uses the upstream status (400) of a Haiku row in the same session. It reads no request text and keeps nothing new. The plugin reads only the named keys of the user's own settings files (the betas setting, the recap setting, `ANTHROPIC_SMALL_FAST_MODEL` and `ANTHROPIC_DEFAULT_HAIKU_MODEL`) on the user's machine, and sends none of them to the proxy (#615). | | Beep (`beep.sh`, the `beep` plugin option) | none | 🟡 Works, no data | 🟡 Works, no data | 🟡 Works, no data | It plays a sound when Claude finishes or waits for you. It reads no stored data and never reads the hook payload. It sends nothing to the proxy or anywhere else. The level changes nothing. | diff --git a/docs/superpowers/plans/2026-10-28-proxy-accountant-savings.md b/docs/superpowers/plans/2026-10-28-proxy-accountant-savings.md new file mode 100644 index 00000000..dc7d8e12 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-proxy-accountant-savings.md @@ -0,0 +1,1394 @@ +# Proxy savings from `accountant` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +> **Renamed, 2026-11-02.** The library this plan calls `counterfactual` is now `github.com/rossoctl/accountant`, in its own repo. The in-repo module, `docs/counterfactual/` and the `--savings` flag with its `legacy`, `shadow` and `counterfactual` modes no longer exist: the proxy always uses the accountant figure when a dashboard is on. The module paths, file paths and mode names below are the plan as written then. + +**Goal:** Make `accountant.Compute` the only source of savings dollars in the context-guru proxy, and delete the rules in `dash/event.go` that it replaces. + +**Architecture:** The pipeline records per-message byte changes on each `components.Report`. `apply` turns them into `counterfactual.Operation`s (positions in wire space, `ChangeID` from a content hash). A proxy-side `savingsTracker` owns one `counterfactual.Trackers`, runs `BeginRequest` / `Add` / `EndRequest` / `Compute` around each request, and hands the `Result` to `dash.Event`, which prices tiers with `modelinfo.Price`. A `--savings` mode (`legacy` | `shadow` | `counterfactual`) lets the new figure run beside the old one before it replaces it. + +**Tech Stack:** Go 1.26.4, `github.com/rossoctl/context-guru/counterfactual` (stdlib-only module), `gjson`, SQLite via `dash`. + +**Spec:** `docs/superpowers/specs/2026-10-28-proxy-accountant-savings-design.md`. Method: `docs/counterfactual/design.md`. Host wiring: `docs/counterfactual/cortex-wiring.md` (W5, W7.3, W9). + +## Global Constraints + +- Cortex is out of scope. No file outside this repo changes. +- `counterfactual` imports only the standard library (`counterfactual/boundary_test.go` enforces it). The keep-alive conversion therefore takes a library-local struct, never `components.KeepAliveReport`. +- Operation sizes are bytes only. The library converts bytes to tokens (design §4.2 rule 7). +- The proxy never adds token tiers into one token total. Tokens stay per tier; dollars are summed. +- A counterfactual never enters spend: `CostUSD` is untouched by every change here. +- A shadow run still calls `Compute`, so `ThreadState` advances (cortex-wiring W7.3 step 3). +- No `dash` schema change. New figures go into existing columns (`CompRow.SavedUSD`, `BaselineCostUSD`, `KeepAliveSavedUSD`). +- Do not delete `cachesplitSavedUSD` or its inputs. The library does not implement the `split` effect (`counterfactual/doc.go`, "Not implemented"). +- Commits: `git commit -s`, trailer `Assisted-By: Claude (Anthropic AI) `. Never `Co-Authored-By`. +- Before each commit: `gofmt -l ` (always give a path) and `go vet .//...`. + +## Review Focus + +1. **A change that ended and is reduced again.** The client compacts, the engine re-reduces the same content under the same `ChangeID`. The host must report it again. Test: Task 6, `TestSavings_ReReportsAfterChangeEnded`. +2. **Message count changes.** Summarize, sweep and coref drop messages, so `changedIdx` is nil. They must still produce a `MessageSplice`, or every later position in the thread is wrong. Test: Task 5. +3. **Two requests of one thread in flight.** `Compute` returns `Recomputed` when another request replaced the `ThreadState` first. The host must not crash or double-count. Test: Task 6, `TestSavings_ConcurrentRequests`. +4. **No usage, a streamed error, or an unpriced model.** The row must stay unpriced, not read as zero saving. Test: Task 8. +5. **An `observe`-mode (dry-run) run.** Its operations carry `DryRun`, which must never reach a dollar total. Test: Task 8, `TestPriceCF_DryRunNeverCounted`. + +--- + +## File Structure + +| File | Responsibility | +|---|---| +| `counterfactual/host.go` (new) | `BillingFrom`, `PingOps`, `Trackers.IsActive`. Pure host helpers. | +| `components/pipeline.go` (modify) | Fill `Report.Changes` in `runOne`. | +| `components/component.go` (modify) | `Change` type, `Report.Changes`, `KeepAliveReport.Window`. | +| `apply/ops.go` (new) | `OpsFromRun`: `RunReport` + slots → `[]counterfactual.Operation`. | +| `apply/apply.go` (modify) | Fill `Trace.Ops`. | +| `proxy/cfsavings.go` (new) | `savingsTracker`: bodies, breakpoints, Begin/Add/End/Compute, replay filter. | +| `proxy/cfsavings_body.go` (new) | `cfBodies(raw []byte)` and `cfBreakpoints(raw []byte)`. | +| `proxy/dashcapture.go` (modify) | Hold the open request; call `Compute` in `finish`. | +| `dash/cfprice.go` (new) | `Event.PriceCF`: tiers → dollars, per component. | +| `dash/event.go` (modify) | Call `PriceCF` by mode; delete replaced rules in Task 11. | +| `cmd/context-guru-proxy/main.go` (modify) | `--savings` flag; `dashEmitter` converts pings. | + +--- + +### Task 1: Library host helpers + +**Files:** +- Create: `counterfactual/host.go` +- Test: `counterfactual/host_test.go` + +**Interfaces:** +- Consumes: `Trackers`, `Tracker`, `ThreadState`, `Operation`, `RequestOps`, `ModelCall`, `CallUsage`, `Billing`, `Thread` from `types.go` / `state.go`. +- Produces: + - `func BillingFrom(mode BillingMode) Billing` + - `type Ping struct { Session, ThreadID, Model, Host string; Usage CallUsage; At time.Time; Window time.Time }` + - `type PingFacts struct { LastPing time.Time; PingWindow string }` + - `func PingOps(p Ping, b Billing) (RequestOps, PingFacts)` + - `func (ts *Trackers) IsActive(session string, t Thread, changeID string) bool` + +- [ ] **Step 1: Write the failing test** + +```go +// counterfactual/host_test.go +package counterfactual + +import ( + "testing" + "time" +) + +func TestPingOps_OneSpendWithWindowChangeID(t *testing.T) { + at := time.Date(2026, 10, 28, 12, 0, 0, 0, time.UTC) + win := at.Add(-4 * time.Minute) + ops, facts := PingOps(Ping{ + Session: "s1", ThreadID: "", Model: "m", Host: "h", + Usage: CallUsage{Usage: Usage{CacheRead: 1000, Input: 10}, Output: 1}, + At: at, Window: win, + }, BillingFrom(BillingAPI)) + + if ops.Thread.ID != "main" { + t.Fatalf("thread = %q, want main", ops.Thread.ID) + } + if len(ops.Ops) != 1 || ops.Ops[0].Kind != KindSpend || ops.Ops[0].Call == nil { + t.Fatalf("want one spend with a call, got %+v", ops.Ops) + } + wantID := "ka:main:" + win.UTC().Format(time.RFC3339) + if ops.Ops[0].ChangeID != wantID || facts.PingWindow != wantID { + t.Fatalf("ChangeID = %q, PingWindow = %q, want %q", ops.Ops[0].ChangeID, facts.PingWindow, wantID) + } + if !facts.LastPing.Equal(at) { + t.Fatalf("LastPing = %v, want %v", facts.LastPing, at) + } +} + +func TestIsActive_FalseForUnknownThread(t *testing.T) { + ts := NewTrackers() + if ts.IsActive("s", Thread{ID: "main"}, "cg:x") { + t.Fatal("an unseen thread has no active change") + } +} +``` + +- [ ] **Step 2: Run it and confirm it fails** + +Run: `go test ./counterfactual/ -run 'TestPingOps|TestIsActive' -v` +Expected: FAIL, `undefined: PingOps`. + +- [ ] **Step 3: Write the implementation** + +```go +// counterfactual/host.go +package counterfactual + +import "time" + +// BillingFrom wraps a billing mode the host read from the request's credential. +func BillingFrom(mode BillingMode) Billing { return Billing{Mode: mode} } + +// Ping is a keep-alive ping in the library's own terms, so the library does not import a host +// type. The host maps its report onto this. +type Ping struct { + Session, ThreadID, Model, Host string + Usage CallUsage + At time.Time // when the ping started + Window time.Time // the start of the idle gap the ping served +} + +// PingFacts are what the thread's next real request needs in EntryFacts (cortex-wiring W4.6). +type PingFacts struct { + LastPing time.Time + PingWindow string +} + +// PingOps turns a ping into a RequestOps with one spend, and returns the facts for the next +// request. The thread "" is the primary thread and becomes "main". +func PingOps(p Ping, b Billing) (RequestOps, PingFacts) { + id := p.ThreadID + if id == "" { + id = "main" + } + change := "ka:" + id + ":" + p.Window.UTC().Format(time.RFC3339) + return RequestOps{ + Version: 1, + Session: p.Session, + Thread: Thread{ID: id}, + Model: p.Model, + At: p.At, + Billing: b, + Ops: []Operation{{ + Kind: KindSpend, + Operator: "context-guru.keepalive", + ChangeID: change, + Call: &ModelCall{Host: p.Host, Model: p.Model, Usage: p.Usage}, + }}, + }, + PingFacts{LastPing: p.At, PingWindow: change} +} + +// IsActive reports whether the thread still applies the change again on each request. The host +// uses it to report a change once and not again while the library still carries it (§4.6). +func (ts *Trackers) IsActive(session string, t Thread, changeID string) bool { + ts.mu.Lock() + tr, ok := ts.m[ThreadKey(session, t)] + ts.mu.Unlock() + if !ok { + return false + } + for _, op := range tr.snapshot().Active { + if op.ChangeID == changeID { + return true + } + } + return false +} +``` + +- [ ] **Step 4: Run it and confirm it passes** + +Run: `go test ./counterfactual/... && gofmt -l counterfactual && go vet ./counterfactual/...` +Expected: PASS, no gofmt output. + +- [ ] **Step 5: Commit** + +```bash +git add counterfactual/host.go counterfactual/host_test.go +git commit -s -m "feat(counterfactual): Add the host helpers BillingFrom, PingOps and IsActive + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 2: Make the root module see `counterfactual` + +**Files:** +- Modify: `go.mod` + +**Interfaces:** +- Produces: `import cf "github.com/rossoctl/context-guru/counterfactual"` compiles in the root module with or without `go.work`. + +- [ ] **Step 1: Write a failing compile check** + +Create `proxy/cf_import_test.go`: + +```go +package proxy + +import ( + "testing" + + cf "github.com/rossoctl/context-guru/counterfactual" +) + +func TestCounterfactualImportable(t *testing.T) { + if cf.NewTrackers() == nil { + t.Fatal("nil Trackers") + } +} +``` + +- [ ] **Step 2: Run it with the workspace off** + +Run: `GOWORK=off go test ./proxy/ -run TestCounterfactualImportable` +Expected: FAIL, `no required module provides package github.com/rossoctl/context-guru/counterfactual`. + +- [ ] **Step 3: Add the require and replace** + +```bash +go mod edit -require=github.com/rossoctl/context-guru/counterfactual@v0.0.0 \ + -replace=github.com/rossoctl/context-guru/counterfactual=./counterfactual +``` + +- [ ] **Step 4: Verify both ways** + +Run: `GOWORK=off go test ./proxy/ -run TestCounterfactualImportable && go test ./proxy/ -run TestCounterfactualImportable && GOWORK=off go mod tidy -diff` +Expected: PASS twice, empty tidy diff. + +- [ ] **Step 5: Commit** + +```bash +git add go.mod go.sum proxy/cf_import_test.go +git commit -s -m "build(counterfactual): Require the counterfactual module from the root module + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 3: Record per-message byte changes on `Report` + +`changedIdx` says which messages changed, not by how many bytes. The sum check needs bytes (design §6.5), and `ChangeID` needs a content hash. Fill both in `runOne`, so no offloader changes. + +**Files:** +- Modify: `components/component.go` (type `Report`, near line 807) +- Modify: `components/pipeline.go` (`runOne`, default branch near line 211) +- Test: `components/pipeline_changes_test.go` (new) + +**Interfaces:** +- Produces: + - `type Change struct { Idx int; BytesBefore, BytesAfter int; Hash string }` in package `components` + - `Report.Changes []Change`, one per entry of `ChangedIdx`, same order. `Hash` is the first 12 hex characters of sha256 over the message text before the run. + - `Report.CountChanged bool`: the run changed the message count (`len(before) != len(after)`), so `Changes` is empty and Task 5 handles it. + +- [ ] **Step 1: Write the failing test** + +```go +// components/pipeline_changes_test.go +package components + +import "testing" + +func TestRunOne_RecordsByteChangesAndStableHash(t *testing.T) { + big := string(make([]byte, 4000)) + run := func() Report { + p := NewPipeline(shrinkToolOutputs{}) + rr := p.Run(newReq(userMsg("hi"), toolMsg(big)), &Ctx{}) + return rr.Components[0] + } + a, b := run(), run() + if len(a.Changes) != 1 { + t.Fatalf("Changes = %+v, want one", a.Changes) + } + c := a.Changes[0] + if c.Idx != 1 || c.BytesBefore <= c.BytesAfter || c.BytesAfter <= 0 { + t.Fatalf("change = %+v", c) + } + if c.Hash == "" || c.Hash != b.Changes[0].Hash { + t.Fatalf("hash %q vs %q: must be equal for equal input", c.Hash, b.Changes[0].Hash) + } +} +``` + +Use the existing helpers in `components/pipeline_test.go` for `newReq`, `userMsg`, `toolMsg` and a shrinking component. Open that file first, and if the helper names differ, use the names it defines. Do not add parallel helpers. + +- [ ] **Step 2: Run it and confirm it fails** + +Run: `go test ./components/ -run TestRunOne_RecordsByteChanges -v` +Expected: FAIL, `Changes undefined`. + +- [ ] **Step 3: Add the type and fill it** + +In `components/component.go`, above `type Report`: + +```go +// Change is one message a component rewrote, in bytes. The pipeline fills it, so no component +// has to. Hash is of the text BEFORE the run: the same content re-sent next turn gets the same +// hash, which is what lets the savings code recognise a replay (counterfactual ChangeID). +type Change struct { + Idx int // index in req.Input + BytesBefore, BytesAfter int + Hash string +} +``` + +In `type Report`, after `ChangedIdx []int`: + +```go + // Changes is ChangedIdx with sizes and a content hash. Empty when CountChanged. + Changes []Change + // CountChanged says the run added or removed messages (a summary, a drop). ChangedIdx and + // Changes are empty then, because indices before and after do not line up. + CountChanged bool +``` + +In `runOne`, in the `default:` branch right after `rep.ChangedIdx = changedIdx(before, req.Input)`: + +```go + rep.CountChanged = len(before) != len(req.Input) + rep.Changes = changesOf(before, req.Input, rep.ChangedIdx) +``` + +Add to `components/pipeline.go`: + +```go +// changesOf sizes each changed message and hashes its pre-run text. +func changesOf(before, after []schemas.ChatMessage, idx []int) []Change { + out := make([]Change, 0, len(idx)) + for _, i := range idx { + if i < 0 || i >= len(before) || i >= len(after) { + continue + } + bt, at := schema.MessageText(before[i]), schema.MessageText(after[i]) + sum := sha256.Sum256([]byte(bt)) + out = append(out, Change{Idx: i, BytesBefore: len(bt), BytesAfter: len(at), + Hash: hex.EncodeToString(sum[:6])}) + } + return out +} +``` + +`restoreOwn` can rewrite `req.Input` after `changedIdx`, so compute `Changes` after the `restoreOwn` call, not before it. + +- [ ] **Step 4: Run it and confirm it passes** + +Run: `go test ./components/... && gofmt -l components && go vet ./components/...` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add components/component.go components/pipeline.go components/pipeline_changes_test.go +git commit -s -m "feat(components): Record the bytes and a content hash of each rewritten message + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 4: `apply` builds Operations + +**Files:** +- Create: `apply/ops.go` +- Modify: `apply/apply.go` (struct `Trace` near line 221; where `Trace.Run` and `slots` are both in scope, near line 487 and the end of `BodyOpts`) +- Test: `apply/ops_test.go` + +**Interfaces:** +- Consumes: `components.RunReport`, `components.Report.Changes`, `components.Change`, `wireGroups(slots []slot) []int`. +- Produces: + - `func OpsFromRun(rr components.RunReport, wire []int, applied bool) (ops []cf.Operation, dropped int)` + - `Trace.Ops []cf.Operation` and `Trace.OpsDropped int`. + - Operator name: `"context-guru." + Report.Component`. `ChangeID`: `"cg:" + Component + ":" + Change.Hash`. + +`wire[i]` is the wire message index for `req.Input[i]`. When `wire` is nil, `wireGroups` could not map a slot, so `OpsFromRun` returns no ops and counts every change as dropped. Fail open, never guess a position. + +- [ ] **Step 1: Write the failing test** + +```go +// apply/ops_test.go +package apply + +import ( + "testing" + + cf "github.com/rossoctl/context-guru/counterfactual" + "github.com/rossoctl/context-guru/components" +) + +func TestOpsFromRun_MapsToWirePositionAndSkipsRevertedAndSkipped(t *testing.T) { + rr := components.RunReport{Components: []components.Report{ + {Component: "mask", Changes: []components.Change{{Idx: 1, BytesBefore: 4000, BytesAfter: 100, Hash: "aa11"}}}, + {Component: "dedup", Reverted: true, Changes: []components.Change{{Idx: 0, BytesBefore: 9, BytesAfter: 1, Hash: "bb22"}}}, + {Component: "toon", Skipped: true}, + }} + ops, dropped := OpsFromRun(rr, []int{0, 3}, true) + if dropped != 0 || len(ops) != 1 { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } + o := ops[0] + if o.Kind != cf.KindRewrite || o.Operator != "context-guru.mask" || o.ChangeID != "cg:mask:aa11" { + t.Fatalf("op = %+v", o) + } + if o.At != (cf.Position{Section: cf.SectionMessages, Index: 3}) { + t.Fatalf("At = %+v, want messages[3] (wire index, not req.Input index)", o.At) + } + if o.SizeBefore.Bytes != 4000 || o.SizeAfter.Bytes != 100 || o.DryRun { + t.Fatalf("sizes/dryrun = %+v", o) + } +} + +func TestOpsFromRun_DryRunWhenNotApplied(t *testing.T) { + rr := components.RunReport{Components: []components.Report{ + {Component: "mask", Changes: []components.Change{{Idx: 0, BytesBefore: 10, BytesAfter: 1, Hash: "cc"}}}, + }} + ops, _ := OpsFromRun(rr, []int{0}, false) + if len(ops) != 1 || !ops[0].DryRun { + t.Fatalf("a run that was not applied must be DryRun: %+v", ops) + } +} + +func TestOpsFromRun_NilWireDropsEverything(t *testing.T) { + rr := components.RunReport{Components: []components.Report{ + {Component: "mask", Changes: []components.Change{{Idx: 0, BytesBefore: 10, BytesAfter: 1, Hash: "cc"}}}, + }} + ops, dropped := OpsFromRun(rr, nil, true) + if len(ops) != 0 || dropped != 1 { + t.Fatalf("ops = %+v dropped = %d", ops, dropped) + } +} +``` + +- [ ] **Step 2: Run it and confirm it fails** + +Run: `go test ./apply/ -run TestOpsFromRun -v` +Expected: FAIL, `undefined: OpsFromRun`. + +- [ ] **Step 3: Write the implementation** + +```go +// apply/ops.go +package apply + +import ( + cf "github.com/rossoctl/context-guru/counterfactual" + "github.com/rossoctl/context-guru/components" +) + +// OpsFromRun turns a run's per-message changes into counterfactual operations, in pipeline +// order. wire maps a req.Input index to the wire message index (wireGroups). applied is false for +// an observe-mode run: its operations are DryRun, computed but never sent (R12). +// +// A reverted or skipped report changed nothing upstream and reports nothing. A run that changed +// the message count is Task 5's (spliceOps) and is not handled here. +func OpsFromRun(rr components.RunReport, wire []int, applied bool) (ops []cf.Operation, dropped int) { + for _, r := range rr.Components { + if r.Skipped || r.Reverted { + continue + } + for _, c := range r.Changes { + if wire == nil || c.Idx < 0 || c.Idx >= len(wire) { + dropped++ + continue + } + sz := func(n int) cf.Size { return cf.Size{Bytes: n} } + ops = append(ops, cf.Operation{ + Kind: cf.KindRewrite, + Operator: "context-guru." + r.Component, + DryRun: !applied, + At: cf.Position{Section: cf.SectionMessages, Index: wire[c.Idx]}, + SizeBefore: sz(c.BytesBefore), + SizeAfter: sz(c.BytesAfter), + ChangeID: "cg:" + r.Component + ":" + c.Hash, + }) + } + } + return ops, dropped +} +``` + +In `apply/apply.go`, add to `Trace`: + +```go + // Ops are the rewrite operations of this request for the savings computation, in pipeline + // order, with wire positions. OpsDropped counts changes that could not be placed. + Ops []cf.Operation + OpsDropped int +``` + +and import `cf "github.com/rossoctl/context-guru/counterfactual"`. Where `BodyOpts` has both `slots` and the finished `RunReport` (find the assignment to `tr.Run`), add: + +```go + tr.Ops, tr.OpsDropped = OpsFromRun(*tr.Run, wireGroups(slots), applied) +``` + +`applied` is true when the body that is forwarded carries the rewrite. Read how `BodyOpts` decides `Changed` and the observe mode, and pass the matching boolean. Do not invent a new rule. + +- [ ] **Step 4: Run it and confirm it passes** + +Run: `go test ./apply/... && gofmt -l apply && go vet ./apply/...` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apply/ops.go apply/ops_test.go apply/apply.go +git commit -s -m "feat(apply): Build counterfactual operations from the pipeline run + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 5: Operations for components that change the message count + +Summarize, sweep and coref remove or add whole messages. `changedIdx` returns nil for them (`len(before) != len(after)`), so Task 4 reports nothing, and every later position in the thread would be wrong. + +**Files:** +- Modify: `components/component.go` (`Change` gains `Splice *Splice`) +- Modify: `components/pipeline.go` (`runOne`) +- Modify: `apply/ops.go` +- Test: `components/pipeline_changes_test.go`, `apply/ops_test.go` + +**Interfaces:** +- Consumes: `Report.CountChanged` from Task 3. +- Produces: for a count-changing run, `Report.Changes` has one entry with `Splice: &components.Splice{At, Removed, Inserted int}`, `BytesBefore` = bytes of the removed messages, `BytesAfter` = bytes of the inserted ones. `OpsFromRun` maps it to `Operation.MessageSplice` and `Spans`. + +- [ ] **Step 1: Find which components do this** + +Run: `grep -rn "len(req.Input)\|req.Input = append\|req.Input\[:" components/offload/*.go components/reformat/*.go | grep -v _test` +Expected: a list. Note which components remove messages. Record them in the commit message. + +- [ ] **Step 2: Write the failing pipeline test** + +```go +func TestRunOne_CountChangeRecordsASplice(t *testing.T) { + // dropMiddle removes messages 1..2 and inserts one summary message at index 1. + p := NewPipeline(dropMiddle{}) + rr := p.Run(newReq(userMsg("a"), toolMsg("bbbb"), toolMsg("cccc"), userMsg("d")), &Ctx{}) + r := rr.Components[0] + if !r.CountChanged || len(r.Changes) != 1 || r.Changes[0].Splice == nil { + t.Fatalf("report = %+v", r) + } + s := r.Changes[0].Splice + if s.At != 1 || s.Removed != 2 || s.Inserted != 1 { + t.Fatalf("splice = %+v", s) + } + if r.Changes[0].BytesBefore != 8 { + t.Fatalf("bytes before = %d, want the 8 bytes removed", r.Changes[0].BytesBefore) + } +} +``` + +Define `dropMiddle` in the test file as an `Offload` that does that rewrite and returns one cache key. + +- [ ] **Step 3: Run it and confirm it fails** + +Run: `go test ./components/ -run TestRunOne_CountChange -v` +Expected: FAIL. + +- [ ] **Step 4: Implement** + +Add `Splice *Splice` to `Change`, and `type Splice struct{ At, Removed, Inserted int }`. In `runOne`, when `rep.CountChanged`, compute the splice with a common prefix and suffix scan over message text: + +```go +// spliceOf finds the one contiguous range that differs between before and after. +func spliceOf(before, after []schemas.ChatMessage) (Change, bool) { + p := 0 + for p < len(before) && p < len(after) && reflect.DeepEqual(before[p], after[p]) { + p++ + } + s := 0 + for s < len(before)-p && s < len(after)-p && + reflect.DeepEqual(before[len(before)-1-s], after[len(after)-1-s]) { + s++ + } + removed, inserted := before[p:len(before)-s], after[p:len(after)-s] + var bb, ba int + h := sha256.New() + for _, m := range removed { + t := schema.MessageText(m) + bb += len(t) + h.Write([]byte(t)) + } + for _, m := range inserted { + ba += len(schema.MessageText(m)) + } + return Change{Idx: p, BytesBefore: bb, BytesAfter: ba, + Hash: hex.EncodeToString(h.Sum(nil)[:6]), + Splice: &Splice{At: p, Removed: len(removed), Inserted: len(inserted)}}, true +} +``` + +In `runOne`: `if rep.CountChanged { if c, ok := spliceOf(before, req.Input); ok { rep.Changes = []Change{c} } }`. + +In `apply/ops.go`, for a change with `Splice != nil`, set `MessageSplice: &cf.MessageSplice{At: wire[c.Splice.At], Removed: c.Splice.Removed, Inserted: c.Splice.Inserted}`. Guard `At == len(wire)` (an append at the end): use `len(wire)` itself as the wire index. Add an `apply` test for the append case and one for a splice at 0. + +- [ ] **Step 5: Run and commit** + +Run: `go test ./components/... ./apply/... && gofmt -l components apply && go vet ./components/... ./apply/...` +Expected: PASS. + +```bash +git add components apply +git commit -s -m "feat(components): Report a message splice when a run changes the message count + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 6: `savingsTracker` in the proxy + +**Files:** +- Create: `proxy/cfsavings_body.go`, `proxy/cfsavings.go` +- Test: `proxy/cfsavings_body_test.go`, `proxy/cfsavings_test.go` + +**Interfaces:** +- Consumes: `cf.NewBody`, `cf.Trackers`, `cf.RequestMeta`, `cf.Breakpoint`, `cf.EntryFacts`, `(*cf.Trackers).IsActive`, `apply.Trace.Ops`, `proxy.Usage`. +- Produces: + - `func cfBodies(raw []byte) cf.Body` — hash of tools, system, messages of a request body + - `func cfBreakpoints(raw []byte) []cf.Breakpoint` + - `type savingsTracker struct{ trackers *cf.Trackers }`; `func newSavingsTracker() *savingsTracker` + - `func (s *savingsTracker) begin(meta cf.RequestMeta, clientBody []byte) *cf.Request` + - `func (s *savingsTracker) end(req *cf.Request, th cf.Thread, forwarded []byte, ops []cf.Operation, facts cf.EntryFacts)` + - `func (s *savingsTracker) compute(req *cf.Request, u Usage) cf.Result` + +`end` filters `ops`: it skips a rewrite whose `ChangeID` is already active on the thread, and `Add`s the rest, then calls `req.EndRequest`. Filtering happens in `end` because the thread is known only there. + +- [ ] **Step 1: Write the body tests** + +```go +// proxy/cfsavings_body_test.go +package proxy + +import ( + "testing" + + cf "github.com/rossoctl/context-guru/counterfactual" +) + +const cfSample = `{"tools":[{"name":"a"}],"system":"sys", + "messages":[{"role":"user","content":"hi"}, + {"role":"assistant","content":[{"type":"text","text":"yo","cache_control":{"type":"ephemeral","ttl":"1h"}}]}]}` + +func TestCfBodies_SectionsAndSizes(t *testing.T) { + b := cfBodies([]byte(cfSample)) + if b.Start != [3]int{0, 1, 2} || len(b.Pos) != 4 { + t.Fatalf("start = %v pos = %d, want [0 1 2] and 4", b.Start, len(b.Pos)) + } +} + +func TestCfBodies_EqualInputEqualHashes(t *testing.T) { + a, b := cfBodies([]byte(cfSample)), cfBodies([]byte(cfSample)) + for i := range a.Pos { + if a.Pos[i] != b.Pos[i] { + t.Fatalf("position %d differs", i) + } + } +} + +func TestCfBreakpoints_FindsBlockLevelMarkAndTTL(t *testing.T) { + got := cfBreakpoints([]byte(cfSample)) + want := []cf.Breakpoint{{Pos: cf.Position{Section: cf.SectionMessages, Index: 1}, TTL: "1h"}} + if len(got) != 1 || got[0] != want[0] { + t.Fatalf("got %+v, want %+v", got, want) + } +} +``` + +- [ ] **Step 2: Run, see it fail, then implement** + +Run: `go test ./proxy/ -run 'TestCfBodies|TestCfBreakpoints' -v` → FAIL, undefined. + +```go +// proxy/cfsavings_body.go +package proxy + +import ( + "github.com/tidwall/gjson" + + cf "github.com/rossoctl/context-guru/counterfactual" +) + +// cfBodies hashes a request body in the provider's order: tools, system, messages. A string +// system prompt is one block. Raw JSON is hashed as sent, so two hosts hash the same bytes alike. +func cfBodies(raw []byte) cf.Body { + var tools, system, msgs [][]byte + gjson.GetBytes(raw, "tools").ForEach(func(_, v gjson.Result) bool { + tools = append(tools, []byte(v.Raw)) + return true + }) + if s := gjson.GetBytes(raw, "system"); s.IsArray() { + s.ForEach(func(_, v gjson.Result) bool { system = append(system, []byte(v.Raw)); return true }) + } else if s.Exists() { + system = append(system, []byte(s.Raw)) + } + gjson.GetBytes(raw, "messages").ForEach(func(_, v gjson.Result) bool { + msgs = append(msgs, []byte(v.Raw)) + return true + }) + return cf.NewBody(tools, system, msgs) +} + +// cfBreakpoints lists every cache_control mark, as a position in the same space as cfBodies. +// A mark inside a message's content blocks belongs to that message. ttl defaults to "5m". +func cfBreakpoints(raw []byte) []cf.Breakpoint { + var out []cf.Breakpoint + ttl := func(cc gjson.Result) string { + if t := cc.Get("ttl").String(); t == "1h" { + return "1h" + } + return "5m" + } + scan := func(sec cf.Section, arr gjson.Result) { + i := 0 + arr.ForEach(func(_, el gjson.Result) bool { + if cc := el.Get("cache_control"); cc.Exists() { + out = append(out, cf.Breakpoint{Pos: cf.Position{Section: sec, Index: i}, TTL: ttl(cc)}) + } + el.Get("content").ForEach(func(_, blk gjson.Result) bool { + if cc := blk.Get("cache_control"); cc.Exists() { + out = append(out, cf.Breakpoint{Pos: cf.Position{Section: sec, Index: i}, TTL: ttl(cc)}) + } + return true + }) + i++ + return true + }) + } + scan(cf.SectionTools, gjson.GetBytes(raw, "tools")) + if s := gjson.GetBytes(raw, "system"); s.IsArray() { + scan(cf.SectionSystem, s) + } + scan(cf.SectionMessages, gjson.GetBytes(raw, "messages")) + return out +} +``` + +- [ ] **Step 3: Write the tracker tests** + +```go +// proxy/cfsavings_test.go +package proxy + +import ( + "sync" + "testing" + "time" + + cf "github.com/rossoctl/context-guru/counterfactual" +) + +func rewriteOp(change string, at, before, after int) cf.Operation { + return cf.Operation{Kind: cf.KindRewrite, Operator: "context-guru.mask", ChangeID: change, + At: cf.Position{Section: cf.SectionMessages, Index: at}, + SizeBefore: cf.Size{Bytes: before}, SizeAfter: cf.Size{Bytes: after}} +} + +func TestSavings_ReportsOnceAndReReportsAfterChangeEnded(t *testing.T) { + s := newSavingsTracker() + th := cf.Thread{ID: "main"} + big := `{"messages":[{"role":"user","content":"` + string(make([]byte, 0)) + `a"},{"role":"user","content":"b"}]}` + run := func(n int, ops ...cf.Operation) cf.Result { + meta := cf.RequestMeta{RequestID: string(rune('0' + n)), Session: "s", At: time.Unix(int64(1000+n*10), 0), Billing: cf.BillingFrom(cf.BillingAPI)} + r := s.begin(meta, []byte(big)) + s.end(r, th, []byte(big), ops, cf.EntryFacts{TTL: 5 * time.Minute, Lifetime: cf.LifetimeExact}) + return s.compute(r, Usage{FreshInput: 10, CacheRead: 100}) + } + run(1, rewriteOp("cg:mask:a1", 1, 4000, 100)) + if !s.trackers.IsActive("s", th, "cg:mask:a1") { + t.Fatal("change must stay active after its first request") + } + // the engine replays the reduction: it must not be reported a second time + res := run(2, rewriteOp("cg:mask:a1", 1, 4000, 100)) + n := 0 + for _, o := range res.Ops { + if o.ChangeID == "cg:mask:a1" { + n++ + } + } + if n != 1 { + t.Fatalf("replayed change priced %d times, want 1 (applied again, not reported again)", n) + } +} + +func TestSavings_ConcurrentRequests(t *testing.T) { + s := newSavingsTracker() + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + body := []byte(`{"messages":[{"role":"user","content":"x"}]}`) + r := s.begin(cf.RequestMeta{RequestID: string(rune('a' + i)), Session: "s", At: time.Now(), Billing: cf.BillingFrom(cf.BillingAPI)}, body) + s.end(r, cf.Thread{ID: "main"}, body, nil, cf.EntryFacts{TTL: 5 * time.Minute, Lifetime: cf.LifetimeExact}) + _ = s.compute(r, Usage{FreshInput: 5}) + }(i) + } + wg.Wait() // run with -race: no data race, no panic +} +``` + +Add `TestSavings_ReReportsAfterChangeEnded`: send a first body, then a second body whose first message differs (a client edit at or before the change position). Assert `IsActive` is false after the second request, then run a third request that reports the same `ChangeID` and assert it is priced (present in `res.Ops` with `Saved` non-zero). + +- [ ] **Step 4: Implement** + +```go +// proxy/cfsavings.go +package proxy + +import cf "github.com/rossoctl/context-guru/counterfactual" + +// savingsTracker runs the counterfactual calls around one request. It holds the one Trackers +// of the proxy: a thread's state lives there between requests. +type savingsTracker struct{ trackers *cf.Trackers } + +func newSavingsTracker() *savingsTracker { return &savingsTracker{trackers: cf.NewTrackers()} } + +// begin opens the request record with the client body, before any component ran. +func (s *savingsTracker) begin(meta cf.RequestMeta, client []byte) *cf.Request { + return s.trackers.BeginRequest(meta, cfBodies(client), cfBreakpoints(client)) +} + +// end records the operations, then the thread and the forwarded body. A rewrite whose ChangeID +// the thread still applies is not added: the library applies it again by itself (§4.6). One that +// ended since is added again, which is how a change after a compaction is priced. +func (s *savingsTracker) end(req *cf.Request, th cf.Thread, forwarded []byte, ops []cf.Operation, facts cf.EntryFacts) { + for _, op := range ops { + if op.Kind == cf.KindRewrite && op.ChangeID != "" && !op.DryRun && + s.trackers.IsActive(req.Meta.Session, th, op.ChangeID) { + continue + } + req.Add(op) + } + req.EndRequest(th, cfBodies(forwarded), cfBreakpoints(forwarded), facts) +} + +// compute prices the request. It stores the new ThreadState (so a shadow run still advances it). +func (s *savingsTracker) compute(req *cf.Request, u Usage) cf.Result { + return req.Compute(cf.Usage{Input: int(u.FreshInput), CacheRead: int(u.CacheRead), + CacheWrite: int(u.CacheWrite), CacheWrite1h: int(u.CacheWrite1h)}) +} +``` + +- [ ] **Step 5: Run and commit** + +Run: `go test -race ./proxy/ -run 'TestCf|TestSavings' && gofmt -l proxy && go vet ./proxy/...` +Expected: PASS. + +```bash +git add proxy/cfsavings.go proxy/cfsavings_body.go proxy/cfsavings_test.go proxy/cfsavings_body_test.go +git commit -s -m "feat(proxy): Add the savings tracker around counterfactual.Compute + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 7: Wire the tracker into the request path (no output used yet) + +**Files:** +- Modify: `proxy/proxy.go` (field on `Handler` near line 231 and 366; the chat path near lines 1203 to 1340) +- Modify: `proxy/dashcapture.go` (struct `capture` near line 34; `finish` near line 438) +- Modify: `cmd/context-guru-proxy/main.go` (flag) +- Test: `proxy/cfwire_test.go` + +**Interfaces:** +- Consumes: `savingsTracker` (Task 6), `apply.Trace.Ops`. +- Produces: + - `Options.Savings string`: `"legacy"` (default) | `"shadow"` | `"counterfactual"`. + - `capture.cfReq *cf.Request`, `capture.cfResult *cf.Result` set in `finish`. + - `Event.cf *cf.Result` (unexported, set via `Event.SetCounterfactual(r cf.Result)`), consumed by Task 8. + +- [ ] **Step 1: Write the failing test** + +Use the existing proxy test harness (look at how `proxy/dashcapture_test.go` or `proxy/proxy_test.go` builds a `Handler` with a fake upstream, and reuse it). The test sends two chat requests on one session with `Savings: "shadow"` and asserts the recorded events carry a counterfactual result: + +```go +func TestShadowMode_AttachesCounterfactualResultAndKeepsLegacyFigure(t *testing.T) { + h, rec := newTestProxyWithDashboard(t, Options{Savings: "shadow"}) // existing helper; adapt the name + post(t, h, chatBody("hello")) + post(t, h, chatBody("hello", "again")) + evs := rec.Events(t) + if len(evs) != 2 { + t.Fatalf("events = %d", len(evs)) + } + for _, e := range evs { + if !e.HasCounterfactual() { + t.Fatal("shadow mode must run Compute on every request") + } + } + // legacy figure unchanged: the event's own Price path ran as before + if evs[1].BaselineCostUSD < evs[1].CostUSD { + t.Fatalf("legacy baseline broke: %+v", evs[1]) + } +} +``` + +Add `func (e *Event) HasCounterfactual() bool` in `dash`. + +- [ ] **Step 2: Run, see it fail** + +Run: `go test ./proxy/ -run TestShadowMode -v` → FAIL, `Options.Savings undefined`. + +- [ ] **Step 3: Implement** + +1. `Options.Savings string`; the constructor rejects any value other than `""`, `legacy`, `shadow`, `counterfactual`. `""` means `legacy`. +2. `Handler.savings *savingsTracker`, created in the constructor only when `Savings != "legacy"`. +3. In the chat path, right where the inbound body is final and before `apply.BodyOpts` runs, call `cp.cfBegin(h.savings, meta, body)`. `meta.Billing` is `cf.BillingFrom(cf.BillingAPI)`. The proxy has no subscription detection today, so every request is `api`. Add a one-line comment saying so. +4. After `apply.BodyOpts`, call `cp.cfEnd(th, forwardedBody, tr.Ops, facts)`. `th` is `cf.Thread{ID: tr.Thread.ID, Parent: }`. Use `"main"` when `tr.Thread.ID` is `thread.Primary`. `facts.TTL` comes from the forwarded body's first breakpoint at or after the boundary: 1h if any breakpoint is `1h`, else 5m, `Lifetime: cf.LifetimeUnknown` for now. Keep-alive ping facts are added in Task 9. +5. In `capture.finish`, before `e.Price(...)`, if `c.cfReq != nil && usageOK`, call `res := c.savings.compute(c.cfReq, usage)` and `e.SetCounterfactual(res)`. If `!usageOK`, do not call `Compute`. The request leaves `ThreadState` untouched, and the row stays unpriced. +6. For a fork (`Inherits`), copy the parent's state before `EndRequest`. Add `(*cf.Trackers).Fork(session string, parent, child Thread)` to the library in this task (a `snapshot` of the parent installed as the child's initial state when the child has none). Test it in `counterfactual/host_test.go`. + +- [ ] **Step 4: Run and commit** + +Run: `go test -race ./proxy/... ./dash/... ./counterfactual/... && gofmt -l proxy dash cmd counterfactual && go vet ./proxy/... ./dash/...` +Expected: PASS; every pre-existing test still passes in `legacy` mode. + +```bash +git add -A proxy dash cmd counterfactual +git commit -s -m "feat(proxy): Run counterfactual.Compute beside the legacy figure with --savings + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 8: Price the result into dollars (`Event.PriceCF`) + +**Files:** +- Create: `dash/cfprice.go` +- Test: `dash/cfprice_test.go` + +**Interfaces:** +- Consumes: `cf.Result`, `cf.PricedOp`, `cf.CacheEffect`, `cf.TierTokens`, `modelinfo.Price`. +- Produces: + - `func tierUSD(p modelinfo.Price, t cf.TierTokens) float64` + - `type CFFigure struct { ByComponent map[string]float64; DeltaUSD float64; KeepAliveSavedUSD float64; UnattributedBreakUSD float64; Unpriced []string }` + - `func (e *Event) CounterfactualFigure(p modelinfo.Price) CFFigure` + +Tier rates: input `p.Input`; read `p.CacheRead`; 5m write `p.CacheWrite`; 1h write `2*p.Input`, the same premium rule as `Price.CostWithCacheWrite1h`. + +- [ ] **Step 1: Write the failing tests** + +```go +// dash/cfprice_test.go +package dash + +import ( + "math" + "testing" + + cf "github.com/rossoctl/context-guru/counterfactual" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +var testPrice = modelinfo.Price{Input: 3e-6, CacheRead: 0.3e-6, CacheWrite: 3.75e-6, Output: 15e-6} + +func near(a, b float64) bool { return math.Abs(a-b) < 1e-12 } + +func TestTierUSD(t *testing.T) { + got := tierUSD(testPrice, cf.TierTokens{Input: 1000, CacheRead: 1000, CacheWrite5m: 1000, CacheWrite1h: 1000}) + want := 1000*3e-6 + 1000*0.3e-6 + 1000*3.75e-6 + 1000*6e-6 + if !near(got, want) { + t.Fatalf("got %v want %v", got, want) + } +} + +func TestFigure_NetOfBreakAttributedByChangeID(t *testing.T) { + e := &Event{} + e.SetCounterfactual(cf.Result{ + Ops: []cf.PricedOp{{Operator: "context-guru.mask", ChangeID: "c1", + Saved: cf.TierTokens{CacheWrite5m: 1000}}}, + CacheEffects: []cf.CacheEffect{{Kind: cf.EffectBreak, ChangeID: "c1", + Caused: cf.TierTokens{CacheWrite5m: 200}, Saved: cf.TierTokens{CacheRead: 200}}}, + }) + f := e.CounterfactualFigure(testPrice) + want := 1000*3.75e-6 - (200*3.75e-6 - 200*0.3e-6) + if !near(f.ByComponent["mask"], want) || !near(f.DeltaUSD, want) { + t.Fatalf("component = %v delta = %v, want %v", f.ByComponent["mask"], f.DeltaUSD, want) + } +} + +func TestPriceCF_DryRunNeverCounted(t *testing.T) { + e := &Event{} + e.SetCounterfactual(cf.Result{Ops: []cf.PricedOp{{Operator: "context-guru.mask", ChangeID: "c1", + DryRun: true, Saved: cf.TierTokens{Input: 5000}}}}) + f := e.CounterfactualFigure(testPrice) + if f.DeltaUSD != 0 || len(f.ByComponent) != 0 { + t.Fatalf("a dry run must not reach a total: %+v", f) + } +} + +func TestFigure_UnattributedBreakStaysOutOfComponents(t *testing.T) { + e := &Event{} + e.SetCounterfactual(cf.Result{CacheEffects: []cf.CacheEffect{{Kind: cf.EffectBreak, + Caused: cf.TierTokens{CacheWrite5m: 100}}}}) + f := e.CounterfactualFigure(testPrice) + if f.UnattributedBreakUSD <= 0 || len(f.ByComponent) != 0 || f.DeltaUSD != 0 { + t.Fatalf("%+v", f) + } +} + +func TestFigure_UnpricedOperationIsListedNotZeroed(t *testing.T) { + e := &Event{} + e.SetCounterfactual(cf.Result{Unpriced: []cf.Unpriced{{OpID: "x", Reason: "no_usage"}}}) + if f := e.CounterfactualFigure(testPrice); len(f.Unpriced) != 1 { + t.Fatalf("%+v", f) + } +} +``` + +- [ ] **Step 2: Run, see it fail** + +Run: `go test ./dash/ -run 'TestTierUSD|TestFigure|TestPriceCF' -v` → FAIL, undefined. + +- [ ] **Step 3: Implement** + +```go +// dash/cfprice.go +package dash + +import ( + "strings" + + cf "github.com/rossoctl/context-guru/counterfactual" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +// SetCounterfactual attaches the library's result for this request. Unexported field: it is +// never stored, only priced (no schema change). +func (e *Event) SetCounterfactual(r cf.Result) { e.cfRes = &r } + +// HasCounterfactual reports whether Compute ran for this request. +func (e *Event) HasCounterfactual() bool { return e.cfRes != nil } + +// tierUSD prices tokens by tier. A 1h write is 2x the input rate, the rule of +// Price.CostWithCacheWrite1h. +func tierUSD(p modelinfo.Price, t cf.TierTokens) float64 { + return float64(t.Input)*p.Input + float64(t.CacheRead)*p.CacheRead + + float64(t.CacheWrite5m)*p.CacheWrite + float64(t.CacheWrite1h)*2*p.Input +} + +// CFFigure is the request's savings in dollars, from the library's tokens. +type CFFigure struct { + ByComponent map[string]float64 // net of the break each component caused + DeltaUSD float64 // sum of ByComponent: the baseline delta + KeepAliveSavedUSD float64 // rescue effects + UnattributedBreakUSD float64 // client edits, expired caches: shown, never charged to a component + Unpriced []string // "opID:reason"; an unpriced operation is not a zero saving +} + +// CounterfactualFigure prices Result. A DryRun operation is skipped: computed, never counted. +func (e *Event) CounterfactualFigure(p modelinfo.Price) CFFigure { + f := CFFigure{ByComponent: map[string]float64{}} + if e.cfRes == nil { + return f + } + owner := map[string]string{} // ChangeID -> component + for _, o := range e.cfRes.Ops { + if o.DryRun || o.Call != nil { + continue + } + c := strings.TrimPrefix(o.Operator, "context-guru.") + owner[o.ChangeID] = c + net := tierUSD(p, o.Saved) - tierUSD(p, o.Caused) + f.ByComponent[c] += net + f.DeltaUSD += net + } + for _, ce := range e.cfRes.CacheEffects { + cost := tierUSD(p, ce.Caused) - tierUSD(p, ce.Saved) + switch ce.Kind { + case cf.EffectBreak: + if c, ok := owner[ce.ChangeID]; ok { + f.ByComponent[c] -= cost + f.DeltaUSD -= cost + } else { + f.UnattributedBreakUSD += cost + } + case cf.EffectRescue: + f.KeepAliveSavedUSD += tierUSD(p, ce.Saved) - tierUSD(p, ce.Caused) + } + } + for _, u := range e.cfRes.Unpriced { + f.Unpriced = append(f.Unpriced, u.OpID+":"+u.Reason) + } + return f +} +``` + +Add `cfRes *cf.Result` to `Event` in `dash/event.go`. Confirm it is unexported so `encoding/json` and the SQLite writer ignore it. + +- [ ] **Step 4: Run and commit** + +Run: `go test ./dash/... && gofmt -l dash && go vet ./dash/...` +Expected: PASS. + +```bash +git add dash +git commit -s -m "feat(dash): Price counterfactual tokens into per-component dollars + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 9: Keep-alive pings and detached calls + +**Files:** +- Modify: `components/component.go` (`KeepAliveReport.Window time.Time`) +- Modify: `keepalive/keeper.go` and `proxy/keepalive.go` (set `Window`) +- Modify: `cmd/context-guru-proxy/main.go` (`dashEmitter`) +- Modify: `proxy/proxy.go` (pass ping facts into `EntryFacts`) +- Create: `proxy/cfping.go` +- Test: `proxy/cfping_test.go` + +**Interfaces:** +- Consumes: `cf.PingOps`, `cf.ComputeStandalone`, `components.KeepAliveReport`. +- Produces: + - `KeepAliveReport.Window time.Time` — when the idle gap that the ping served began + - `type pingFacts struct` store: `func (s *pingStore) put(key string, f cf.PingFacts)`, `func (s *pingStore) take(key string) (cf.PingFacts, bool)`; key is `cf.ThreadKey(session, thread)` + +- [ ] **Step 1: Find where the window starts** + +Run: `grep -n "KeepAliveReport{" keepalive/*.go proxy/*.go | grep -v _test` and read the lines around each hit. +Expected: the place where the report is built. The window start is the time of the last real request on the entry. The keeper already tracks it as the entry's last-activity time. Use that field. If there is no such field, use `TS - interval` of the first ping after the request and say so in a comment. + +- [ ] **Step 2: Write the failing test** + +```go +// proxy/cfping_test.go +package proxy + +import ( + "testing" + "time" + + cf "github.com/rossoctl/context-guru/counterfactual" + "github.com/rossoctl/context-guru/components" +) + +func TestPingFromReport_MapsAndPrices(t *testing.T) { + at := time.Date(2026, 10, 28, 12, 0, 0, 0, time.UTC) + r := components.KeepAliveReport{Session: "s", Thread: "", Model: "m", Route: "h", + CacheRead: 5000, FreshInput: 10, Output: 1, TS: at.UnixMilli(), Window: at.Add(-4 * time.Minute)} + p := pingFromReport(r) + if p.Usage.CacheRead != 5000 || p.Usage.Output != 1 || p.Host != "h" || p.Session != "s" { + t.Fatalf("%+v", p) + } + ops, facts := cf.PingOps(p, cf.BillingFrom(cf.BillingAPI)) + res := cf.ComputeStandalone(ops) + if len(res.Ops) != 1 || res.Ops[0].Call == nil || facts.PingWindow == "" { + t.Fatalf("res = %+v facts = %+v", res, facts) + } +} + +func TestPingStore_TakeClears(t *testing.T) { + s := newPingStore() + s.put("k", cf.PingFacts{PingWindow: "w"}) + if _, ok := s.take("k"); !ok { + t.Fatal("missing") + } + if _, ok := s.take("k"); ok { + t.Fatal("facts must be cleared after the next real request takes them") + } +} +``` + +- [ ] **Step 3: Implement** + +`proxy/cfping.go` holds `pingFromReport(components.KeepAliveReport) cf.Ping` (field map from cortex-wiring W4.6 table) and `pingStore` (mutex + map). The ping's `Usage` is `cf.CallUsage{Usage: cf.Usage{Input: int(r.FreshInput), CacheRead: int(r.CacheRead), CacheWrite: int(r.CacheWrite), CacheWrite1h: int(r.CacheWrite1h)}, Output: int(r.Output)}`. + +In `dashEmitter.KeepAlivePing`, after the existing `d.rec.Record(ev)`, call the shared `pingStore.put(cf.ThreadKey(r.Session, cf.Thread{ID: normalize(r.Thread)}), facts)`. In Task 7's `cfEnd`, call `take` and fill `EntryFacts.LastPing`, `PingWindow`, and `PingRead` (the extent the ping read; use `nil` and leave a comment that the library then falls back to the usage). The existing ping `dash.Event` row stays, so nothing the dashboard reads for pings changes. + +A ping does not call `Compute`. It calls `ComputeStandalone` and attaches the spend to nothing in this task. Log it at debug level. Prices for pings stay in the legacy columns until Task 11. + +- [ ] **Step 4: Run and commit** + +Run: `go test -race ./proxy/... ./keepalive/... ./components/... ./cmd/... && gofmt -l proxy keepalive components cmd && go vet ./proxy/... ./keepalive/...` +Expected: PASS. + +```bash +git add -A components keepalive proxy cmd +git commit -s -m "feat(proxy): Convert keep-alive pings into spend operations and pass their facts on + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 10: Shadow comparison and the agreement test + +**Files:** +- Create: `proxy/cfshadow.go` +- Create: `dash/cfagree_test.go` +- Test: `proxy/cfshadow_test.go` + +**Interfaces:** +- Consumes: `Event.CounterfactualFigure`, legacy `Event.Price`, `opscases.Differences()`, `opscases.All()`, existing `Play` in `dash/counterfactual_play_test.go`. +- Produces: + - `func shadowDiff(e *dash.Event, p modelinfo.Price) (legacy, cfUSD float64)` + - a structured log line `slog` key `cf_shadow` with `session`, `thread`, `legacy_usd`, `cf_usd`, `delta_usd`, `components` + - `TestProxyPathMatchesOpscases` — plays each `opscases.Case` through `savingsTracker` + `CounterfactualFigure` and compares with `Expected` + +- [ ] **Step 1: Write the agreement test** + +```go +// dash/cfagree_test.go +package dash_test + +import ( + "testing" + + "github.com/rossoctl/context-guru/counterfactual/opscases" +) + +// Every case must price the same through the proxy's path as through the library directly. +// A mismatch that has a `differs` entry in differences.json is allowed; any other fails. +func TestProxyPathMatchesOpscases(t *testing.T) { + diffs := opscases.Differences() + for _, c := range opscases.All() { + c := c + t.Run(c.ID, func(t *testing.T) { + got := playThroughProxyPath(t, c) // new helper, below + want := expectedMicros(c) // from opscases.Case.StepPicos + for i := range want { + if got[i] == want[i] { + continue + } + if d, ok := diffs[c.ID]; ok && d.Allows(i, "") { + continue + } + t.Errorf("step %d: proxy path %d µ$, expected %d µ$", i, got[i], want[i]) + } + }) + } +} +``` + +`playThroughProxyPath` mirrors `Play` in `counterfactual_play_test.go`, but drives `Trackers.BeginRequest/Add/EndRequest/Compute` and `Event.CounterfactualFigure`. Because `savingsTracker` lives in `package proxy`, and `dash` must not import `proxy`, the helper uses `counterfactual.Trackers` directly, with the same replay filter (`IsActive`). Keep that filter in one tiny exported function in the library (`counterfactual.Request.AddUnlessActive(op)`) and have `savingsTracker.end` call it too, so the test and the proxy share the code. Add `AddUnlessActive` to `counterfactual/host.go` with a unit test. + +Read `opscases.Difference` (in `opscases.go`) for the real name of the allow-check method. Use that method name, not `Allows`, if it differs. + +- [ ] **Step 2: Run it** + +Run: `go test ./dash/ -run TestProxyPathMatchesOpscases -v 2>&1 | tail -40` +Expected: either PASS, or FAIL listing steps with no `differs` entry. Each failure is a real gap. Triage each one: +- a gap in the library's "Not implemented" list → add a `notCovered` note in the case, do not "fix" the figure; +- a bug in the proxy path → fix it here. + +- [ ] **Step 3: Write the shadow log and its test** + +```go +func TestShadowDiff_LogsBothFigures(t *testing.T) { + // build an Event with legacy Components set and a Result attached + // call shadowDiff; assert legacy and cf come back and delta = cf - legacy +} +``` + +Fill the body from the first passing case in `opscases.All()`, so the numbers are real. `shadowDiff` calls `e.Price(p, true)` first (legacy), then `CounterfactualFigure(p)`, and returns `legacy = e.BaselineCostUSD - e.CostUSD`, `cfUSD = fig.DeltaUSD`. Log with `slog.Info("cf_shadow", ...)`. Call it from `capture.finish` only when `Options.Savings == "shadow"`. + +- [ ] **Step 4: Run and commit** + +Run: `go test ./dash/... ./proxy/... ./counterfactual/... && gofmt -l dash proxy counterfactual && go vet ./dash/... ./proxy/...` +Expected: PASS. + +```bash +git add -A dash proxy counterfactual +git commit -s -m "test(counterfactual): Compare the proxy's path with the opscases and log the shadow difference + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +### Task 11: Switch, then delete + +**Files:** +- Modify: `dash/event.go` (`Price` near line 795; delete rules) +- Modify: `proxy/dashcapture.go` +- Modify: `dash/capture.go` (the `MarkUnique` call site) +- Test: `dash/event_cf_switch_test.go`, then the existing `dash` and `proxy` suites + +**Interfaces:** +- Consumes: `CFFigure` (Task 8), `Options.Savings`. +- Produces: with `Savings == "counterfactual"`, `Price` writes `CompRow.SavedUSD`, `BaselineCostUSD = CostUSD + fig.DeltaUSD`, and `KeepAliveSavedUSD = fig.KeepAliveSavedUSD` from the figure. `CachesplitSavedUSD` stays on the legacy rule. + +This task has two parts. Land the switch first, merge it, and watch real sessions. Delete only in a later commit. + +- [ ] **Step 1: Write the failing switch test** + +```go +func TestPrice_CounterfactualModeUsesTheLibraryFigure(t *testing.T) { + e := &Event{Model: "m", FreshInput: 100, CacheRead: 1000, Components: []CompRow{ + {Component: "mask", Acted: true, SavedGross: 9999, SavedUnique: 9999}, // legacy would price this + }} + e.SetCounterfactual(cf.Result{Ops: []cf.PricedOp{{Operator: "context-guru.mask", ChangeID: "c", + Saved: cf.TierTokens{CacheRead: 1000}}}}) + e.PriceMode(testPrice, true, SavingsCounterfactual) + want := 1000 * 0.3e-6 + if !near(e.Components[0].SavedUSD, want) || !near(e.BaselineCostUSD-e.CostUSD, want) { + t.Fatalf("component = %v baseline delta = %v, want %v", + e.Components[0].SavedUSD, e.BaselineCostUSD-e.CostUSD, want) + } +} +``` + +- [ ] **Step 2: Run, see it fail, implement `PriceMode`** + +Add `type SavingsMode int` with `SavingsLegacy`, `SavingsShadow`, `SavingsCounterfactual`. `PriceMode(p, complete, mode)` calls the existing `Price`; when `mode == SavingsCounterfactual && e.cfRes != nil`, it then overwrites `Components[i].SavedUSD`, `BaselineCostUSD` and `KeepAliveSavedUSD` from `CounterfactualFigure(p)`. When `e.cfRes == nil` (no usage, or a path that never opened a request), it keeps the legacy figures and sets nothing new. Change the one call site in `capture.finish` to `PriceMode`. Make `Price` call `PriceMode(..., SavingsLegacy)` so every other caller is unchanged. + +- [ ] **Step 3: Run the whole suite** + +Run: `go test ./... 2>&1 | tail -30 && GOWORK=off go vet ./...` +Expected: PASS. Fix nothing by editing expected figures unless a `differs` entry in `differences.json` already explains it (cite the entry in the commit message). + +- [ ] **Step 4: Commit the switch and flip the default** + +Change the `--savings` default to `shadow` first. After a soak on real sessions, change it to `counterfactual` in a separate commit. + +```bash +git add -A dash proxy cmd +git commit -s -m "feat(dash): Price savings from counterfactual with --savings=counterfactual + +Assisted-By: Claude (Anthropic AI) " +``` + +- [ ] **Step 5: Delete the replaced rules (separate commit, after the soak)** + +Delete, and fix every compile error, in this order: +1. `dash/event.go`: `uniqueRate`, `repeatRate`, `ComponentSavedUSD`, the reduction part of `baselineDeltaUSD`, `keepaliveSavedUSD`. +2. The `MarkUnique` split in `dash/capture.go` and the `unique` map in `proxy/dashcapture.go` (`unique: map[string]int{}`), plus `Event.FromTrace`'s use of it. +3. Dead callers: `dash/readvalue.go` (`DecomposeComponentSavedUSD`, `EstimateComponentSavedUSD`), `dash/summarizeattr.go`, `dash/replaycap.go`, `dash/sweepattr.go`. These reprice **stored** rows with `ComponentSavedUSD`. Rows written before the switch still need that rule. Therefore do not delete these four until a migration has rewritten the stored rows, or the retention window has passed. If neither is true, leave them and the helpers they call, and record why in the commit message. + +Do not delete: `cachesplitSavedUSD` (no `split` effect in the library), `TailChanged`, or anything in design §9.1. + +- [ ] **Step 6: Final checks** + +Run, from each module you touched: + +```bash +go build ./... && go test ./... && gofmt -l . && go vet ./... +for m in . counterfactual; do (cd $m && GOWORK=off go mod tidy -diff); done +``` + +Expected: all pass, empty `gofmt` and `tidy` output. + +```bash +git add -A +git commit -s -m "refactor(dash): Remove the savings rules that counterfactual replaced + +Assisted-By: Claude (Anthropic AI) " +``` + +--- + +## Self-Review + +**Spec coverage.** +- Spec §4.1 engine Operations → Tasks 3, 4, 5. Deviation: `ChangeID` comes from a content hash made in the pipeline, so no offloader changes. The spec expected per-offloader work. Check that the hash is stable across a component's replays on real traffic during the shadow run. +- Spec §4.2 Tracker → Tasks 6, 7. `Fork` for `Inherits` → Task 7 step 6. +- Spec §4.2 keep-alive, `OpsEmitter` → Task 9. `OpsEmitter` for detached calls is **not** built: the engine's detached summarizer spend has no emitter hook yet. Add it in a follow-up once Task 9 shows how pings reach the store. It does not block the switch. +- Spec §4.2 `MaxCachedIdx` vs B\* counter → **not covered**. Add as Task 7 step 7 if the shadow run shows disagreement: compare `Trace.MaxCachedIdx` with `Result.Boundary` in `cfEnd`/`finish` and count mismatches in `/stats`. +- Spec §4.3 pricing → Task 8. Spec §3 and §5 deletion → Task 11. +- Library additions the spec assumed but the code lacks (`BillingFrom`, ping conversion, `IsActive`, `Fork`, `AddUnlessActive`) → Tasks 1, 7, 10. + +**Placeholder scan.** Three steps say "read the file, then use its real name": Task 3 test helpers, Task 4 the `applied` boolean, Task 10 the allow-check method. Each names the file and the question. The first two exist because I did not read those lines; the third because `Difference` was not printed. None hides logic. + +**Type consistency.** `Change` / `Splice` (Task 3 → 5), `Trace.Ops` (4 → 7), `savingsTracker.begin/end/compute` (6 → 7, 10), `Event.SetCounterfactual` / `CounterfactualFigure` (8 → 10, 11), `PingOps` / `PingFacts` (1 → 9) agree across tasks. + +**Open decisions this plan makes without asking.** +1. `Billing` is `api` for every request (no subscription detection exists in the proxy). +2. `EntryFacts.Lifetime` is `unknown` until the keeper reports a proven TTL, so the phase is often `unknown` and carries `penalty_unknown`. +3. Task 11 leaves the four stored-row repricers alone until stored rows are migrated. diff --git a/docs/superpowers/specs/2026-10-28-proxy-accountant-savings-design.md b/docs/superpowers/specs/2026-10-28-proxy-accountant-savings-design.md new file mode 100644 index 00000000..a78e9040 --- /dev/null +++ b/docs/superpowers/specs/2026-10-28-proxy-accountant-savings-design.md @@ -0,0 +1,128 @@ +# Proxy savings from `accountant`: design + +**Status:** proposed. +**Renamed, 2026-11-02.** The library this spec calls `counterfactual` is now `github.com/rossoctl/accountant`, in its own repo. The in-repo module, `docs/counterfactual/` and the `--savings` flag with its modes were removed. Paths and mode names below are the design as written then. +**Branch:** `feat/counterfactual-savings`, based on `docs/context-operations-spec`. +**Method:** `docs/counterfactual/design.md`. Host wiring: `docs/counterfactual/cortex-wiring.md` (W5, W7.3, W9). + +## 1. Goal and scope + +`accountant.Compute` becomes the only source of savings figures in the context-guru proxy. +It replaces the savings rules in `dash/event.go`. + +In scope: the proxy and the engine, in this repo. This is W9 steps 3 to 6, and the proxy half of step 10. +Out of scope: Cortex. Cortex does not change in this work. + +The flow: + +1. The proxy creates a Tracker for a thread when the thread first appears. +2. The components create Operations while they change a request. +3. The proxy calls `BeginRequest`, `Add`, `EndRequest` and `Compute`. +4. The proxy prices the returned tokens with `internal/modelinfo.Price` and stores the dollars. + +## 2. Assumptions + +1. The library is the module `github.com/rossoctl/context-guru/counterfactual`. The proxy uses it through `go.work` + or a `replace` until the library is on `main`. +2. The library returns tokens per tier. It never returns dollars. The proxy prices. +3. The `dash` columns stay (`ComponentSavedUSD`, `baselineDeltaUSD`). Only their source changes. +4. The library is the fingerprint Tracker. The push model (design §11, question 4) is not decided and is not used. + +## 3. What is replaced + +In `dash/event.go`: + +| Rule | After | +|---|---| +| `ComponentSavedUSD`, `uniqueRate`, `repeatRate` | removed. A component's dollars come from its `PricedOp`s. | +| `baselineDeltaUSD` (reduction part) | removed. The delta is the sum of the priced operations. | +| `cachesplitSavedUSD`, `keepaliveSavedUSD` | removed. They are cache effects found by `Compute`. | +| `MarkUnique` split in the recorder | removed. The tier comes from the position of a change, not from a unique fraction. | +| `TailChanged`, cold-start attribution | read `Result.Boundary` and `Result.Phase`. | + +Kept, per design §9.1: gates, events, tool inventory, holdout replay and `KeepAliveReport`. + +## 4. Components + +### 4.1 Engine: Operations (W9 step 3) + +- Each component reports an `Operation` for each change it makes. +- `apply.Result` gains `Ops []counterfactual.Operation` and `Thread`. +- Each change has a stable `ChangeID`, derived from the content hash. The offload state is already keyed by + content hash (`components/offload/state.go`). +- Replays are counted only per component today (`Report.Replays`). Each offloader must record a `ChangeID` for + each change. This is new work. It is not a read of existing fields. +- Marker and apply-only changes are built by `apply` (design §4.2 rule 8). +- Sizes are bytes only. The library converts bytes to tokens with the request's ratio. + +### 4.2 Proxy: Tracker (W9 step 4) + +- One `counterfactual.Trackers`, keyed by `thread.Key(session, thread)`. It lives beside `dash.Recorder`'s `lastTail`. +- Per request: + 1. `BeginRequest` with the client body and the client breakpoints. + 2. `Add` for each Operation in `apply.Result.Ops`. + 3. `EndRequest` with the thread, the forwarded body, the breakpoints and `EntryFacts`. + 4. `Compute(usage)` after the response. `Compute` stores the new `ThreadState`. +- A thread that sets `Inherits` starts from a copy of its parent's `ThreadState`. +- `Preview` is used only to price an old request again. It stores nothing. +- `Billing` comes from `counterfactual.BillingFrom`, from the credential the request used. +- Keep-alive: `dashEmitter` converts each `KeepAliveReport` into a `spend` Operation with the library. It prices + the result with `ComputeStandalone`. It keeps the last ping facts of each thread and passes them in + `EntryFacts.LastPing`, `PingRead` and `PingWindow`. A ping does not move `ThreadState`. +- `dashEmitter` implements `OpsEmitter` for a detached model call. +- The proxy checks `MaxCachedIdx` against B\*. It counts each disagreement. + +### 4.3 Proxy: pricing (W9 steps 5 and 6) + +- Each `PricedOp` is converted to dollars with `modelinfo.Price` for the request's model. +- Dollars are stored in `ComponentSavedUSD` and `baselineDeltaUSD`. +- Tokens stay per tier. The proxy never adds tiers into one token total. +- An operation with a reason in `Unpriced` is shown as unpriced. It is not shown as zero. + +## 5. Rollout + +Each step merges alone and leaves the totals correct. + +| # | Change | Verification | +|---|---|---| +| 1 | Engine emits Operations and `ChangeID`s (4.1). | Replay tests: the same `ChangeID` on every turn. | +| 2 | Tracker, keep-alive conversion and `OpsEmitter` (4.2). Nothing reads the output. | The cache cases (design §5.2) pass through the proxy. The B\* disagreement counter is zero on captured sessions. | +| 3 | Shadow run. `Compute` runs on every request. The old figure is stored. The new figure and the difference are logged. | Every logged difference matches a `differs` entry in `counterfactual/opscases/differences.json`. | +| 4 | Switch. The stored figure comes from `Compute`. The recorder reads `Result.Boundary` and `Result.Phase`. | Dashboard totals before and after, on the same captured sessions. | +| 5 | Delete the rules in section 3. | `go mod tidy -diff` in every module. All cases pass. | + +A shadow run still calls `Compute`. Only the use of the figure is shadowed. `ThreadState` must advance. + +## 6. Expected differences + +Each one corrects a known error (W9): + +- The cache-break penalty is lower. The old one used the whole `cache_write` as an upper bound. +- The tier of a change comes from its position, not from a unique fraction of the request. +- The `context_guru_expand` declaration becomes a cost. The old figure did not count it. +- A cache-break penalty starts at the read extent, not at the break. +- A keep-alive window whose pings rescued nothing is a negative figure for its thread. + +## 7. Risks + +- **Library gaps** (`counterfactual/doc.go`, "Not implemented"): + - the `split`, `ttlPremium` and `ttlRescue` effects, so `cachesplit` and head-TTL savings lose their source; + - the order and conservation checks; + - the long-context tier and the minimum cacheable length. + + The shadow run shows how far these gaps move the figures. Step 5 must not delete a rule that a `notCovered` + entry still uses. +- **State loss on restart.** Active Operations are not persisted. Until each change ends, later requests of the + thread miss their saving and fail the sum check. Recovery is open (design §11, question 13). +- **Unbounded active list.** The list of active operations stays bounded only if the end-of-change rule works + (design §11, question 12). +- **`ChangeID` work in step 1** is the largest unknown. It touches every offloader. +- **Tokenizer estimate.** Bytes are converted to tokens with a ratio. Every priced rewrite is `Estimated`. + The accuracy of the ratio is open (design §11, question 3). + +## 8. Testing + +- The `opscases` list is the contract. Every case passes in the library before the proxy uses it. +- The shadow-run difference log is checked against `differences.json`. +- The existing `TestAggregator`-style invariant is kept: a counterfactual never enters spend. +- Dashboard totals are compared before and after on captured sessions at the switch. diff --git a/go.mod b/go.mod index 0ff123bc..2fe77349 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,9 @@ go 1.26.4 // plus the proxy engine the context-guru-proxy binary embeds. require github.com/maximhq/bifrost/core v1.7.0 +// accountant prices context operations. This module's tests import it and its shared test cases. +require github.com/rossoctl/accountant v0.1.1-0.20261103073505-583b9f916ea2 + require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 github.com/alexaandru/go-sitter-forest/c_sharp v1.9.6 diff --git a/go.sum b/go.sum index a7983817..62f104aa 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,12 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rossoctl/accountant v0.1.1-0.20261103052133-c5f558098d3d h1:uZjLQLg2JAm6JsN1sBhzjPaH6Q0yyNjmkJe2vk7BIzQ= +github.com/rossoctl/accountant v0.1.1-0.20261103052133-c5f558098d3d/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261103072919-fc9dde70e622 h1:OQrj6zCScMDgYMK7Lza5nQeHXewAwVc+EUOiPPzoXcs= +github.com/rossoctl/accountant v0.1.1-0.20261103072919-fc9dde70e622/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261103073505-583b9f916ea2 h1:ZM6txJq6SoYNWeJHI4syFTHLPhkxSQebCnZLSm7Lkvg= +github.com/rossoctl/accountant v0.1.1-0.20261103073505-583b9f916ea2/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/internal/modelinfo/cachefrac.go b/internal/modelinfo/cachefrac.go index 702b8d75..eee4adf8 100644 --- a/internal/modelinfo/cachefrac.go +++ b/internal/modelinfo/cachefrac.go @@ -44,8 +44,9 @@ func OpenAI30MinuteCache(provider, model string) bool { // Gemini do not charge a per-token cache-write premium. // // Why that mattered even though those providers never report cache_write: Event.Price uses -// CacheWrite for cost_usd, where a zero cache_write count makes the rate irrelevant, AND -// baselineDeltaUSD uses it to price SAVED tokens, where the count is our own and the rate +// CacheWrite for cost_usd, where a zero cache_write count makes the rate irrelevant, AND the +// legacy savings rule (now the read-time estimate of old rows, Event.ComponentSavedUSD) used it +// to price SAVED tokens, where the count is our own and the rate // is applied regardless of what the request billed. So the fabricated premium reached the // savings figure on traffic that could never have paid it. // diff --git a/internal/modelinfo/cachefrac_test.go b/internal/modelinfo/cachefrac_test.go index cd779fce..8f795afc 100644 --- a/internal/modelinfo/cachefrac_test.go +++ b/internal/modelinfo/cachefrac_test.go @@ -62,7 +62,7 @@ func TestFabricatedPremiumDoesNotReachNonAnthropicSavings(t *testing.T) { if got, want := claude/gpt, 1.25; got != want { t.Fatalf("premium ratio = %v, want %v", got, want) } - // 100k tokens of savings, the shape baselineDeltaUSD prices. + // 100k tokens of savings, the shape the legacy baseline rule priced. if over := 100_000 * (input*1.25 - gpt); over <= 0 { t.Fatalf("expected the old flat multiple to over-report, got %v", over) } diff --git a/internal/modelinfo/modelinfo.go b/internal/modelinfo/modelinfo.go index 07089086..e84d9409 100644 --- a/internal/modelinfo/modelinfo.go +++ b/internal/modelinfo/modelinfo.go @@ -135,6 +135,7 @@ func (p Price) Cost(fresh, cacheRead, cacheWrite, output int64) float64 { // bookkeeping (proxy/keepalive.go's record1) price the SAME request shape identically instead // of each inlining this formula — the two had drifted apart once already (see issue #382's // review) when the keep-alive mechanism was decoupled from dash's event type. +// dash.tierUSD (dash/accountant_price.go) mirrors the 1h rule (2x Input); change both together. func (p Price) CostWithCacheWrite1h(fresh, cacheRead, cacheWrite, output, cacheWrite1h int64) float64 { cost := p.Cost(fresh, cacheRead, cacheWrite, output) if cacheWrite1h > 0 { diff --git a/keepalive/keeper.go b/keepalive/keeper.go index 9c60a0a3..5aa79640 100644 --- a/keepalive/keeper.go +++ b/keepalive/keeper.go @@ -68,6 +68,11 @@ type kaEntry struct { // over the production window, that one change takes X=280 from **+$164.46 to −$241.99**, // with ping cost tripling to $408.85. startedAt time.Time + // gapStart is the start of the last REAL upstream request on this entry, which is the start of + // the current idle gap. Set once when a real request is recorded (the same instant as + // startedAt then); NEVER moved by a ping, a manual press or a substitute call, unlike + // startedAt. Every ping of one gap reports it as KeepAliveReport.Window. + gapStart time.Time // st is the tenant's state store, held so a successful ping can record that it refreshed the // provider's entry — see apply.RecordCacheTouch and issue #243. Held for the same interval as // the masked body below, and for the same reason: the entry exists exactly between one request @@ -157,8 +162,8 @@ type kaEntry struct { // spent is what this session's pings have cost so far. Reported, never a cap — see // Policy.MaxUSDPerPing for why the guard is per ping. spent float64 - // refreshed is what the last ping read from cache. It is the ceiling on what the next - // real request may credit to this mechanism — see dash.Event.keepaliveSavedUSD. + // refreshed is what the last ping read from cache, reported on the next real request's + // row. The keep-alive's credit is the accountant library's rescue. refreshed int64 // stopped means this session will not be pinged again: a ping wrote instead of reading, // or the upstream refused in a way that repeating cannot fix. @@ -932,7 +937,7 @@ func (k *Keeper) Record(r Record) { emitter = components.NopEmitter{} } e := &kaEntry{ - key: r.Key, tenant: r.Key.Tenant, session: session, client: r.Client, thread: threadID, startedAt: startedAt, + key: r.Key, tenant: r.Key.Tenant, session: session, client: r.Client, thread: threadID, startedAt: startedAt, gapStart: startedAt, cred: r.Cred, touchAlias: r.TouchAlias, body: owned, endpoint: r.Endpoint, st: r.Store, @@ -1314,7 +1319,7 @@ func (k *Keeper) Sweep(now time.Time) int { due = append(due, pingJob{e: e, raw: raw, hdr: e.hdr.Clone(), key: e.key, ref: e.ref, route: e.route, provider: e.provider, model: e.model, tenant: e.tenant, session: e.session, thread: e.thread, st: e.st, emitter: e.emitter, touchAlias: e.touchAlias, - ping: e.pings + 1, extra: true, prevStartedAt: e.startedAt}) + ping: e.pings + 1, extra: true, prevStartedAt: e.startedAt, gapStart: e.gapStart}) continue } if e.stopped || e.pings >= e.pol.MaxPings || !e.pingable() { @@ -1360,7 +1365,7 @@ func (k *Keeper) Sweep(now time.Time) int { due = append(due, pingJob{e: e, raw: raw, hdr: e.hdr.Clone(), key: e.key, ref: e.ref, route: e.route, provider: e.provider, model: e.model, tenant: e.tenant, session: e.session, thread: e.thread, st: e.st, emitter: e.emitter, touchAlias: e.touchAlias, - ping: e.pings, prevStartedAt: prevStartedAt}) + ping: e.pings, prevStartedAt: prevStartedAt, gapStart: e.gapStart}) } k.mu.Unlock() @@ -1412,6 +1417,9 @@ type pingJob struct { // until this ping expires" instead of "how close was this ENTRY to expiring" — answering a // different, always-nearly-full-TTL question. prevStartedAt time.Time + // gapStart is e.gapStart when the job was created: the idle gap this ping serves, reported as + // KeepAliveReport.Window. Unlike prevStartedAt it is the same for every ping of the gap. + gapStart time.Time } // fire sends one ping and accounts for it. Always fails open and quietly: the agent is not @@ -1745,6 +1753,7 @@ func (k *Keeper) record1(j pingJob, u Usage, status int, ms float64, startedAt t // column that is blank because nobody filled it in is the same defect at a smaller // scale. MaxTokens: pingOutputBudget(route) + j.vary%pingVary, + Window: j.gapStart, }) } return cost diff --git a/keepalive/manual.go b/keepalive/manual.go index 1af353cb..b2755fa1 100644 --- a/keepalive/manual.go +++ b/keepalive/manual.go @@ -110,7 +110,9 @@ func (k *Keeper) PingNow(r ManualRequest) (ManualResult, error) { mask(raw) job := pingJob{e: e, raw: raw, hdr: e.hdr.Clone(), key: e.key, ref: e.ref, route: e.route, provider: e.provider, model: e.model, tenant: e.tenant, session: e.session, - thread: e.thread, st: e.st, emitter: e.emitter, touchAlias: e.touchAlias} + thread: e.thread, st: e.st, emitter: e.emitter, touchAlias: e.touchAlias, + // The gap this press serves began at the last real request; a press never moves it. + gapStart: e.gapStart} endpoint, prefix, agent, preset := e.endpoint, e.prefix, e.agent, e.preset k.mu.Unlock() defer zero(raw) @@ -187,6 +189,7 @@ func (k *Keeper) PingNow(r ManualRequest) (ManualResult, error) { Agent: agent, Preset: preset, StopReason: u.StopReason, FreshInput: u.FreshInput, CacheWrite1h: u.CacheWrite1h, MaxTokens: pingOutputBudget(job.route) + job.vary%pingVary, Manual: true, Strategy: ManualStrategyID, RewriteUSD: res.AvoidedUSD, + Window: job.gapStart, }) } return res, nil diff --git a/keepalive/manual_test.go b/keepalive/manual_test.go index 9390ecd7..0ec7ec4d 100644 --- a/keepalive/manual_test.go +++ b/keepalive/manual_test.go @@ -266,6 +266,79 @@ func TestPingStatesAgreeWithPingNowNotHeld(t *testing.T) { } } +// The Window of every ping of one idle gap is the start of the last REAL request, whichever path +// sent the ping and however often it has moved the entry's own clock. +func TestPingReportsCarryTheWindowTheyServed(t *testing.T) { + k, sink, clock := manualKeeper(t) + gap := clock.now() + clock.advance(100 * time.Second) + if _, err := k.PingNow(ManualRequest{Key: sess1, MinGap: 30 * time.Second}); err != nil { + t.Fatal(err) + } + if n := k.Sweep(clock.advance(300 * time.Second)); n != 1 { + t.Fatalf("scheduled ping fired %d times", n) + } + if n := k.Sweep(clock.advance(300 * time.Second)); n != 1 { + t.Fatalf("second scheduled ping fired %d times", n) + } + reps := sink.all() + if len(reps) != 3 { + t.Fatalf("reports = %d, want 3", len(reps)) + } + if gap.IsZero() { + t.Fatal("test clock is zero") + } + for i, r := range reps { + if !r.Window.Equal(gap) { + t.Errorf("report %d (manual=%v) Window = %v, want the real request's start %v", i, r.Manual, r.Window, gap) + } + } +} + +// A new real request opens a new gap: the next ping's Window is that request's start. +func TestPingWindowFollowsTheNewestRealRequest(t *testing.T) { + k, sink, clock := manualKeeper(t) + first := clock.now() + if n := k.Sweep(clock.advance(281 * time.Second)); n != 1 { + t.Fatalf("fired %d", n) + } + second := clock.advance(10 * time.Second) + recordWithSink(k, sink, sess1, second) + if n := k.Sweep(clock.advance(281 * time.Second)); n != 1 { + t.Fatalf("fired %d", n) + } + reps := sink.all() + if len(reps) != 2 || !reps[0].Window.Equal(first) || !reps[1].Window.Equal(second) || reps[0].Window.Equal(reps[1].Window) { + t.Fatalf("windows %+v", reps) + } +} + +// A manual-only hold never moves its clock on a press, and its Window stays the real request's start. +func TestManualHoldPressKeepsTheGapWindow(t *testing.T) { + k, fd, clock := testKeeper(t) + fd.setPrices(proxyPricer{fixedPrice{modelinfo.Price{Input: 3e-6, Output: 15e-6, CacheRead: 3e-7, CacheWrite: 3.75e-6}}}) + sink := &reportSink{} + at := clock.now() + pol := offPolicy + pol.ManualHold = true + for _, d := range []time.Duration{-time.Second, 0} { + k.Record(Record{Key: sess1, StartedAt: at.Add(d), Body: []byte(kaBody), Provider: "anthropic", + Route: "/v1/messages", Status: http.StatusOK, Usage: Usage{CacheRead: 48576, CacheWrite: 1200}, + UsageOK: true, Policy: pol, Header: kaHeader(), Emitter: sink, Hold: fakeOf(k).hold(sess1)}) + } + clock.advance(60 * time.Second) + for i := 0; i < 2; i++ { + if _, err := k.PingNow(ManualRequest{Key: sess1, MinGap: time.Second}); err != nil { + t.Fatal(err) + } + clock.advance(60 * time.Second) + } + reps := sink.all() + if len(reps) != 2 || !reps[0].Window.Equal(at) || !reps[1].Window.Equal(at) { + t.Fatalf("windows %+v, want both %v", reps, at) + } +} + // #771: a press on a STOPPED thread runs the checks the scheduled ping runs right before it sends. A tripped // route or a dead credential refuses it and reports "unavailable"; once the mark clears the thread is // pingable again while its copy is held. A thread stopped by a miss or a refused ping is not blocked. diff --git a/keepalive/substitute.go b/keepalive/substitute.go index 4ba4d8bc..693edd88 100644 --- a/keepalive/substitute.go +++ b/keepalive/substitute.go @@ -282,6 +282,7 @@ func (k *Keeper) recordSubstitutePing(j pingJob, model string, u Usage, ms float // an ordinary ping would have, so reporting the ping's own budget here would misdescribe // what was actually requested. MaxTokens: 0, + Window: j.gapStart, }) } return cost diff --git a/keepalive/substitute_test.go b/keepalive/substitute_test.go index 514166c3..4647cfab 100644 --- a/keepalive/substitute_test.go +++ b/keepalive/substitute_test.go @@ -642,3 +642,24 @@ func TestKeepAliveSummaryCallThatDoesNotCommitIsDeferred(t *testing.T) { "(in=281, one ordinary detail row); got in=%d calls=%v", in, calls) } } + +// A summary-substitute ping reports the same gap Window an ordinary ping does. +func TestSubstitutePingCarriesTheGapWindow(t *testing.T) { + k, _, clock := testKeeper(t) + st := store.NewMemory(store.Options{MaxEntries: 400}) + kaCandidate(t, "sess-1", st, &fakeSummaryModel{out: "x"}) + sink := &reportSink{} + at := clock.now() + recordWithSink(k, sink, sess1, at) + if n := k.Sweep(clock.advance(281 * time.Second)); n != 1 { + t.Fatalf("fired %d", n) + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && len(sink.all()) == 0 { + time.Sleep(time.Millisecond) + } + reps := sink.all() + if len(reps) != 1 || !reps[0].Window.Equal(at) { + t.Fatalf("reports %+v, want one with Window %v", reps, at) + } +} diff --git a/keepalive/sweeptextturn_test.go b/keepalive/sweeptextturn_test.go index 9a3ccacb..eecf9c72 100644 --- a/keepalive/sweeptextturn_test.go +++ b/keepalive/sweeptextturn_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/rossoctl/context-guru/internal/modelinfo" "github.com/rossoctl/context-guru/store" ) @@ -214,3 +215,33 @@ func TestGuardNoPlannedPingIsReplacedAfterATextTurn(t *testing.T) { } } } + +// The request that takes the extra slot reports the idle gap it served, like a planned ping does. +// A zero Window is not converted to a spend operation by the host, so the sweep's cost would never +// reach the thread's next request. +func TestTheExtraSlotReportCarriesTheGapWindow(t *testing.T) { + k, fs, clock := testKeeper(t) + fs.setPrices(proxyPricer{fixedPrice{modelinfo.Price{Input: 3e-6, Output: 15e-6, CacheRead: 3e-7, CacheWrite: 3.75e-6}}}) + sink := &reportSink{} + gap := clock.now() + recordWithSink(k, sink, sess1, gap) + asker := &sweepAsker{cacheRead: 48576} + registerSweep(t, store.NewMemory(store.Options{MaxEntries: 400}), asker, "claude-sonnet-5", sweepConversation(3)) + k.Sweep(clock.advance(281 * time.Second)) + k.Sweep(clock.advance(281 * time.Second)) + waitPings(t, k, 2) + k.Sweep(clock.advance(250 * time.Second)) + waitSweepOutcome(t, k, fs) + if asker.calls.Load() != 1 { + t.Fatalf("sweep asks = %d, want 1 (the extra slot)", asker.calls.Load()) + } + reps := sink.all() + if len(reps) != 3 { + t.Fatalf("reports = %d, want 2 planned pings and the extra request", len(reps)) + } + for i, r := range reps { + if !r.Window.Equal(gap) { + t.Errorf("report %d Window = %v, want the real request's start %v", i, r.Window, gap) + } + } +} diff --git a/mkdocs.yml b/mkdocs.yml index 0cfb7e10..faa0eba4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -168,6 +168,8 @@ nav: - Rule-based and learned TTL arms, scored: results/kv-ttl-predictor-arms.md - TTL predictors compared, pooled and per tenant: results/kv-ttl-predictor-comparison.md - "Strategies vs Overview: reconciling keep-alive savings": results/kv-ttl-savings-reconciliation.md + - "Accountant: where cg's saving differs from the expected figure": accountant/cg-differences.md + - "Accountant: root causes of the differences": accountant/cg-differences-root-causes.md - "The keep-alive reuse model: features, fit, and what it is worth": results/kv-cache-keepalive-reusemodel.md - "SWE-bench Verified: per-arm detail": results/swebench-arms.md - "Terminal-Bench 2.0": diff --git a/proxy/accountant_envelope.go b/proxy/accountant_envelope.go new file mode 100644 index 00000000..668d6eec --- /dev/null +++ b/proxy/accountant_envelope.go @@ -0,0 +1,283 @@ +package proxy + +import ( + "hash/fnv" + "strconv" + + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/expand" + "github.com/rossoctl/context-guru/internal/adjudicate" +) + +// The envelope is the part of a request no pipeline component sees: the tools array, the system +// array, and the cache marks. The proxy changes it in several places (apply's toolschema, +// toolfilter, cachesplit, headttl and tail mark; expand.Inject and adjudicate.Inject after the +// pipeline), and none of them reports an operation. The library's sum check then fails on the +// tools and system sections, and the event books no saving. +// +// envelopeOps reports each change from the two bodies instead: the client body and the body that +// went upstream. A reporter that diffs bodies cannot miss a rewrite that forgot to report itself, +// which is why it is one function here and not a report in each of the seven places. +// +// The forms are the shared cases' (accountant test/cases): +// - a declaration the forwarded body added is a rewrite at its forwarded index, SizeBefore 0 +// (rewrite/expand-first-injection/break-once/5m); +// - declarations it removed are one rewrite with a span for each, at the client indexes, SizeAfter +// 0 (rewrite/four-operations/hot/5m); +// - a declaration both bodies carry with different bytes is a rewrite in place +// (rewrite/toolschema-mid-session/break-then-saving/5m); +// - a system block split in two is one rewrite on the client block, its SizeAfter the bytes of +// both halves (split/delivered/stable-half-read), with a cg:cachesplit: ChangeID, the prefix +// the library recognises a split by; +// - a system block whose bytes changed in place (a ttl label, a moved mark) is a rewrite in place. +// +// Every ChangeID is derived from what changed, never from the turn, so the same change gets the +// same ChangeID on every request and the library's "a reported operation replaces the active one +// with its ChangeID" keeps it applied once. + +// envelopeOwner names the operator of a tools change. expand and adjudicate inject a declaration +// under a name only they use; every other tools change is the envelope rewrite of apply, which the +// pipeline's own report attributes to toolschema (a rewrite in place) or toolfilter (a removal). +func envelopeOwner(name string, added, removed bool) string { + switch { + case added && name == expand.ToolName: + return "expand" + case added && name == adjudicate.ToolName: + return "adjudicate" + case removed: + return "toolfilter" + default: + return "toolschema" + } +} + +// envelopeOps lists the operations that explain the tools and system sections' difference between +// the client body and the forwarded body. It returns nil when both bodies have no messages array +// (accBodies gives an empty Body for them). +func envelopeOps(client, forwarded []byte) []accountant.Operation { + if !gjson.GetBytes(client, "messages").IsArray() || !gjson.GetBytes(forwarded, "messages").IsArray() { + return nil + } + ops := toolsOps(gjson.GetBytes(client, "tools").Array(), gjson.GetBytes(forwarded, "tools").Array()) + return append(ops, systemOps(systemBlocks(client), systemBlocks(forwarded))...) +} + +// toolsOps matches declarations by name. An unnamed declaration (an Anthropic server tool carries a +// type, and a name too, but a malformed one may not) is matched by its position in the unnamed ones. +// A name that repeats is matched by its occurrence: the second "x" of the client body is the second +// "x" of the forwarded body, so an append or an in-place rewrite keeps matching the bytes. A removal +// among repeated names cannot be placed (the declarations are indistinguishable, so which one went is +// unknown), and the reporter returns nil: the sum check then sends the event to the no-saving +// fallback, as for any change it cannot explain. +func toolsOps(cl, fw []gjson.Result) []accountant.Operation { + repeated := false + key := func(arr []gjson.Result) ([]string, map[string]int) { + keys, at := make([]string, len(arr)), map[string]int{} + seen := map[string]int{} + for i, d := range arr { + base := d.Get("name").String() + if base == "" { + base = "\x00" // unnamed declarations share one counter + } + // The first occurrence keeps the bare name, so the owner check and the ChangeID hash see + // the name as before; a repeat gets its occurrence number. + k := base + if n := seen[base]; n > 0 { + k = base + "\x00" + strconv.Itoa(n) + } + seen[base]++ + repeated = repeated || seen[base] > 1 + keys[i] = k + at[k] = i + } + return keys, at + } + ck, cat := key(cl) + fk, fat := key(fw) + var ops []accountant.Operation + var removed []accountant.Span + removedBytes, removedHash := 0, fnv.New64a() + for i, k := range ck { + if _, ok := fat[k]; ok { + continue + } + n := len(cl[i].Raw) + removed = append(removed, accountant.Span{Section: accountant.SectionTools, From: i, To: i, + SizeBefore: accountant.Size{Bytes: n}, SizeAfter: accountant.Size{Bytes: 0}}) + removedBytes += n + removedHash.Write([]byte(k)) + removedHash.Write([]byte{0}) + } + if len(removed) > 0 && repeated { + return nil + } + if len(removed) > 0 { + op := envelopeOp("toolfilter", accountant.Position{Section: accountant.SectionTools, Index: removed[0].From}, + removedBytes, 0, "decl-removed:"+accHex(removedHash.Sum64())) + if len(removed) > 1 { + op.Spans = removed + } + ops = append(ops, op) + } + for i, k := range fk { + j, inClient := cat[k] + switch { + case !inClient: + ops = append(ops, envelopeOp(envelopeOwner(k, true, false), + accountant.Position{Section: accountant.SectionTools, Index: i}, 0, len(fw[i].Raw), "decl-added:"+hashOf(k))) + case cl[j].Raw != fw[i].Raw: + at := accountant.Position{Section: accountant.SectionTools, Index: j} + // apply rewrites the declaration first and relabels its mark to 1h last, so the rewrite ends + // at the forwarded bytes without the ttl. Reported as one operation, the relabel would carry + // no headttl ChangeID and the library would not book its premium. + if mid, ok := withoutHeadTTL(cl[j].Raw, fw[i].Raw); ok { + if mid != cl[j].Raw { + ops = append(ops, envelopeOp(envelopeOwner(k, false, false), at, len(cl[j].Raw), len(mid), + "decl-rewritten:"+hashOf(k))) + } + ops = append(ops, envelopeOpID("headttl", at, len(mid), len(fw[i].Raw), "headttl:"+hashOf(k))) + continue + } + ops = append(ops, envelopeOp(envelopeOwner(k, false, false), at, len(cl[j].Raw), len(fw[i].Raw), + "decl-rewritten:"+hashOf(k))) + } + } + return ops +} + +// systemBlocks is the system section as accBodies hashes it: a string system prompt is one block. +func systemBlocks(raw []byte) []string { + s := gjson.GetBytes(raw, "system") + switch { + case s.IsArray(): + var out []string + s.ForEach(func(_, v gjson.Result) bool { out = append(out, v.Raw); return true }) + return out + case s.Exists(): + return []string{s.Raw} + } + return nil +} + +// systemOps explains the system section. The forwarded body has the client's blocks, one of them +// possibly split in two (cachesplit), and any block possibly relabelled in place (headttl). The +// split is found as the one client block whose text equals the text of two adjacent forwarded +// blocks; nothing else in the proxy changes the system block count. +func systemOps(cl, fw []string) []accountant.Operation { + var ops []accountant.Operation + split := -1 + if len(fw) == len(cl)+1 { + for i := range cl { + if i+1 < len(fw) && textOf(cl[i]) != "" && textOf(cl[i]) == textOf(fw[i])+textOf(fw[i+1]) { + split = i + break + } + } + if split < 0 { + return nil // a count change this reporter cannot explain: the sum check says so + } + } else if len(fw) != len(cl) { + return nil + } + for i, j := 0, 0; i < len(cl); i, j = i+1, j+1 { + pos := accountant.Position{Section: accountant.SectionSystem, Index: i} + if i == split { + // apply relabels the head to 1h before it splits the block, so the split starts from the + // relabelled bytes. The relabel is its own operation: the library books the 1h premium + // only through a headttl: ChangeID. + before := cl[i] + if relabelled := withHeadTTL(cl[i], fw[j]); relabelled != cl[i] { + ops = append(ops, envelopeOpID("headttl", pos, len(cl[i]), len(relabelled), "headttl:"+hashOf(textOf(cl[i])))) + before = relabelled + } + ops = append(ops, envelopeOpID("cachesplit", pos, len(before), len(fw[j])+len(fw[j+1]), + "cg:cachesplit:"+hashOf(textOf(cl[i])))) + j++ + continue + } + if cl[i] != fw[j] { + // The library finds the head by the prefix "headttl:" (design §4.4), not "cg:". Its state is per + // thread, so the thread part of the spelled-out form is not needed to keep two heads apart. + ops = append(ops, envelopeOpID("headttl", pos, len(cl[i]), len(fw[j]), "headttl:"+hashOf(textOf(cl[i])))) + } + } + return ops +} + +// withHeadTTL is client with its cache mark relabelled to 1h, when forwarded carries a 1h mark and the +// client's did not: the bytes upgradeHeadTTL leaves a block before anything else rewrites it. It is +// client itself when there is no relabel to explain. +// +// It relies on upgradeHeadTTL (apply/headttl.go) only ever relabelling a mark the client already +// sent. If that function learns to ADD a mark where the client had none, headTTLRelabelled is false +// for it, the relabel is absorbed into the split or the rewrite again, and the library never books +// the 1h premium. Change both together. +func withHeadTTL(client, forwarded string) string { + if !headTTLRelabelled(client, forwarded) { + return client + } + out, err := sjson.Set(client, "cache_control.ttl", "1h") + if err != nil { + return client + } + return out +} + +// withoutHeadTTL splits a tools declaration's change at the relabel: mid is the forwarded declaration +// without the 1h label, the bytes the declaration's own rewrite produced. ok is false when the +// forwarded declaration carries no relabel. When relabelling the client's declaration gives the +// forwarded one exactly, mid is the client's, and the change is the relabel alone. +func withoutHeadTTL(client, forwarded string) (mid string, ok bool) { + if !headTTLRelabelled(client, forwarded) { + return "", false + } + if withHeadTTL(client, forwarded) == forwarded { + return client, true + } + mid, err := sjson.Delete(forwarded, "cache_control.ttl") + if err != nil { + return "", false + } + return mid, true +} + +// headTTLRelabelled is true when the forwarded mark asks for 1h and the client's did not. +func headTTLRelabelled(client, forwarded string) bool { + cc := gjson.Get(client, "cache_control") + return cc.IsObject() && cc.Get("ttl").String() != "1h" && gjson.Get(forwarded, "cache_control.ttl").String() == "1h" +} + +func textOf(raw string) string { + r := gjson.Parse(raw) + if r.Type == gjson.String { + return r.String() + } + return r.Get("text").String() +} + +// envelopeOp is one rewrite with the ChangeID cg::. +func envelopeOp(component string, at accountant.Position, before, after int, key string) accountant.Operation { + return envelopeOpID(component, at, before, after, "cg:"+component+":"+key) +} + +func envelopeOpID(component string, at accountant.Position, before, after int, changeID string) accountant.Operation { + return accountant.Operation{ + Kind: accountant.KindRewrite, + Operator: "context-guru." + component, + At: at, + SizeBefore: accountant.Size{Bytes: before}, + SizeAfter: accountant.Size{Bytes: after}, + ChangeID: changeID, + } +} + +func hashOf(s string) string { + h := fnv.New64a() + h.Write([]byte(s)) + return accHex(h.Sum64()) +} + +func accHex(v uint64) string { return strconv.FormatUint(v, 16) } diff --git a/proxy/accountant_envelope_test.go b/proxy/accountant_envelope_test.go new file mode 100644 index 00000000..07c89d3a --- /dev/null +++ b/proxy/accountant_envelope_test.go @@ -0,0 +1,230 @@ +package proxy + +import ( + "testing" + "time" + + "github.com/rossoctl/accountant" +) + +// sectionDelta is the client-minus-forwarded bytes of one section, the sum check's "bodies" side. +func sectionDelta(cl, fw []byte, s accountant.Section) int { + sec := map[accountant.Section]int{accountant.SectionTools: 0, accountant.SectionSystem: 1, accountant.SectionMessages: 2}[s] + bytes := func(b accountant.Body) int { + end := len(b.Pos) + if sec < 2 { + end = b.Start[sec+1] + } + n := 0 + for _, p := range b.Pos[b.Start[sec]:end] { + n += int(p.Bytes) + } + return n + } + return bytes(accBodies(cl)) - bytes(accBodies(fw)) +} + +// opsDelta is the operations' before-minus-after bytes of one section, the sum check's "ops" side. +func opsDelta(ops []accountant.Operation, s accountant.Section) int { + n := 0 + for _, o := range ops { + if o.At.Section == s { + n += o.SizeBefore.Bytes - o.SizeAfter.Bytes + } + } + return n +} + +func TestEnvelopeOpsExplainEveryByte(t *testing.T) { + const msgs = `"messages":[{"role":"user","content":"hi"}]` + tA := `{"name":"a","description":"x","input_schema":{"type":"object","$schema":"s"}}` + tB := `{"name":"b","description":"y"}` + tC := `{"name":"c","description":"z"}` + big := `{"type":"text","text":"STABLE\ngit","cache_control":{"type":"ephemeral"}}` + tM := `{"name":"m","description":"x","input_schema":{"type":"object","$schema":"s"},"cache_control":{"type":"ephemeral"}}` + for _, tc := range []struct { + name string + cl, fw string + operators []string + changeIDs []string + wantNoOps bool + }{ + {name: "no change", + cl: `{"tools":[` + tA + `],` + msgs + `}`, fw: `{"tools":[` + tA + `],` + msgs + `}`, wantNoOps: true}, + {name: "expand injected", + cl: `{"tools":[` + tA + `],` + msgs + `}`, + fw: `{"tools":[` + tA + `,{"name":"context_guru_expand","description":"e"}],` + msgs + `}`, + operators: []string{"context-guru.expand"}}, + {name: "two removed, one rewritten in place", + cl: `{"tools":[` + tA + `,` + tB + `,` + tC + `],` + msgs + `}`, + fw: `{"tools":[{"name":"a","description":"x","input_schema":{"type":"object"}}],` + msgs + `}`, + operators: []string{"context-guru.toolfilter", "context-guru.toolschema"}}, + {name: "system split", + cl: `{"system":[` + big + `],` + msgs + `}`, + fw: `{"system":[{"type":"text","text":"STABLE\n","cache_control":{"type":"ephemeral"}},{"type":"text","text":"git"}],` + msgs + `}`, + operators: []string{"context-guru.cachesplit"}}, + {name: "system relabelled to 1h", + cl: `{"system":[` + big + `],` + msgs + `}`, + fw: `{"system":[{"type":"text","text":"STABLE\ngit","cache_control":{"type":"ephemeral","ttl":"1h"}}],` + msgs + `}`, + operators: []string{"context-guru.headttl"}}, + // apply relabels the head first and splits second, so the forwarded first half carries the 1h mark. + {name: "system relabelled to 1h and split", + cl: `{"system":[` + big + `],` + msgs + `}`, + fw: `{"system":[{"type":"text","text":"STABLE\n","cache_control":{"type":"ephemeral","ttl":"1h"}},{"type":"text","text":"git"}],` + msgs + `}`, + operators: []string{"context-guru.headttl", "context-guru.cachesplit"}}, + {name: "tool rewritten and relabelled to 1h", + cl: `{"tools":[` + tM + `],` + msgs + `}`, + fw: `{"tools":[{"name":"m","description":"x","cache_control":{"type":"ephemeral","ttl":"1h"}}],` + msgs + `}`, + operators: []string{"context-guru.toolschema", "context-guru.headttl"}}, + {name: "tool relabelled to 1h only", + cl: `{"tools":[` + tM + `],` + msgs + `}`, + fw: `{"tools":[{"name":"m","description":"x","input_schema":{"type":"object","$schema":"s"},"cache_control":{"type":"ephemeral","ttl":"1h"}}],` + msgs + `}`, + operators: []string{"context-guru.headttl"}}, + } { + t.Run(tc.name, func(t *testing.T) { + ops := envelopeOps([]byte(tc.cl), []byte(tc.fw)) + if tc.wantNoOps { + if len(ops) != 0 { + t.Fatalf("ops %+v, want none", ops) + } + return + } + var got []string + for _, o := range ops { + got = append(got, o.Operator) + if o.ChangeID == "" { + t.Errorf("op %+v has no ChangeID", o) + } + } + if len(got) != len(tc.operators) { + t.Fatalf("operators %v, want %v", got, tc.operators) + } + for i := range got { + if got[i] != tc.operators[i] { + t.Fatalf("operators %v, want %v", got, tc.operators) + } + } + for _, s := range []accountant.Section{accountant.SectionTools, accountant.SectionSystem} { + if o, b := opsDelta(ops, s), sectionDelta([]byte(tc.cl), []byte(tc.fw), s); o != b { + t.Errorf("section %s: ops %d bytes, bodies %d bytes", s, o, b) + } + } + }) + } +} + +// The ChangeID is derived from the change, never from the turn: the same injection on the next turn +// gets the same ChangeID, so the library replaces the active op and prices it once. +func TestEnvelopeChangeIDIsStableAcrossTurns(t *testing.T) { + cl1 := `{"tools":[{"name":"a"}],"messages":[{"role":"user","content":"hi"}]}` + cl2 := `{"tools":[{"name":"a"}],"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"ok"}]}` + inj := func(cl string) string { + return cl[:len(`{"tools":[{"name":"a"}`)] + `,{"name":"context_guru_expand"}` + cl[len(`{"tools":[{"name":"a"}`):] + } + a := envelopeOps([]byte(cl1), []byte(inj(cl1))) + b := envelopeOps([]byte(cl2), []byte(inj(cl2))) + if len(a) != 1 || len(b) != 1 || a[0].ChangeID != b[0].ChangeID { + t.Fatalf("turn 1 %+v, turn 2 %+v: want one op each with the same ChangeID", a, b) + } +} + +// A removal among repeated names cannot be placed, so nothing is reported and the sum check gates the +// event: the first "x" removed and the second rewritten used to come out as the second removed and the +// first rewritten, with the right sizes at the wrong positions. +func TestEnvelopeReportsNothingForARemovalAmongRepeatedNames(t *testing.T) { + const msgs = `"messages":[{"role":"user","content":"hi"}]` + x1, x2, y := `{"name":"x","description":"one"}`, `{"name":"x","description":"two!!"}`, `{"name":"y"}` + cl := []byte(`{"tools":[` + x1 + `,` + x2 + `,` + y + `],` + msgs + `}`) + fw := []byte(`{"tools":[{"name":"x","description":"two!!!"},` + y + `],` + msgs + `}`) + if ops := envelopeOps(cl, fw); len(ops) != 0 { + t.Fatalf("ops %+v, want none for an ambiguous removal", ops) + } +} + +// A tool name that repeats still matches by occurrence, so the operations explain every byte. Before, +// the last duplicate won the name and the operations did not match the bodies (the sum check failed). +func TestEnvelopeOpsExplainDuplicateToolNames(t *testing.T) { + const msgs = `"messages":[{"role":"user","content":"hi"}]` + x1 := `{"name":"x","description":"first"}` + x2 := `{"name":"x","description":"second, longer"}` + for _, tc := range []struct{ name, cl, fw string }{ + {"first duplicate rewritten", `{"tools":[` + x1 + `,` + x2 + `],` + msgs + `}`, + `{"tools":[{"name":"x","description":"1"},` + x2 + `],` + msgs + `}`}, + {"a duplicate added", `{"tools":[` + x1 + `],` + msgs + `}`, `{"tools":[` + x1 + `,` + x2 + `],` + msgs + `}`}, + } { + t.Run(tc.name, func(t *testing.T) { + ops := envelopeOps([]byte(tc.cl), []byte(tc.fw)) + if got, want := opsDelta(ops, accountant.SectionTools), sectionDelta([]byte(tc.cl), []byte(tc.fw), accountant.SectionTools); got != want { + t.Fatalf("ops explain %d tools bytes, bodies differ by %d (ops %+v)", got, want, ops) + } + }) + } +} + +// The head-TTL op reaches the library's premium effect through the real reporter: the proxy relabels +// the client's 5m system breakpoint to 1h, the response bills the write at 1h, and the library books +// the premium on the headttl change. The library finds the head by the prefix "headttl:"; a ChangeID +// under any other prefix is priced as nothing. +func TestHeadTTLRelabelBooksThePremium(t *testing.T) { + sys := func(ttl string) string { + return `"system":[{"type":"text","text":"HEAD","cache_control":{"type":"ephemeral"` + ttl + `}}]` + } + const msgs = `"messages":[{"role":"user","content":"hi"}]` + client := []byte(`{` + sys("") + `,` + msgs + `}`) + fwd := []byte(`{` + sys(`,"ttl":"1h"`) + `,` + msgs + `}`) + ops := envelopeOps(client, fwd) + if len(ops) != 1 || ops[0].Operator != "context-guru.headttl" { + t.Fatalf("ops %+v, want one headttl op", ops) + } + st := newSavingsTracker() + req := st.begin(accountant.RequestMeta{RequestID: "r1", Session: "s", Model: "m", At: time.Unix(1_700_000_000, 0)}, client) + st.end(req, accountant.Thread{ID: "main"}, fwd, ops, accountant.EntryFacts{TTL: time.Hour, Lifetime: accountant.LifetimeExact}) + res := st.compute(req, Usage{CacheWrite: 2000, CacheWrite1h: 2000}) + for _, e := range res.CacheEffects { + if e.Kind == accountant.EffectTTLPremium { + return + } + } + t.Fatalf("no ttlPremium effect for ChangeID %q: %+v", ops[0].ChangeID, res.CacheEffects) +} + +// A head relabelled to 1h INSIDE a block that cachesplit then splits still books the premium: the +// relabel is reported as its own headttl operation, not absorbed into the split. +func TestHeadTTLRelabelOfASplitBlockBooksThePremium(t *testing.T) { + const msgs = `"messages":[{"role":"user","content":"hi"}]` + client := []byte(`{"system":[{"type":"text","text":"STABLE\ngit","cache_control":{"type":"ephemeral"}}],` + msgs + `}`) + fwd := []byte(`{"system":[{"type":"text","text":"STABLE\n","cache_control":{"type":"ephemeral","ttl":"1h"}},{"type":"text","text":"git"}],` + msgs + `}`) + ops := envelopeOps(client, fwd) + assertPremiumBooked(t, client, fwd, ops) +} + +// The same for a tool declaration that was rewritten and then relabelled. +func TestHeadTTLRelabelOfARewrittenToolBooksThePremium(t *testing.T) { + const msgs = `"messages":[{"role":"user","content":"hi"}]` + client := []byte(`{"tools":[{"name":"m","description":"x","input_schema":{"type":"object","$schema":"s"},"cache_control":{"type":"ephemeral"}}],` + msgs + `}`) + fwd := []byte(`{"tools":[{"name":"m","description":"x","cache_control":{"type":"ephemeral","ttl":"1h"}}],` + msgs + `}`) + ops := envelopeOps(client, fwd) + assertPremiumBooked(t, client, fwd, ops) +} + +func assertPremiumBooked(t *testing.T, client, fwd []byte, ops []accountant.Operation) { + t.Helper() + for _, s := range []accountant.Section{accountant.SectionTools, accountant.SectionSystem} { + if o, b := opsDelta(ops, s), sectionDelta(client, fwd, s); o != b { + t.Errorf("section %s: ops %d bytes, bodies %d bytes (ops %+v)", s, o, b, ops) + } + } + st := newSavingsTracker() + req := st.begin(accountant.RequestMeta{RequestID: "r1", Session: "s", Model: "m", At: time.Unix(1_700_000_000, 0)}, client) + st.end(req, accountant.Thread{ID: "main"}, fwd, ops, accountant.EntryFacts{TTL: time.Hour, Lifetime: accountant.LifetimeExact}) + res := st.compute(req, Usage{CacheWrite: 2000, CacheWrite1h: 2000}) + if len(res.Checks) != 0 { + t.Errorf("checks failed: %+v", res.Checks) + } + for _, e := range res.CacheEffects { + if e.Kind == accountant.EffectTTLPremium { + return + } + } + t.Fatalf("no ttlPremium effect: ops %+v, effects %+v", ops, res.CacheEffects) +} diff --git a/proxy/accountant_import_test.go b/proxy/accountant_import_test.go new file mode 100644 index 00000000..f399c2be --- /dev/null +++ b/proxy/accountant_import_test.go @@ -0,0 +1,13 @@ +package proxy + +import ( + "testing" + + "github.com/rossoctl/accountant" +) + +func TestAccountantImportable(t *testing.T) { + if accountant.NewTrackers() == nil { + t.Fatal("nil Trackers") + } +} diff --git a/proxy/accountant_inject_test.go b/proxy/accountant_inject_test.go new file mode 100644 index 00000000..ca179f97 --- /dev/null +++ b/proxy/accountant_inject_test.go @@ -0,0 +1,104 @@ +package proxy + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/tidwall/gjson" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/dash" + "github.com/rossoctl/context-guru/expand" + "github.com/rossoctl/context-guru/internal/adjudicate" +) + +// M4: the savings record ends with the body that goes upstream, AFTER expand.Inject and +// adjudicate.Inject appended their tool declarations: the provider bills those bytes, so the +// library's bytes-to-tokens ratio must count them. Asserted by comparing the Forwarded body the +// library was given with accBodies of the bytes the upstream received, and its breakpoints with the +// upstream body's cache_control marks. +func TestSavingsRecordEndsWithTheInjectedForwardedBody(t *testing.T) { + var fwd atomic.Value + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + fwd.Store(b) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"m1","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],`+ + `"stop_reason":"end_turn","usage":{"input_tokens":20,"output_tokens":1,"cache_creation_input_tokens":50}}`) + })) + defer up.Close() + h, rec, tap := accHandlerYAML(t, up.URL, "pipeline: [extract, extract_llm_sweep]\n") + var got atomic.Pointer[accountant.Request] + h.savings.onEnd = func(r *accountant.Request) { got.Store(r) } + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + body := `{"model":"claude-sonnet-5","max_tokens":64,` + + `"system":[{"type":"text","text":"sys","cache_control":{"type":"ephemeral"}}],` + + `"tools":[{"name":"read_file","description":"read a file","input_schema":{"type":"object"}}],` + + `"messages":[{"role":"user","content":"go"},` + + `{"role":"assistant","content":[{"type":"tool_use","id":"tu_1","name":"read_file","input":{}}]},` + + `{"role":"user","content":[{"type":"tool_result","tool_use_id":"tu_1","content":` + + fmt.Sprintf("%q", bigToolOutput()) + `,"cache_control":{"type":"ephemeral","ttl":"1h"}}]}]}` + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-context-guru-session", "sess-inject") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + sent, _ := fwd.Load().([]byte) + if !expand.HasTool("anthropic", sent) || !adjudicate.HasTool("anthropic", sent) { + t.Fatalf("precondition: both tools must be injected: %s", gjson.GetBytes(sent, "tools")) + } + r := got.Load() + if r == nil { + t.Fatal("the savings record never ended") + } + if want := accBodies(sent); fmt.Sprint(r.Forwarded) != fmt.Sprint(want) { + t.Fatalf("Forwarded given to the library differs from the body sent upstream:\n got %+v\nwant %+v", r.Forwarded, want) + } + if len(r.Forwarded.Pos) == 0 || r.Forwarded.Start[1] != 3 { + t.Fatalf("the forwarded tools section must hold the client tool and the two injected ones: %+v", r.Forwarded.Start) + } + if fmt.Sprint(r.Breakpoints) != fmt.Sprint(accBreakpoints(sent)) || len(r.Breakpoints) == 0 { + t.Fatalf("breakpoints %+v, want %+v", r.Breakpoints, accBreakpoints(sent)) + } + // The injected declarations report no operation of their own; envelopeOps finds them in the body + // diff and reports each as an added declaration. The sum check passes, the override applies, and + // each injection is a cost on a row of its own. + waitForRows(t, rec, 1) + e := tap.all()[0] + res, _ := e.AccountantResult() + for _, c := range res.Checks { + if c.Check == "sum" { + t.Fatalf("sum check failed: %+v", c) + } + } + if fb := e.AccountantFallback(); fb != "" { + t.Fatalf("fell back: %q", fb) + } + for _, name := range []string{"expand", "adjudicate"} { + var row *dash.CompRow + for i := range e.Components { + if e.Components[i].Component == name { + row = &e.Components[i] + } + } + if row == nil || row.Kind != dash.KindEnvelope || row.SavedUSD >= 0 { + t.Fatalf("%s: row %+v, want an envelope row with a cost", name, row) + } + } + var sum float64 + for _, c := range e.Components { + sum += c.SavedUSD + } + if d := sum - (e.BaselineCostUSD - e.CostUSD); d > 1e-12 || d < -1e-12 { + t.Fatalf("rows sum %v, baseline delta %v", sum, e.BaselineCostUSD-e.CostUSD) + } +} diff --git a/proxy/accountant_ping.go b/proxy/accountant_ping.go new file mode 100644 index 00000000..393f98cd --- /dev/null +++ b/proxy/accountant_ping.go @@ -0,0 +1,101 @@ +package proxy + +import ( + "sync" + "time" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/components" +) + +// pingStoreMax bounds PingStore. It is keyed by session and thread, so without a bound a +// long-running proxy would hold one entry per thread that was ever pinged and never asked again. +const pingStoreMax = 4096 + +// PingStore remembers, per session and thread, the facts of the latest keep-alive ping until the +// thread's next real request takes them. The same instance is given to the ping emitter (writer) +// and to the Handler (reader) through Options.Pings. A nil *PingStore is a no-op. +type PingStore struct { + mu sync.Mutex + m map[string]accountant.PingFacts +} + +// NewPingStore returns an empty store holding at most pingStoreMax threads. +func NewPingStore() *PingStore { return &PingStore{m: map[string]accountant.PingFacts{}} } + +// put keeps the latest facts for key; a second ping overwrites the first. When the store is full +// and key is new, an arbitrary other entry is evicted (map order), which only costs the evicted +// thread its LastPing gap on its next request. +func (s *PingStore) put(key string, f accountant.PingFacts) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.m[key]; !ok && len(s.m) >= pingStoreMax { + for k := range s.m { + delete(s.m, k) + break + } + } + s.m[key] = f +} + +// peek returns the facts for key and keeps them. A request reads them before it is sent; it takes +// them (drop) only once it is priced, so a request that books nothing leaves them for the next one. +func (s *PingStore) peek(key string) (accountant.PingFacts, bool) { + if s == nil { + return accountant.PingFacts{}, false + } + s.mu.Lock() + defer s.mu.Unlock() + f, ok := s.m[key] + return f, ok +} + +// take is peek and drop in one step. +func (s *PingStore) take(key string) (accountant.PingFacts, bool) { + f, ok := s.peek(key) + s.drop(key, f.PingWindow) + return f, ok +} + +// drop deletes the facts for key: they belong to the next priced request only. window is the +// PingWindow the request read; facts with another window are a newer ping, put after the request +// read, and stay for the next request. An empty window drops whatever is there. +func (s *PingStore) drop(key, window string) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if f, ok := s.m[key]; ok && (window == "" || f.PingWindow == window) { + delete(s.m, key) + } +} + +// pingFromReport maps a keeper report to the library's Ping. r.CostUSD is deliberately not +// carried: the host prices the ping with its own rates. +func pingFromReport(r components.KeepAliveReport) accountant.Ping { + return accountant.Ping{ + Session: r.Session, ThreadID: r.Thread, Model: r.Model, Host: r.Route, + Usage: accountant.CallUsage{Usage: accountant.Usage{Input: int(r.FreshInput), CacheRead: int(r.CacheRead), + CacheWrite: int(r.CacheWrite), CacheWrite1h: int(r.CacheWrite1h)}, Output: int(r.Output)}, + At: time.UnixMilli(r.TS), Window: r.Window, + } +} + +// PingSavings prices one keep-alive ping as a spend operation and stores its facts in st for the +// thread's next real request. ok is false, and nothing is stored, for a ping that is not +// converted: no Window (its ChangeID would be shared by every such ping), a status outside +// 200-299, or no usage (it billed nothing). Manual and sweep-substitute pings are converted. +func PingSavings(r components.KeepAliveReport, st *PingStore) (accountant.Result, bool) { + if r.Window.IsZero() || r.Status < 200 || r.Status > 299 || + r.FreshInput+r.CacheRead+r.CacheWrite+r.CacheWrite1h+r.Output == 0 { + return accountant.Result{}, false + } + ops, facts := accountant.PingOps(pingFromReport(r), accountant.Billing{Mode: accountant.BillingAPI}) + res := accountant.ComputeStandalone(ops) + st.put(accPingKey(r.Session, accThreadID(r.Thread)), facts) + return res, true +} diff --git a/proxy/accountant_ping_test.go b/proxy/accountant_ping_test.go new file mode 100644 index 00000000..8cf3e556 --- /dev/null +++ b/proxy/accountant_ping_test.go @@ -0,0 +1,302 @@ +package proxy + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/dash" + "github.com/rossoctl/context-guru/internal/thread" +) + +var pingAt = time.Date(2026, 10, 28, 12, 0, 0, 0, time.UTC) + +func okReport() components.KeepAliveReport { + return components.KeepAliveReport{Session: "s", Thread: "", Model: "m", Route: "h", Status: 200, + CacheRead: 5000, CacheWrite: 300, CacheWrite1h: 100, FreshInput: 10, Output: 1, + CostUSD: 9.99, TS: pingAt.UnixMilli(), Window: pingAt.Add(-4 * time.Minute)} +} + +func TestPingFromReportMapsEveryField(t *testing.T) { + p := pingFromReport(okReport()) + want := accountant.Ping{Session: "s", ThreadID: "", Model: "m", Host: "h", + Usage: accountant.CallUsage{Usage: accountant.Usage{Input: 10, CacheRead: 5000, CacheWrite: 300, CacheWrite1h: 100}, Output: 1}, + At: time.UnixMilli(pingAt.UnixMilli()), Window: pingAt.Add(-4 * time.Minute)} + if p != want { + t.Fatalf("got %+v want %+v", p, want) + } + if p.Usage.CostUSD != 0 { + t.Fatal("the host prices a ping with its own rates") + } +} + +func TestPingSavingsPricesAndStoresFacts(t *testing.T) { + st := NewPingStore() + res, ok := PingSavings(okReport(), st) + if !ok || len(res.Ops) != 1 || res.Ops[0].Call == nil { + t.Fatalf("ok=%v res=%+v", ok, res) + } + f, ok := st.take(accPingKey("s", "main")) + if !ok || f.PingWindow == "" || !f.LastPing.Equal(time.UnixMilli(pingAt.UnixMilli())) { + t.Fatalf("facts %+v ok=%v", f, ok) + } +} + +func TestPingSavingsSkipRules(t *testing.T) { + cases := map[string]func(*components.KeepAliveReport){ + "zero window": func(r *components.KeepAliveReport) { r.Window = time.Time{} }, + "status 500": func(r *components.KeepAliveReport) { r.Status = 500 }, + "status 0": func(r *components.KeepAliveReport) { r.Status = 0 }, + "zero usage": func(r *components.KeepAliveReport) { + r.CacheRead, r.CacheWrite, r.CacheWrite1h, r.FreshInput, r.Output = 0, 0, 0, 0, 0 + }, + } + for name, mut := range cases { + st := NewPingStore() + r := okReport() + mut(&r) + if _, ok := PingSavings(r, st); ok { + t.Errorf("%s: converted", name) + } + if len(st.m) != 0 { + t.Errorf("%s: facts stored", name) + } + } + for name, mut := range map[string]func(*components.KeepAliveReport){ + "manual": func(r *components.KeepAliveReport) { r.Manual = true; r.Strategy = "manual" }, + "sweep": func(r *components.KeepAliveReport) { r.SweepAnswerUSD = 0.5 }, + } { + r := okReport() + mut(&r) + if _, ok := PingSavings(r, NewPingStore()); !ok { + t.Errorf("%s: not converted", name) + } + } +} + +func TestPingSavingsNilStoreIsFine(t *testing.T) { + if _, ok := PingSavings(okReport(), nil); !ok { + t.Fatal("result must still be returned") + } +} + +func TestPingStoreTakeDeletesAndOverwriteKeepsLatest(t *testing.T) { + s := NewPingStore() + s.put("k", accountant.PingFacts{PingWindow: "a"}) + s.put("k", accountant.PingFacts{PingWindow: "b"}) + f, ok := s.take("k") + if !ok || f.PingWindow != "b" { + t.Fatalf("%+v %v", f, ok) + } + if _, ok := s.take("k"); ok { + t.Fatal("take must delete") + } + var nilStore *PingStore + nilStore.put("k", accountant.PingFacts{}) + if _, ok := nilStore.take("k"); ok { + t.Fatal("nil store") + } +} + +func TestPingStoreIsBounded(t *testing.T) { + s := NewPingStore() + for i := 0; i < pingStoreMax+500; i++ { + s.put(fmt.Sprint("k", i), accountant.PingFacts{PingWindow: "w"}) + } + if len(s.m) > pingStoreMax { + t.Fatalf("len %d > %d", len(s.m), pingStoreMax) + } + if _, ok := s.take(fmt.Sprint("k", pingStoreMax+499)); !ok { + t.Fatal("the newest entry must survive") + } +} + +func TestEntryFactsReadsPingFactsUntilDropped(t *testing.T) { + st := NewPingStore() + PingSavings(okReport(), st) + th := accountant.Thread{ID: "main"} + f := entryFacts("", "", nil, st, "s", th) + if f.LastPing.IsZero() || f.PingWindow == "" || f.PingRead != nil { + t.Fatalf("%+v", f) + } + st.drop(accPingKey("s", "main"), "") // the capture spends them once the request is priced + g := entryFacts("", "", nil, st, "s", th) + if !g.LastPing.IsZero() || g.PingWindow != "" { + t.Fatalf("second request got %+v", g) + } + if entryFacts("", "", nil, nil, "s", th).TTL != 5*time.Minute { + t.Fatal("nil store changes nothing") + } +} + +func TestNamedThreadKeyRoundTrips(t *testing.T) { + st := NewPingStore() + r := okReport() + r.Thread = "sub-1" + if _, ok := PingSavings(r, st); !ok { + t.Fatal("not converted") + } + if f := entryFacts("", "", nil, st, "s", accountant.Thread{ID: "main"}); !f.LastPing.IsZero() { + t.Fatal("the main thread must not take a named thread's ping") + } + f := entryFacts("", "", nil, st, "s", accountant.Thread{ID: "sub-1"}) + if f.LastPing.IsZero() || f.PingWindow == "" { + t.Fatalf("%+v", f) + } +} + +// The proxy's keep-alive path credits a rescue end to end: turn 1 caches the body, the keeper's ping +// replays it and reads it, and turn 2 after the TTL but inside the ping's renewal is hot with a +// rescue on the ping's window. The extent the ping read is derived from its usage (accountant +// design §5.2): the proxy passes no PingRead. +func TestThePingRescueIsCreditedFromTheUsage(t *testing.T) { + st, pings := newSavingsTracker(), NewPingStore() + t0 := time.Unix(1_700_000_000, 0) + body := func(n int) []byte { + msgs := `[{"role":"user","content":"` + strings.Repeat("x", 4000) + `","cache_control":{"type":"ephemeral"}}` + for i := 1; i < n; i++ { + msgs += `,{"role":"assistant","content":"ok"},{"role":"user","content":"more","cache_control":{"type":"ephemeral"}}` + } + return []byte(`{"model":"m","system":[{"type":"text","text":"sys"}],"messages":` + msgs + `]}`) + } + turn := func(at time.Time, b []byte, u Usage) accountant.Result { + req := st.begin(accountant.RequestMeta{RequestID: at.String(), Session: "s", Model: "m", At: at}, b) + th := accountant.Thread{ID: "main"} + // The lifetime comes from the production rule, not set by hand: an unknown lifetime would + // never rescue after a gap past the TTL. + f := entryFacts(bschemas.Anthropic, "m", b, pings, "s", th) + if f.Lifetime != accountant.LifetimeExact { + t.Fatalf("lifetime %v, want exact", f.Lifetime) + } + st.end(req, th, b, nil, f) + return st.compute(req, u) + } + b1 := body(1) + turn(t0, b1, Usage{CacheWrite: int64(len(b1))}) + r := okReport() + r.Session, r.Thread, r.TS = "s", "", t0.Add(4*time.Minute).UnixMilli() + r.Window = t0 + r.FreshInput, r.CacheRead, r.CacheWrite, r.Output = 0, int64(len(b1)), 0, 1 + if _, ok := PingSavings(r, pings); !ok { + t.Fatal("the ping was not converted") + } + b2 := body(2) + res := turn(t0.Add(8*time.Minute), b2, Usage{CacheRead: int64(len(b1)), CacheWrite: int64(len(b2) - len(b1))}) + if res.Phase != accountant.PhaseHot { + t.Fatalf("phase %q, want hot: the ping renewed the entry", res.Phase) + } + var rescue *accountant.CacheEffect + for i := range res.CacheEffects { + if res.CacheEffects[i].Kind == accountant.EffectRescue { + rescue = &res.CacheEffects[i] + } + } + if rescue == nil || rescue.Saved.CacheWrite5m == 0 || rescue.ChangeID == "" { + t.Fatalf("effects %+v, want a rescue on the ping's window", res.CacheEffects) + } +} + +// The facts carry the lifetime the cold gate uses, per provider. +func TestEntryFactsTakeTheLifetimeFromTheProvider(t *testing.T) { + th := accountant.Thread{ID: "main"} + if f := entryFacts(bschemas.Anthropic, "claude-sonnet-5", nil, nil, "s", th); f.Lifetime != accountant.LifetimeExact || f.TTL != 5*time.Minute { + t.Fatalf("anthropic: %+v", f) + } + one := []byte(`{"system":[{"type":"text","text":"s","cache_control":{"type":"ephemeral","ttl":"1h"}}],"messages":[]}`) + if f := entryFacts(bschemas.Anthropic, "claude-sonnet-5", one, nil, "s", th); f.TTL != time.Hour { + t.Fatalf("every breakpoint asks 1h: %+v", f) + } + // A 1h head over a 5m tail: the tail expires in five minutes, so the entry's TTL is five. + mixed := []byte(`{"system":[{"type":"text","text":"s","cache_control":{"type":"ephemeral","ttl":"1h"}}],` + + `"messages":[{"role":"user","content":[{"type":"text","text":"t","cache_control":{"type":"ephemeral"}}]}]}`) + if f := entryFacts(bschemas.Anthropic, "claude-sonnet-5", mixed, nil, "s", th); f.TTL != 5*time.Minute { + t.Fatalf("a mixed ask keeps the five-minute TTL: %+v", f) + } + if f := entryFacts("", "m", nil, nil, "s", th); f.Lifetime != accountant.LifetimeUnknown { + t.Fatalf("unknown provider: %+v", f) + } +} + +// A request that books nothing leaves the ping facts for the next one: the facts are read when the +// request ends and spent only when it is priced. +func TestPingFactsSurviveARequestThatBooksNothing(t *testing.T) { + st := NewPingStore() + PingSavings(okReport(), st) + key := accPingKey("s", "main") + if _, ok := st.peek(key); !ok { + t.Fatal("no facts after the ping") + } + // Reading the facts for a request (entryFacts) does not spend them. + entryFacts("", "", nil, st, "s", accountant.Thread{ID: "main"}) + if _, ok := st.peek(key); !ok { + t.Fatal("entryFacts spent the ping facts: a request that books nothing would lose them") + } + st.drop(key, "") + if _, ok := st.peek(key); ok { + t.Fatal("drop kept the facts") + } + st.drop(key, "") // idempotent + var nilStore *PingStore + nilStore.drop(key, "") +} + +// A ping that arrives while a request is in flight is not spent by that request: finish drops only +// the facts the request read, and the newer ping stays for the next one. +func TestDropKeepsANewerPing(t *testing.T) { + st := NewPingStore() + key := accPingKey("s", "main") + st.put(key, accountant.PingFacts{PingWindow: "w1", LastPing: pingAt}) + read, _ := st.peek(key) // the request reads window w1 ... + st.put(key, accountant.PingFacts{PingWindow: "w2", LastPing: pingAt.Add(time.Minute)}) + st.drop(key, read.PingWindow) // ... and is priced after w2 arrived + f, ok := st.peek(key) + if !ok || f.PingWindow != "w2" { + t.Fatalf("the newer ping was dropped with the older one: %+v %v", f, ok) + } + st.drop(key, "w2") + if _, ok := st.peek(key); ok { + t.Fatal("the facts the request read were kept") + } +} + +// The capture spends the ping facts only when it prices the request: finish with no usage (nothing +// computed) leaves them for the next request, finish with usage spends them. A finish that dropped +// them unconditionally, or under another key, fails here. +func TestFinishSpendsThePingFactsOnlyWhenItPrices(t *testing.T) { + body := []byte(`{"model":"m","system":[{"type":"text","text":"sys"}],"messages":[{"role":"user","content":"hi"}]}`) + run := func(usage Usage, usageOK bool) (spent bool) { + rec, err := dash.NewRecorder(dash.Options{DBPath: filepath.Join(t.TempDir(), "d.db"), BatchSize: 1, FlushInterval: time.Millisecond}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { rec.Close() }) + pings := NewPingStore() + PingSavings(okReport(), pings) + key := accPingKey("s", "main") + if _, ok := pings.peek(key); !ok { + t.Fatal("the ping stored nothing") + } + c := &capture{rec: rec, savings: newSavingsTracker(), pings: pings, tenant: "t1", provider: "anthropic", + model: "m", route: "/v1/messages", start: time.Now(), status: 200, trace: apply.Trace{Session: "s"}} + c.accBegin(accountant.RequestMeta{RequestID: "r1", Session: "s", Model: "m", At: time.Now()}, body) + c.accEnd(thread.Result{ID: thread.Primary}, body, nil) + if _, ok := pings.peek(key); !ok { + t.Fatal("accEnd spent the facts before the request was priced") + } + c.finish(usage, usageOK, false, 0, 0) + _, still := pings.peek(key) + return !still + } + if run(Usage{}, false) { + t.Error("a request with no usage spent the ping facts: the next request loses the ping gap") + } + if !run(Usage{FreshInput: 10, CacheWrite: 100}, true) { + t.Error("a priced request left the ping facts for the next one") + } +} diff --git a/proxy/accountant_presets_test.go b/proxy/accountant_presets_test.go new file mode 100644 index 00000000..ae0a8384 --- /dev/null +++ b/proxy/accountant_presets_test.go @@ -0,0 +1,90 @@ +package proxy + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/rossoctl/context-guru/config" +) + +// claudeCodeBody is a Claude-Code-shaped request: 30 tools, a big system block with a git-status tail, +// cache_control marks, and turns tool results. +func claudeCodeBody(turns int, tail string) []byte { + var tools []any + for i := 0; i < 30; i++ { + tools = append(tools, map[string]any{"name": fmt.Sprintf("tool_%02d", i), + "description": strings.Repeat("Does a thing with files. ", 20), + "input_schema": map[string]any{"type": "object", "$schema": "http://json-schema.org/draft-07/schema#", + "properties": map[string]any{"path": map[string]any{"type": "string", "description": "the path", "title": "Path"}}, + "additionalProperties": false}}) + } + sys := []any{ + map[string]any{"type": "text", "text": "You are Claude Code."}, + map[string]any{"type": "text", "text": strings.Repeat("Follow the repo rules carefully. ", 600) + + "\n\ngitStatus: " + tail + "\n", "cache_control": map[string]any{"type": "ephemeral"}}, + } + msgs := []any{map[string]any{"role": "user", "content": "fix the bug"}} + for i := 0; i < turns; i++ { + id := fmt.Sprintf("tu_%d", i) + msgs = append(msgs, + map[string]any{"role": "assistant", "content": []any{map[string]any{"type": "tool_use", "id": id, "name": "tool_01", "input": map[string]any{"path": "a.py"}}}}, + map[string]any{"role": "user", "content": []any{map[string]any{"type": "tool_result", "tool_use_id": id, "content": bigToolOutput()}}}) + } + last := msgs[len(msgs)-1].(map[string]any)["content"].([]any)[0].(map[string]any) + last["cache_control"] = map[string]any{"type": "ephemeral"} + b, _ := json.Marshal(map[string]any{"model": "claude-sonnet-5", "max_tokens": 64, "tools": tools, "system": sys, "messages": msgs}) + return b +} + +// Every shipped preset is priced by accountant on a Claude-Code-shaped session: no request falls +// back to the legacy figures, and the component rows sum to the baseline delta. A new rewrite of the +// envelope that reports no operation makes the sum check fail here. +func TestEveryPresetIsPricedByAccountant(t *testing.T) { + for _, preset := range config.PresetNames() { + t.Run(preset, func(t *testing.T) { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"id":"m","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],`+ + `"stop_reason":"end_turn","usage":{"input_tokens":50,"output_tokens":5,`+ + `"cache_read_input_tokens":%d,"cache_creation_input_tokens":%d}}`, len(b)/8, len(b)/16) + })) + defer up.Close() + h, rec, tap := accHandlerYAML(t, up.URL, "preset: "+preset+"\n") + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + const turns = 4 + for i := 1; i <= turns; i++ { + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", + strings.NewReader(string(claudeCodeBody(i, fmt.Sprintf("M file%d.go", i%2))))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-context-guru-session", "s-"+preset) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + } + waitForRows(t, rec, turns) + for i, e := range tap.all() { + r, _ := e.AccountantResult() + if fb := e.AccountantFallback(); fb != "" { + t.Errorf("request %d fell back: %q, checks %+v, unpriced %+v", i+1, fb, r.Checks, r.Unpriced) + continue + } + var sum float64 + for _, c := range e.Components { + sum += c.SavedUSD + } + if d := sum - (e.BaselineCostUSD - e.CostUSD); d > 1e-12 || d < -1e-12 { + t.Errorf("request %d: rows sum %v, baseline delta %v", i+1, sum, e.BaselineCostUSD-e.CostUSD) + } + } + }) + } +} diff --git a/proxy/accountant_rounds_test.go b/proxy/accountant_rounds_test.go new file mode 100644 index 00000000..34fb919c --- /dev/null +++ b/proxy/accountant_rounds_test.go @@ -0,0 +1,56 @@ +package proxy + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +// M3: after an expand round the usage is the LAST round's, billed on a body the loop grew, while +// the savings record holds the first round's body. No Result may be attached, and PriceMode then +// keeps the legacy figures (fallback reason no_result). +func TestAccountantSkipsARequestWithExpandRounds(t *testing.T) { + var calls atomic.Int64 + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + if calls.Add(1) == 1 { + w.Write([]byte(`{"choices":[{"message":{"role":"assistant","tool_calls":[` + + `{"id":"call_1","type":"function","function":{"name":"context_guru_expand","arguments":"{\"id\":\"HASH\"}"}}` + + `]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":5}}`)) + return + } + w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"done"}}],` + + `"usage":{"prompt_tokens":10,"completion_tokens":5}}`)) + })) + defer up.Close() + h, rec, tap := accHandler(t, up.URL) + h.store.Put("HASH", []byte(strings.Repeat("the original content that had to come back\n", 20))) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + body := `{"model":"gpt-x","tools":[{"type":"function","function":{"name":"read_file"}}],"messages":[` + + `{"role":"user","content":"go"},` + + `{"role":"tool","tool_call_id":"a","content":"` + strings.Repeat("a verbose repeated tool output line\\n", 60) + `"}]}` + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/openai/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-context-guru-session", "sess-rounds") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if n := calls.Load(); n != 2 { + t.Fatalf("precondition: expected an expand round (2 upstream calls), got %d", n) + } + waitForRows(t, rec, 1) + e := tap.all()[0] + if e.HasAccountant() { + t.Fatal("a Result is attached to a request with expand rounds") + } + if fb := e.AccountantFallback(); fb != "no_result" { + t.Fatalf("fallback = %q, want no_result", fb) + } +} diff --git a/proxy/accountant_savings.go b/proxy/accountant_savings.go new file mode 100644 index 00000000..87bfe2d1 --- /dev/null +++ b/proxy/accountant_savings.go @@ -0,0 +1,46 @@ +package proxy + +import ( + "github.com/rossoctl/accountant" +) + +// savingsTracker runs the accountant calls around one request. It holds the one Trackers +// of the proxy: a thread's state lives there between requests. +type savingsTracker struct { + trackers *accountant.Trackers + siblings *siblingStore // each thread's last forwarded body, for EntryFacts.SharedWarm + // onEnd, when set, is called with each record after EndRequest. Tests only. + onEnd func(*accountant.Request) +} + +func newSavingsTracker() *savingsTracker { + return &savingsTracker{trackers: accountant.NewTrackers(), siblings: newSiblingStore()} +} + +// begin opens the request record with the client body, before any component ran. The Trackers bound +// their own memory (accountant design §6.1, Memory), so there is nothing to sweep here. +func (s *savingsTracker) begin(meta accountant.RequestMeta, client []byte) *accountant.Request { + return s.trackers.BeginRequest(meta, accBodies(client), accBreakpoints(client)) +} + +// end records the operations, then the thread and the forwarded body. Every op is added, replays +// included: the library replaces an active change by the reported op with the same ChangeID +// (accountant internal/engine/compute.go, items()), so a change is priced once and unique_change never fires. +// Skipping a replay here would lose the op whenever Compute ends the change on this request. +func (s *savingsTracker) end(req *accountant.Request, th accountant.Thread, forwarded []byte, ops []accountant.Operation, facts accountant.EntryFacts) { + for _, op := range ops { + req.Add(op) + } + fwd, bps := accBodies(forwarded), accBreakpoints(forwarded) + facts.SharedWarm = s.siblings.sharedWarm(req.Meta.Session, th.ID, req.Meta.At, facts.TTL, fwd, bps) + req.EndRequest(th, fwd, bps, facts) + if s.onEnd != nil { + s.onEnd(req) + } +} + +// compute prices the request. It stores the new ThreadState (so a shadow run still advances it). +func (s *savingsTracker) compute(req *accountant.Request, u Usage) accountant.Result { + return req.Compute(accountant.Usage{Input: int(u.FreshInput), CacheRead: int(u.CacheRead), + CacheWrite: int(u.CacheWrite), CacheWrite1h: int(u.CacheWrite1h)}) +} diff --git a/proxy/accountant_savings_body.go b/proxy/accountant_savings_body.go new file mode 100644 index 00000000..83cffc1e --- /dev/null +++ b/proxy/accountant_savings_body.go @@ -0,0 +1,85 @@ +package proxy + +import ( + "github.com/tidwall/gjson" + + "github.com/rossoctl/accountant" +) + +// accBodies hashes a request body in the provider's order: tools, system, messages. A string +// system prompt is one block. Raw JSON is hashed as sent, so two hosts hash the same bytes alike. +// Each element of messages is one position, so Position{messages, i} is the wire message index. +// A body without messages (or with invalid JSON) gives an empty Body. +func accBodies(raw []byte) accountant.Body { + if !gjson.GetBytes(raw, "messages").IsArray() { + return accountant.NewBody(nil, nil, nil) + } + var tools, system, msgs [][]byte + gjson.GetBytes(raw, "tools").ForEach(func(_, v gjson.Result) bool { + tools = append(tools, []byte(v.Raw)) + return true + }) + if s := gjson.GetBytes(raw, "system"); s.IsArray() { + s.ForEach(func(_, v gjson.Result) bool { system = append(system, []byte(v.Raw)); return true }) + } else if s.Exists() { + system = append(system, []byte(s.Raw)) + } + gjson.GetBytes(raw, "messages").ForEach(func(_, v gjson.Result) bool { + msgs = append(msgs, []byte(v.Raw)) + return true + }) + return accountant.NewBody(tools, system, msgs) +} + +// accBreakpoints lists the cache_control marks, as positions in the same space as accBodies, one per +// position. A mark inside a message's content blocks belongs to that message, so an element mark +// and a block mark (or two block marks) can fall on one position: it is listed once, with the +// longer TTL (1h over 5m), because the provider keeps the entry for the longest TTL asked for at +// it. ttl defaults to "5m". +func accBreakpoints(raw []byte) []accountant.Breakpoint { + if !gjson.GetBytes(raw, "messages").IsArray() { + return nil + } + var out []accountant.Breakpoint + at := map[accountant.Position]int{} // position -> index in out + add := func(p accountant.Position, ttl string) { + if i, ok := at[p]; ok { + if ttl == "1h" { + out[i].TTL = "1h" + } + return + } + at[p] = len(out) + out = append(out, accountant.Breakpoint{Pos: p, TTL: ttl}) + } + ttl := func(cc gjson.Result) string { + if cc.Get("ttl").String() == "1h" { + return "1h" + } + return "5m" + } + scan := func(sec accountant.Section, arr gjson.Result) { + i := 0 + arr.ForEach(func(_, el gjson.Result) bool { + if cc := el.Get("cache_control"); cc.Exists() { + add(accountant.Position{Section: sec, Index: i}, ttl(cc)) + } + if content := el.Get("content"); content.IsArray() { + content.ForEach(func(_, blk gjson.Result) bool { + if cc := blk.Get("cache_control"); cc.Exists() { + add(accountant.Position{Section: sec, Index: i}, ttl(cc)) + } + return true + }) + } + i++ + return true + }) + } + scan(accountant.SectionTools, gjson.GetBytes(raw, "tools")) + if s := gjson.GetBytes(raw, "system"); s.IsArray() { + scan(accountant.SectionSystem, s) + } + scan(accountant.SectionMessages, gjson.GetBytes(raw, "messages")) + return out +} diff --git a/proxy/accountant_savings_body_test.go b/proxy/accountant_savings_body_test.go new file mode 100644 index 00000000..1ce4aa61 --- /dev/null +++ b/proxy/accountant_savings_body_test.go @@ -0,0 +1,97 @@ +package proxy + +import ( + "testing" + + "github.com/rossoctl/accountant" +) + +const accSample = `{"tools":[{"name":"a"}],"system":"sys", + "messages":[{"role":"user","content":"hi"}, + {"role":"assistant","content":[{"type":"text","text":"yo","cache_control":{"type":"ephemeral","ttl":"1h"}}]}]}` + +func TestCfBodies_SectionsAndSizes(t *testing.T) { + b := accBodies([]byte(accSample)) + if b.Start != [3]int{0, 1, 2} || len(b.Pos) != 4 { + t.Fatalf("start = %v pos = %d, want [0 1 2] and 4", b.Start, len(b.Pos)) + } + if int(b.Pos[2].Bytes) != len(`{"role":"user","content":"hi"}`) { + t.Fatalf("message 0 bytes = %d", b.Pos[2].Bytes) + } +} + +func TestCfBodies_EqualInputEqualHashes(t *testing.T) { + a, b := accBodies([]byte(accSample)), accBodies([]byte(accSample)) + for i := range a.Pos { + if a.Pos[i] != b.Pos[i] { + t.Fatalf("position %d differs", i) + } + } +} + +func TestCfBodies_SystemArrayOneBlockEach(t *testing.T) { + b := accBodies([]byte(`{"system":[{"type":"text","text":"a"},{"type":"text","text":"b"}],"messages":[{"role":"user","content":"x"}]}`)) + if b.Start != [3]int{0, 0, 2} || len(b.Pos) != 3 { + t.Fatalf("start = %v pos = %d", b.Start, len(b.Pos)) + } +} + +func TestCfBodies_NoMessagesAndInvalidJSON(t *testing.T) { + for _, in := range []string{``, `not json`, `{"model":"x"}`, `{"messages":`} { + b := accBodies([]byte(in)) + if len(b.Pos) != 0 { + t.Fatalf("%q: pos = %d, want 0", in, len(b.Pos)) + } + _ = accBreakpoints([]byte(in)) + } +} + +func TestCfBodies_OpenAIShapeLeavesToolsAndSystemEmpty(t *testing.T) { + b := accBodies([]byte(`{"messages":[{"role":"system","content":"s"},{"role":"user","content":"u"}]}`)) + if b.Start != [3]int{0, 0, 0} || len(b.Pos) != 2 { + t.Fatalf("start = %v pos = %d", b.Start, len(b.Pos)) + } +} + +func TestCfBreakpoints_FindsBlockLevelMarkAndTTL(t *testing.T) { + got := accBreakpoints([]byte(accSample)) + want := []accountant.Breakpoint{{Pos: accountant.Position{Section: accountant.SectionMessages, Index: 1}, TTL: "1h"}} + if len(got) != 1 || got[0] != want[0] { + t.Fatalf("got %+v, want %+v", got, want) + } +} + +func TestCfBreakpoints_AllSectionsInBodyOrder(t *testing.T) { + raw := `{"tools":[{"name":"a","cache_control":{"type":"ephemeral"}}], + "system":[{"type":"text","text":"s","cache_control":{"type":"ephemeral","ttl":"1h"}}], + "messages":[{"role":"user","content":"x","cache_control":{"type":"ephemeral"}}]}` + got := accBreakpoints([]byte(raw)) + want := []accountant.Breakpoint{ + {Pos: accountant.Position{Section: accountant.SectionTools, Index: 0}, TTL: "5m"}, + {Pos: accountant.Position{Section: accountant.SectionSystem, Index: 0}, TTL: "1h"}, + {Pos: accountant.Position{Section: accountant.SectionMessages, Index: 0}, TTL: "5m"}, + } + if len(got) != len(want) { + t.Fatalf("got %+v", got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got[%d] = %+v, want %+v", i, got[i], want[i]) + } + } +} + +// S3: an element mark and a block mark in one message are one breakpoint, with the longer TTL, +// whichever order they come in. +func TestCfBreakpointsOnePerPositionLongerTTL(t *testing.T) { + for _, raw := range []string{ + `{"messages":[{"role":"user","cache_control":{"type":"ephemeral"},"content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral","ttl":"1h"}}]}]}`, + `{"messages":[{"role":"user","cache_control":{"type":"ephemeral","ttl":"1h"},"content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral"}}]}]}`, + `{"messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral"}},{"type":"text","text":"b","cache_control":{"type":"ephemeral","ttl":"1h"}}]}]}`, + } { + bps := accBreakpoints([]byte(raw)) + if len(bps) != 1 || bps[0].TTL != "1h" || bps[0].Pos != (accountant.Position{Section: accountant.SectionMessages, Index: 0}) { + t.Fatalf("%s: breakpoints = %+v, want one 1h at messages[0]", raw, bps) + } + } +} diff --git a/proxy/accountant_savings_test.go b/proxy/accountant_savings_test.go new file mode 100644 index 00000000..f666b8f4 --- /dev/null +++ b/proxy/accountant_savings_test.go @@ -0,0 +1,220 @@ +package proxy + +import ( + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/dash" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +func rewriteOp(change string, at, before, after int) accountant.Operation { + return accountant.Operation{Kind: accountant.KindRewrite, Operator: "context-guru.mask", ChangeID: change, + At: accountant.Position{Section: accountant.SectionMessages, Index: at}, + SizeBefore: accountant.Size{Bytes: before}, SizeAfter: accountant.Size{Bytes: after}} +} + +// msg builds one message element and its byte length as hashed. +func msg(role, text string) (string, int) { + s := fmt.Sprintf(`{"role":%q,"content":%q}`, role, text) + return s, len(s) +} + +func body(msgs ...string) []byte { return []byte(`{"messages":[` + strings.Join(msgs, ",") + `]}`) } + +var accFacts = accountant.EntryFacts{TTL: 5 * time.Minute, Lifetime: accountant.LifetimeExact} + +type accRun struct { + s *savingsTracker + th accountant.Thread +} + +func (c accRun) do(n int, client, fwd []byte, usage Usage, ops ...accountant.Operation) accountant.Result { + meta := accountant.RequestMeta{RequestID: fmt.Sprint("r", n), Session: "s", At: time.Unix(int64(1000+n*10), 0), Billing: accountant.Billing{Mode: accountant.BillingAPI}} + r := c.s.begin(meta, client) + c.s.end(r, c.th, fwd, ops, accFacts) + return c.s.compute(r, usage) +} + +func countChange(res accountant.Result, id string) (n int, saved int) { + for _, o := range res.Ops { + if o.ChangeID == id { + n++ + saved += o.Saved.Input + o.Saved.CacheWrite5m + o.Saved.CacheWrite1h + } + } + return +} + +func TestSavings_ReplayedChangeIsPricedOnce(t *testing.T) { + c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} + big, nBig := msg("user", strings.Repeat("tool result ", 400)) + small, nSmall := msg("user", "[masked]") + m0, _ := msg("user", "first") + m2, _ := msg("assistant", "answer") + client, fwd := body(m0, big, m2), body(m0, small, m2) + usage := Usage{FreshInput: 100, CacheRead: 1000} + const id = "cg:mask:a1" + + // (a) first request: reported and priced. + res := c.do(1, client, fwd, usage, rewriteOp(id, 1, nBig, nSmall)) + if len(res.Checks) != 0 { + t.Fatalf("checks = %+v", res.Checks) + } + if n, _ := countChange(res, id); n != 1 { + t.Fatalf("first request priced %d times, want 1", n) + } + + // (b) the change is reported again while active: it is added, and the library prices it once. + client2 := body(m0, big, m2, mustMsg("user", "next")) + fwd2 := body(m0, small, m2, mustMsg("user", "next")) + req := c.s.begin(accountant.RequestMeta{RequestID: "r2", Session: "s", At: time.Unix(1020, 0), Billing: accountant.Billing{Mode: accountant.BillingAPI}}, client2) + c.s.end(req, c.th, fwd2, []accountant.Operation{rewriteOp(id, 1, nBig, nSmall)}, accFacts) + if len(req.Ops) != 1 { + t.Fatalf("replayed change must be added, req.Ops = %d", len(req.Ops)) + } + res = c.s.compute(req, usage) + if len(res.Checks) != 0 { + t.Fatalf("checks = %+v (unique_change must not fire)", res.Checks) + } + if n, _ := countChange(res, id); n != 1 { + t.Fatalf("replayed change priced %d times, want 1", n) + } + once := 0 + for _, o := range res.Ops { + if o.ChangeID == id { + once++ + } + } + if once != 1 { + t.Fatalf("res.Ops holds the change %d times, want 1", once) + } +} + +func mustMsg(role, text string) string { s, _ := msg(role, text); return s } + +func TestSavings_ReReportsAfterChangeEnded(t *testing.T) { + c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} + big, nBig := msg("user", strings.Repeat("tool result ", 400)) + small, nSmall := msg("user", "[masked]") + m0, _ := msg("user", "first") + m2, _ := msg("assistant", "answer") + usage := Usage{FreshInput: 100, CacheRead: 1000} + const id = "cg:mask:a1" + + c.do(1, body(m0, big, m2), body(m0, small, m2), usage, rewriteOp(id, 1, nBig, nSmall)) + + // The client edits message 0 (at or before the change): the library ends the change. + m0b, _ := msg("user", "first, edited") + c.do(2, body(m0b, big, m2), body(m0b, big, m2), usage) + + // Request 3 reports the same ChangeID again: it is added and priced (no filter). + req := c.s.begin(accountant.RequestMeta{RequestID: "r3", Session: "s", At: time.Unix(1030, 0), Billing: accountant.Billing{Mode: accountant.BillingAPI}}, body(m0b, big, m2)) + c.s.end(req, c.th, body(m0b, small, m2), []accountant.Operation{rewriteOp(id, 1, nBig, nSmall)}, accFacts) + if len(req.Ops) != 1 { + t.Fatalf("ended change must be re-added, req.Ops = %d", len(req.Ops)) + } + res := c.s.compute(req, usage) + if len(res.Checks) != 0 { + t.Fatalf("checks = %+v", res.Checks) + } + n, saved := countChange(res, id) + if n != 1 || saved == 0 { + t.Fatalf("re-reported change: priced %d times, saved %d tokens; want 1 and non-zero", n, saved) + } +} + +func TestSavings_DryRunAndNonRewriteAlwaysAdded(t *testing.T) { + c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} + big, nBig := msg("user", strings.Repeat("x", 2000)) + small, nSmall := msg("user", "y") + usage := Usage{FreshInput: 10} + c.do(1, body(big), body(small), usage, rewriteOp("c1", 0, nBig, nSmall)) + dry := rewriteOp("c1", 0, nBig, nSmall) + dry.DryRun = true + spend := accountant.Operation{Kind: accountant.KindSpend, ChangeID: "c1"} + req := c.s.begin(accountant.RequestMeta{RequestID: "r2", Session: "s", At: time.Unix(1020, 0)}, body(big)) + c.s.end(req, c.th, body(big), []accountant.Operation{dry, spend}, accFacts) + if len(req.Ops) != 2 { + t.Fatalf("req.Ops = %d, want 2", len(req.Ops)) + } +} + +func TestSavings_ConcurrentRequests(t *testing.T) { + s := newSavingsTracker() + var wg sync.WaitGroup + var mu sync.Mutex + var fresh, recomputed int + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + b := body(mustMsg("user", "x")) + r := s.begin(accountant.RequestMeta{RequestID: fmt.Sprint("r", i), Session: "s", At: time.Now(), Billing: accountant.Billing{Mode: accountant.BillingAPI}}, b) + s.end(r, accountant.Thread{ID: "main"}, b, nil, accFacts) + res := s.compute(r, Usage{FreshInput: 5}) + mu.Lock() + defer mu.Unlock() + if res.Recomputed { + recomputed++ + } else { + fresh++ + } + }(i) + } + wg.Wait() + if fresh < 1 { + t.Fatalf("fresh = %d, recomputed = %d; want at least one non-recomputed result", fresh, recomputed) + } +} + +// M2: a ChangeID that changes between turns for the same rewrite (an earlier component started or +// stopped acting) leaves the old change active and reports the new one: the library prices both, +// twice the real saving. The sum check catches it, and PriceMode falls back to the legacy figures +// with reason check_sum, so no inflated override is stored. +func TestSavings_ChangedChangeIDDoubleCountIsGated(t *testing.T) { + c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} + big, nBig := msg("user", strings.Repeat("tool result ", 400)) + small, nSmall := msg("user", "[masked]") + m0, _ := msg("user", "first") + m2, _ := msg("assistant", "answer") + usage := Usage{FreshInput: 100, CacheRead: 1000} + r1 := c.do(1, body(m0, big, m2), body(m0, small, m2), usage, rewriteOp("cg:mask:1:h1", 1, nBig, nSmall)) + if len(r1.Checks) != 0 { + t.Fatalf("first request checks = %+v", r1.Checks) + } + next := mustMsg("user", "next") + r2 := c.do(2, body(m0, big, m2, next), body(m0, small, m2, next), usage, rewriteOp("cg:mask:1:h2", 1, nBig, nSmall)) + n1, _ := countChange(r2, "cg:mask:1:h1") + n2, _ := countChange(r2, "cg:mask:1:h2") + if n1 != 1 || n2 != 1 { + t.Fatalf("precondition (the double count): h1 priced %d, h2 priced %d", n1, n2) + } + sum := false + for _, ch := range r2.Checks { + sum = sum || ch.Check == "sum" + } + if !sum { + t.Fatalf("the sum check must fire on the double count: %+v", r2.Checks) + } + e := &dash.Event{Model: "m", FreshInput: 100, CacheRead: 1000, OutputTokens: 1, TokensBefore: 100000, + Components: []dash.CompRow{{Component: "mask", Acted: true, Mutated: true, SavedGross: 10, SavedUnique: 10}}} + leg := *e + leg.Components = append([]dash.CompRow(nil), e.Components...) + leg.Price(accPrice, true) + e.SetAccountant(r2) + e.PriceMode(accPrice, true) + if fb := e.AccountantFallback(); fb != dash.FallbackCheckSum { + t.Fatalf("fallback = %q, want %q", fb, dash.FallbackCheckSum) + } + if e.Components[0].SavedUSD != leg.Components[0].SavedUSD || e.BaselineCostUSD != leg.BaselineCostUSD { + t.Fatalf("an inflated override was stored: %v/%v vs legacy %v/%v", + e.Components[0].SavedUSD, e.BaselineCostUSD, leg.Components[0].SavedUSD, leg.BaselineCostUSD) + } +} + +var accPrice = modelinfo.Price{Input: 3e-6, CacheWrite: 3.75e-6, CacheRead: 0.3e-6, Output: 15e-6} diff --git a/proxy/accountant_shadow.go b/proxy/accountant_shadow.go new file mode 100644 index 00000000..a911bfc9 --- /dev/null +++ b/proxy/accountant_shadow.go @@ -0,0 +1,27 @@ +package proxy + +import ( + "log/slog" + "strings" + + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/dash" +) + +// logFallback writes the acc_fallback line for an event the accountant library could not price, so it +// books no saving, +// with the reason (dash.Fallback*). +func logFallback(e *dash.Event) { + r, _ := e.AccountantResult() + slog.Info("acc_fallback", "session", e.SessionID, "thread", e.ThreadID, + "reason", e.AccountantFallback(), "checks", checkNames(r), "recomputed", r.Recomputed) +} + +// checkNames lists the names of the failed library checks, comma-separated, in order. +func checkNames(r accountant.Result) string { + n := make([]string, len(r.Checks)) + for i, c := range r.Checks { + n[i] = c.Check + } + return strings.Join(n, ",") +} diff --git a/proxy/accountant_shadow_test.go b/proxy/accountant_shadow_test.go new file mode 100644 index 00000000..623f23c9 --- /dev/null +++ b/proxy/accountant_shadow_test.go @@ -0,0 +1,42 @@ +package proxy + +import ( + "bytes" + "log/slog" + "math" + "strings" + "sync" + "testing" + + "github.com/rossoctl/context-guru/dash" +) + +func near(a, b float64) bool { return math.Abs(a-b) < 1e-9 } + +type syncBuf struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuf) Write(p []byte) (int, error) { s.mu.Lock(); defer s.mu.Unlock(); return s.b.Write(p) } +func (s *syncBuf) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() } + +// A acc_fallback line is logged once for each event left on the legacy rules, with the reason. The +// usage here bills far fewer tokens than the forwarded body holds, so the library's bytes-to-tokens +// ratio saves more than the request had and both events fall back (implausible_size). +func TestFallbackReasonIsLogged(t *testing.T) { + var buf syncBuf + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(prev) }) + evs, _ := accRunUsage(t, 12, 9000, 1500) + if fb := evs[0].AccountantFallback(); fb != dash.FallbackImplausibleSize { + t.Fatalf("second event: fallback %q, want %q", fb, dash.FallbackImplausibleSize) + } + if n := strings.Count(buf.String(), `"msg":"acc_fallback"`); n != 2 { + t.Fatalf("acc_fallback lines = %d, want 2 (one for each event):\n%s", n, buf.String()) + } + if !strings.Contains(buf.String(), `"reason":"implausible_size"`) { + t.Fatalf("acc_fallback lacks the reason:\n%s", buf.String()) + } +} diff --git a/proxy/accountant_shared.go b/proxy/accountant_shared.go new file mode 100644 index 00000000..8892900e --- /dev/null +++ b/proxy/accountant_shared.go @@ -0,0 +1,153 @@ +package proxy + +import ( + "sync" + "time" + + "github.com/rossoctl/accountant" +) + +// siblingsMax bounds siblingStore by session, and siblingThreadsMax by thread within a session. A +// session past the thread cap drops its oldest thread: a sibling that old is past any TTL anyway. +const ( + siblingsMax = 4096 + siblingThreadsMax = 64 +) + +// siblingStore remembers, per session, the last forwarded body each thread sent and when. It gives +// EntryFacts.SharedWarm: the prefix of a request's forwarded body that another thread of the session +// kept warm inside the TTL, which a keep-alive rescue must not credit (accountant R54). It holds the +// library's Body (hashes and sizes only, R2) and the breakpoints, never content. +// +// The store lock is held only to find or create a session; each session has its own lock, so +// requests of different sessions do not wait on each other's scans. +type siblingStore struct { + mu sync.Mutex + m map[string]*siblingSession + tick uint64 // a use counter, for least-recently-used eviction of sessions +} + +type siblingSession struct { + mu sync.Mutex + used uint64 // the store's tick at the last use; read and written under the store lock + threads map[string]siblingEntry +} + +type siblingEntry struct { + at time.Time + fwd accountant.Body + bps []accountant.Breakpoint +} + +func newSiblingStore() *siblingStore { return &siblingStore{m: map[string]*siblingSession{}} } + +// session returns the session's entry, creating it; at the cap it drops the least recently used +// session, which only loses that session's sibling facts. +func (s *siblingStore) session(id string) *siblingSession { + s.mu.Lock() + defer s.mu.Unlock() + s.tick++ + if ss, ok := s.m[id]; ok { + ss.used = s.tick + return ss + } + if len(s.m) >= siblingsMax { + oldest, at := "", uint64(0) + for k, v := range s.m { + if oldest == "" || v.used < at { + oldest, at = k, v.used + } + } + delete(s.m, oldest) + } + ss := &siblingSession{used: s.tick, threads: map[string]siblingEntry{}} + s.m[id] = ss + return ss +} + +// sharedWarm returns the last position of fwd that a sibling of thread kept warm inside ttl before at, +// then records this request as the thread's last. A prefix counts as warm up to the last breakpoint +// the sibling sent inside the prefix both bodies share: the provider caches at breakpoints, so a +// shared prefix with no breakpoint of the sibling's in it was not kept warm. Nil when no sibling did. +func (s *siblingStore) sharedWarm(session, thread string, at time.Time, ttl time.Duration, + fwd accountant.Body, bps []accountant.Breakpoint) *accountant.Position { + if s == nil || session == "" { + return nil + } + ss := s.session(session) + ss.mu.Lock() + defer ss.mu.Unlock() + best := -1 + for id, e := range ss.threads { + if id == thread || at.Sub(e.at) > ttl || e.at.After(at) { + continue + } + common := commonPrefix(e.fwd, fwd) + for _, b := range bps2flat(e.fwd, e.bps) { + if b <= common && b > best { + best = b + } + } + } + if _, ok := ss.threads[thread]; !ok && len(ss.threads) >= siblingThreadsMax { + oldest := "" + for id, e := range ss.threads { + if oldest == "" || e.at.Before(ss.threads[oldest].at) { + oldest = id + } + } + delete(ss.threads, oldest) + } + ss.threads[thread] = siblingEntry{at: at, fwd: fwd, bps: bps} + if best < 0 { + return nil + } + p := positionOf(fwd, best) + return &p +} + +// commonPrefix is the last index at which a and b hold the same prefix, or -1. The hashes are +// cumulative, so equal hashes at an index mean equal content up to it. +func commonPrefix(a, b accountant.Body) int { + i := -1 + for i+1 < len(a.Pos) && i+1 < len(b.Pos) && a.Pos[i+1].Hash == b.Pos[i+1].Hash { + i++ + } + return i +} + +// bps2flat maps breakpoints to flat indexes of b. +func bps2flat(b accountant.Body, bps []accountant.Breakpoint) []int { + var out []int + for _, bp := range bps { + if i, ok := flatOf(b, bp.Pos); ok { + out = append(out, i) + } + } + return out +} + +var sectionOrder = [3]accountant.Section{accountant.SectionTools, accountant.SectionSystem, accountant.SectionMessages} + +func flatOf(b accountant.Body, p accountant.Position) (int, bool) { + for s, sec := range sectionOrder { + if sec != p.Section { + continue + } + end := len(b.Pos) + if s < 2 { + end = b.Start[s+1] + } + i := b.Start[s] + p.Index + return i, p.Index >= 0 && i < end + } + return 0, false +} + +func positionOf(b accountant.Body, i int) accountant.Position { + s := 2 + for s > 0 && i < b.Start[s] { + s-- + } + return accountant.Position{Section: sectionOrder[s], Index: i - b.Start[s]} +} diff --git a/proxy/accountant_shared_test.go b/proxy/accountant_shared_test.go new file mode 100644 index 00000000..39facb06 --- /dev/null +++ b/proxy/accountant_shared_test.go @@ -0,0 +1,112 @@ +package proxy + +import ( + "fmt" + "testing" + "time" + + "github.com/rossoctl/accountant" +) + +func sharedBody(msgs ...string) accountant.Body { + var m [][]byte + for _, s := range msgs { + m = append(m, []byte(s)) + } + return accountant.NewBody(nil, [][]byte{[]byte("sys")}, m) +} + +func bpAt(sec accountant.Section, i int) []accountant.Breakpoint { + return []accountant.Breakpoint{{Pos: accountant.Position{Section: sec, Index: i}, TTL: "5m"}} +} + +func TestSharedWarm(t *testing.T) { + t0 := time.Unix(1_700_000_000, 0) + ttl := 5 * time.Minute + sys0 := accountant.Position{Section: accountant.SectionSystem, Index: 0} + msg1 := accountant.Position{Section: accountant.SectionMessages, Index: 1} + for _, tc := range []struct { + name string + sibAt time.Duration + sibBody accountant.Body + sibBPs []accountant.Breakpoint + sibID string + want *accountant.Position + }{ + {"a sibling shares the system block", time.Minute, sharedBody("x"), bpAt(accountant.SectionSystem, 0), "sub", &sys0}, + {"a sibling shares two messages", time.Minute, sharedBody("a", "b", "z"), bpAt(accountant.SectionMessages, 1), "sub", &msg1}, + {"the sibling's mark is past the shared prefix", time.Minute, sharedBody("a", "z"), bpAt(accountant.SectionMessages, 1), "sub", nil}, + {"the sibling is older than the TTL", -time.Minute, sharedBody("a", "b"), bpAt(accountant.SectionMessages, 1), "sub", nil}, + {"the same thread is not a sibling", time.Minute, sharedBody("a", "b"), bpAt(accountant.SectionMessages, 1), "main", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newSiblingStore() + sibAt := t0.Add(tc.sibAt) + if tc.sibAt < 0 { + sibAt = t0.Add(-ttl - time.Minute) + } + _ = s.sharedWarm("s1", tc.sibID, sibAt, ttl, tc.sibBody, tc.sibBPs) + at := t0.Add(2 * time.Minute) + if tc.sibID == "main" { + // the thread's own earlier request is replaced, not compared + at = t0.Add(2 * time.Minute) + } + got := s.sharedWarm("s1", "main", at, ttl, sharedBody("a", "b", "c"), bpAt(accountant.SectionMessages, 2)) + switch { + case tc.want == nil && got != nil: + t.Fatalf("SharedWarm %+v, want nil", *got) + case tc.want != nil && (got == nil || *got != *tc.want): + t.Fatalf("SharedWarm %v, want %+v", got, *tc.want) + } + }) + } +} + +// Another session's thread is never a sibling. +func TestSharedWarmIsPerSession(t *testing.T) { + s := newSiblingStore() + t0 := time.Unix(1_700_000_000, 0) + _ = s.sharedWarm("other", "sub", t0, 5*time.Minute, sharedBody("a"), bpAt(accountant.SectionSystem, 0)) + if got := s.sharedWarm("s1", "main", t0.Add(time.Minute), 5*time.Minute, sharedBody("a"), nil); got != nil { + t.Fatalf("SharedWarm %+v across sessions", *got) + } +} + +// A session keeps at most siblingThreadsMax threads: the oldest goes first. +func TestSharedWarmBoundsThreadsPerSession(t *testing.T) { + s := newSiblingStore() + t0 := time.Unix(1_700_000_000, 0) + for i := 0; i < siblingThreadsMax+10; i++ { + s.sharedWarm("s1", fmt.Sprintf("t%03d", i), t0.Add(time.Duration(i)*time.Second), time.Hour, sharedBody("a"), nil) + } + ss := s.session("s1") + if n := len(ss.threads); n != siblingThreadsMax { + t.Fatalf("threads = %d, want %d", n, siblingThreadsMax) + } + if _, ok := ss.threads["t000"]; ok { + t.Fatal("the oldest thread was kept") + } + if _, ok := ss.threads[fmt.Sprintf("t%03d", siblingThreadsMax+9)]; !ok { + t.Fatal("the newest thread was dropped") + } +} + +// The store keeps at most siblingsMax sessions and drops the least recently used. +func TestSharedWarmBoundsSessions(t *testing.T) { + s := newSiblingStore() + t0 := time.Unix(1_700_000_000, 0) + for i := 0; i < siblingsMax; i++ { + s.sharedWarm(fmt.Sprintf("s%d", i), "main", t0, time.Hour, sharedBody("a"), nil) + } + s.sharedWarm("s0", "main", t0, time.Hour, sharedBody("a"), nil) // s0 is now the most recent + s.sharedWarm("new", "main", t0, time.Hour, sharedBody("a"), nil) + if len(s.m) != siblingsMax { + t.Fatalf("sessions = %d, want %d", len(s.m), siblingsMax) + } + if _, ok := s.m["s0"]; !ok { + t.Fatal("the recently used session was dropped") + } + if _, ok := s.m["s1"]; ok { + t.Fatal("the least recently used session was kept") + } +} diff --git a/proxy/accountant_wire.go b/proxy/accountant_wire.go new file mode 100644 index 00000000..64f59adc --- /dev/null +++ b/proxy/accountant_wire.go @@ -0,0 +1,88 @@ +package proxy + +import ( + "time" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/accountant" + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/internal/modelinfo" + "github.com/rossoctl/context-guru/internal/thread" +) + +// accThreadID maps the primary thread to "main", the library's name for it. +func accThreadID(id string) string { + if id == thread.Primary { + return "main" + } + return id +} + +// accBegin opens the accountant record with the body the agent sent. Nil-safe. +func (c *capture) accBegin(meta accountant.RequestMeta, client []byte) { + if c == nil || c.savings == nil { + return + } + c.accReq = c.savings.begin(meta, client) + c.accClient = client +} + +// accEnd records the thread, the forwarded body and the operations. A thread that continues +// another by content names it as its parent, but starts with no state of its own: accountant carries +// Thread.Parent and does not read it (design §4.1). Nil-safe; a no-op when accBegin did not open a +// record. +func (c *capture) accEnd(th thread.Result, forwarded []byte, ops []accountant.Operation) { + if c == nil || c.savings == nil || c.accReq == nil { + return + } + t := accountant.Thread{ID: accThreadID(th.ID)} + c.accThread = t.ID + if parent, ok := th.InheritsFrom(); ok { + t.Parent = accThreadID(parent) + } + // The envelope changes report no operation of their own: envelopeOps finds them from the two + // bodies. Appended after the pipeline's ops, because the tools and system positions do not move + // with a message splice. + ops = append(append([]accountant.Operation(nil), ops...), envelopeOps(c.accClient, forwarded)...) + facts := entryFacts(bschemas.ModelProvider(c.provider), c.accReq.Meta.Model, forwarded, c.pings, c.accReq.Meta.Session, t) + c.accPingWindow = facts.PingWindow + c.savings.end(c.accReq, t, forwarded, ops, facts) +} + +// entryFacts builds the facts the library needs about the entry this request wrote. A keep-alive +// ping for the thread since its last request is read from pings (nil = none; the capture drops it +// once the request is priced) and supplies +// LastPing, PingWindow and the ping's usage. PingRead stays nil: the keeper replays the previous +// request's forwarded body, so the library derives the extent the ping read from its usage +// (accountant design §5.2). +// +// The Lifetime kind comes from apply.CacheLifetime, the same rule the cold gate uses: an exact expiry +// for a provider with explicit breakpoints, a minimum for OpenAI's 30-minute entries, else unknown. +// An unknown lifetime cannot prove an expiry, so the library would never rescue after a gap past +// the TTL (design §5.2). The TTL is 1h only when EVERY breakpoint of the forwarded body asks for it, +// as the keeper decides: a 1h head over a 5m tail leaves a tail that expires in five minutes. The +// keeper also needs the billed tier, which is known only after the response; the facts are written +// before it, so a request that asks for 1h and is billed 5m keeps the 1h here. The library then +// treats a gap of 5 to 60 minutes as hot, which can overstate a rescue (an edge case: a gateway that +// downgrades the tier). +func entryFacts(provider bschemas.ModelProvider, model string, forwarded []byte, pings *PingStore, + session string, t accountant.Thread) accountant.EntryFacts { + facts := accountant.EntryFacts{TTL: 5 * time.Minute, Lifetime: accountant.LifetimeUnknown} + // The kind comes from the cold gate's rule. The duration is taken again with the keeper's tier + // (every breakpoint asks 1h): apply.CacheLifetime lengthens it for a single 1h ask. + switch ttl, kind := apply.CacheLifetime(provider, model, forwarded); kind { + case apply.CacheLifetimeExact: + facts.TTL, facts.Lifetime = modelinfo.CacheLifetime(string(provider), model, apply.AllBreakpointsAskExtendedTTL(forwarded)), + accountant.LifetimeExact + case apply.CacheLifetimeMinimum: + facts.TTL, facts.Lifetime = ttl, accountant.LifetimeMinimum + } + if pf, ok := pings.peek(accPingKey(session, t.ID)); ok { + pf.Apply(&facts) + } + return facts +} + +// accPingKey is the key the ping store keeps one thread's ping facts under: the session and the +// thread id as the accountant sees it ("main" for the primary thread). +func accPingKey(session, threadID string) string { return session + "\x00" + threadID } diff --git a/proxy/accountant_wire_test.go b/proxy/accountant_wire_test.go new file mode 100644 index 00000000..0979e91d --- /dev/null +++ b/proxy/accountant_wire_test.go @@ -0,0 +1,238 @@ +package proxy + +import ( + "context" + "fmt" + "io" + "math" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/tidwall/gjson" + + "github.com/rossoctl/context-guru/config" + "github.com/rossoctl/context-guru/dash" + "github.com/rossoctl/context-guru/metrics" + "github.com/rossoctl/context-guru/store" +) + +type accTap struct { + mu sync.Mutex + evs []dash.Event +} + +func (c *accTap) add(e *dash.Event) { c.mu.Lock(); c.evs = append(c.evs, *e); c.mu.Unlock() } +func (c *accTap) all() []dash.Event { + c.mu.Lock() + defer c.mu.Unlock() + return append([]dash.Event(nil), c.evs...) +} + +func accHandler(t *testing.T, up string) (*Handler, *dash.Recorder, *accTap) { + t.Helper() + return accHandlerYAML(t, up, "preset: codesafe\n") +} + +// accHandlerYAML is accHandler with the config document spelled out. +func accHandlerYAML(t *testing.T, up, yaml string) (*Handler, *dash.Recorder, *accTap) { + t.Helper() + cfg, err := config.LoadBytes([]byte(yaml)) + if err != nil { + t.Fatal(err) + } + agg := metrics.NewAggregator() + pipe, err := cfg.Build(agg) + if err != nil { + t.Fatal(err) + } + tap := &accTap{} + rec, err := dash.NewRecorder(dash.Options{DBPath: filepath.Join(t.TempDir(), "d.db"), + BatchSize: 1, FlushInterval: time.Millisecond, Tap: tap.add}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { rec.Close() }) + h := New(pipe, store.NewMemory(store.Options{}), agg, Options{ + AnthropicUpstream: up, OpenAIUpstream: up, Preset: cfg.Preset, + Dashboard: rec, Prices: fixedPricer{}, + }) + t.Cleanup(h.Close) + return h, rec, tap +} + +func accPost(t *testing.T, h *Handler, session string) { + t.Helper() + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", + strings.NewReader(string(anthropicRequest("aws/claude-sonnet-5")))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-context-guru-session", session) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() +} + +func TestNoUsageMeansNoAccountantResult(t *testing.T) { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"error":"boom"}`, http.StatusInternalServerError) + })) + defer up.Close() + h, rec, tap := accHandler(t, up.URL) + accPost(t, h, "sess-err") + waitForRows(t, rec, 1) + for _, e := range tap.all() { + if e.HasAccountant() { + t.Error("an event without usage carries an accountant result") + } + } +} + +// accRunUsage posts two requests through a fake upstream that reports the given prompt usage and +// returns the second recorded event and the byte length of the body the upstream received. +func accRunUsage(t *testing.T, in, cr, cw int) ([]dash.Event, int) { + evs, fwd := accRunUsageBody(t, in, cr, cw) + return evs, len(fwd) +} + +// accRunUsageBody is accRunUsage returning the forwarded body itself. +func accRunUsageBody(t *testing.T, in, cr, cw int) ([]dash.Event, []byte) { + t.Helper() + var fwd atomic.Value + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + fwd.Store(b) + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"id":"msg_1","type":"message","role":"assistant", + "content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn", + "usage":{"input_tokens":%d,"output_tokens":34, + "cache_read_input_tokens":%d,"cache_creation_input_tokens":%d}}`, in, cr, cw) + })) + defer up.Close() + h, rec, tap := accHandler(t, up.URL) + // Two requests: the first has no previous request to classify cache breaks against, so the + // library leaves it with an unpriced "penalty_unknown" and the event keeps its legacy figures. + accPost(t, h, "sess-cf") + accPost(t, h, "sess-cf") + waitForRows(t, rec, 2) + b, _ := fwd.Load().([]byte) + return tap.all()[1:], b +} + +// costOnlyOf prices a copy of a recorded event with Event.Price alone: its cost and no saving, +// which is what an event the accountant library could not price carries. +func costOnlyOf(e dash.Event) dash.Event { + e.Components = append([]dash.CompRow(nil), e.Components...) + p, _ := fixedPricer{}.Price(context.Background(), e.Model) + e.Price(p, true) + return e +} + +func extractRow(t *testing.T, e dash.Event) dash.CompRow { + t.Helper() + for _, c := range e.Components { + if c.Component == "extract" { + return c + } + } + t.Fatal("no extract row") + return dash.CompRow{} +} + +// A usage consistent with the forwarded body: 70 tokens over the ~490 message bytes of the forwarded +// body, ~0.14 token per byte (real traffic is 0.25-0.33; what matters is that the usage bills the +// forwarded body, not a different one). Hand derivation of the extract figure: +// +// positions = the body's tools, system and messages elements (proxy accBodies); here only +// messages, so the envelope (model, max_tokens) is in no position +// fwdBytes = sum of the forwarded message elements' bytes +// removed = client message bytes - forwarded message bytes (one tool_result rewritten) +// usage = 20 input + 50 cache write = 70 tokens +// ratio = 70 tokens / fwdBytes (the library's bytes-to-tokens ratio) +// saved tokens = floor(before*70/fwdBytes) - floor(after*70/fwdBytes), in the Input tier: the +// removed span was never cached (no previous request cached it) +// SavedUSD = saved tokens * 2e-6 (fixedPricer Input rate) +// +// The op sizes are wire bytes, so the derivation is exact up to the two floors (1 token). +// It runs through the real proxy path, and the Result must carry no failed check: before the op +// sizes were wire bytes, the sum check failed on every request (M1). +func TestRealisticUsageMatchesHandDerivation(t *testing.T) { + evs, fwd := accRunUsageBody(t, 20, 0, 50) + if len(evs) != 1 { + t.Fatalf("events = %d", len(evs)) + } + e := evs[0] + r, ok := e.AccountantResult() + if !ok || len(r.Checks) != 0 { + t.Fatalf("Result attached %v, checks %+v: the real path must pass every library check", ok, r.Checks) + } + if fb := e.AccountantFallback(); fb != "" { + t.Fatalf("the override must apply, fell back: %s", fb) + } + msgBytes := func(b []byte) int { + n := 0 + for _, m := range gjson.GetBytes(b, "messages").Array() { + n += len(m.Raw) + } + return n + } + client, fb := msgBytes(anthropicRequest("aws/claude-sonnet-5")), msgBytes(fwd) + want := float64((client-fb)*70/fb) * 2e-6 + got := extractRow(t, e).SavedUSD + if math.Abs(got-want) > 2*2e-6 { + t.Fatalf("extract SavedUSD = %v, hand-derived %v (client %d forwarded %d message bytes)", got, want, client, fb) + } + t.Logf("library %v hand %v", got, want) +} + +// The old fixture: 10,512 prompt tokens for a few-hundred-byte forwarded body. The library's +// token figure then exceeds the request's size, so PriceMode must book no saving, and for that +// reason: the Result has counted operations whose Saved tokens exceed TokensBefore and nothing +// unpriced, so the plausibility bound is what fired, not some earlier gate. +func TestAbsurdUsageBooksNoSaving(t *testing.T) { + cfs, _ := accRunUsage(t, 12, 9000, 1500) + if got := extractRow(t, cfs[0]).SavedUSD; got != 0 || cfs[0].BaselineCostUSD != cfs[0].CostUSD { + t.Fatalf("absurd usage booked a saving: component %v, baseline %v, cost %v", + got, cfs[0].BaselineCostUSD, cfs[0].CostUSD) + } + r, ok := cfs[0].AccountantResult() + if !ok || len(r.Unpriced) != 0 || len(r.Checks) != 0 { + t.Fatalf("Result %v: unpriced %+v checks %+v, want a clean Result", ok, r.Unpriced, r.Checks) + } + saved := 0 + for _, o := range r.Ops { + if !o.DryRun && o.Call == nil { + saved += o.Saved.Input + o.Saved.CacheRead + o.Saved.CacheWrite5m + o.Saved.CacheWrite1h + } + } + if saved <= cfs[0].TokensBefore { + t.Fatalf("counted Saved %d <= TokensBefore %d: the bound would not fire", saved, cfs[0].TokensBefore) + } + if fb := cfs[0].AccountantFallback(); fb != dash.FallbackImplausibleSize { + t.Fatalf("fallback reason %q, want %q", fb, dash.FallbackImplausibleSize) + } +} + +// The tracker needs a dashboard: the Result is attached to the recorded event, so without a +// recorder there is nowhere to put it and nothing is built. +func TestNoDashboardBuildsNoTracker(t *testing.T) { + up := fakeUpstream(t) + defer up.Close() + h := New(nil, store.NewMemory(store.Options{}), nil, Options{AnthropicUpstream: up.URL, Prices: fixedPricer{}}) + defer h.Close() + if h.savings != nil { + t.Error("a handler without a dashboard built a savings tracker") + } + hd, _, _ := accHandler(t, up.URL) + if hd.savings == nil { + t.Error("a handler with a dashboard must build the tracker") + } +} diff --git a/proxy/dash_test.go b/proxy/dash_test.go index 8cdaf2c4..179b04c1 100644 --- a/proxy/dash_test.go +++ b/proxy/dash_test.go @@ -3,6 +3,7 @@ package proxy import ( "context" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -70,16 +71,36 @@ func anthropicRequest(model string) []byte { return b } +// billedUpstream answers like a provider that bills the body it received: 4 bytes a token, the +// first 80% read from cache and the rest written. A saving priced by the accountant library is +// checked against the request's own size (PriceMode's plausibility bound), so a test that asserts +// a saving needs a usage that bills the forwarded body, as a real provider's does. +func billedUpstream(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + n := len(b) / 4 + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"id":"msg_1","type":"message","role":"assistant", + "content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn", + "usage":{"input_tokens":12,"output_tokens":34, + "cache_read_input_tokens":%d,"cache_creation_input_tokens":%d}}`, n*8/10, n-n*8/10-12) + })) +} + // fakeUpstream returns an Anthropic-shaped response with real usage tiers. -func fakeUpstream(t *testing.T) *httptest.Server { +func fakeUpstream(t *testing.T) *httptest.Server { return fakeUpstreamUsage(t, 12, 9000, 1500) } + +// fakeUpstreamUsage is fakeUpstream with the prompt usage the caller chooses. +func fakeUpstreamUsage(t *testing.T, in, cacheRead, cacheWrite int) *httptest.Server { t.Helper() return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.Copy(io.Discard, r.Body) w.Header().Set("Content-Type", "application/json") - w.Write([]byte(`{"id":"msg_1","type":"message","role":"assistant", + fmt.Fprintf(w, `{"id":"msg_1","type":"message","role":"assistant", "content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn", - "usage":{"input_tokens":12,"output_tokens":34, - "cache_read_input_tokens":9000,"cache_creation_input_tokens":1500}}`)) + "usage":{"input_tokens":%d,"output_tokens":34, + "cache_read_input_tokens":%d,"cache_creation_input_tokens":%d}}`, in, cacheRead, cacheWrite) })) } @@ -201,10 +222,9 @@ func TestDashboardCapturesARealRequestEndToEnd(t *testing.T) { t.Fatalf("the pipeline did not compact (%d -> %d); the assertions below would be vacuous", e.TokensBefore, e.TokensAfter) } - if e.BaselineCostUSD <= e.CostUSD { - t.Errorf("baseline %v is not above actual %v despite %d tokens removed", - e.BaselineCostUSD, e.CostUSD, e.TokensBefore-e.TokensAfter) - } + // The saving itself is the accountant library's, checked against a usage that bills the body + // (TestDashboardBooksTheAccountantSaving): this upstream's fixed usage does not, so the library's + // figure fails the plausibility bound and the event books no saving. if e.CGLatencyMs <= 0 || e.UpstreamMs <= 0 { t.Errorf("latency not captured: cg=%v upstream=%v", e.CGLatencyMs, e.UpstreamMs) } @@ -654,3 +674,47 @@ func TestDashboardUnderConcurrentTraffic(t *testing.T) { t.Errorf("%d insert errors", s.Errors) } } + +// A real request through the proxy books the accountant library's saving: the pipeline compacted, +// the upstream billed the forwarded body, and the stored baseline is above the cost. +func TestDashboardBooksTheAccountantSaving(t *testing.T) { + up := billedUpstream(t) + defer up.Close() + h, rec := dashHandler(t, up.URL, dash.Options{}) + h.tracker.SetStartedAt(time.Now().Add(-2 * time.Hour).UnixMilli()) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", + strings.NewReader(string(anthropicRequest("aws/claude-sonnet-5")))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-context-guru-session", "sess-acc") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + waitForRows(t, rec, 1) + page, err := rec.DB().Requests(dash.Filter{}, 0, 10) + if err != nil { + t.Fatal(err) + } + e := page.Requests[0] + if e.TokensBefore <= e.TokensAfter { + t.Fatalf("the pipeline did not compact (%d -> %d)", e.TokensBefore, e.TokensAfter) + } + if e.BaselineCostUSD <= e.CostUSD { + t.Fatalf("baseline %v is not above actual %v despite %d tokens removed", + e.BaselineCostUSD, e.CostUSD, e.TokensBefore-e.TokensAfter) + } + full, err := rec.DB().Request(e.ID, false) + if err != nil { + t.Fatal(err) + } + priced := false + for _, c := range full.Components { + priced = priced || c.PricedBy == dash.PricedByAccountant + } + if !priced { + t.Fatalf("no component row is priced by the accountant library: %+v", full.Components) + } +} diff --git a/proxy/dashcapture.go b/proxy/dashcapture.go index 236da42e..d9487fbb 100644 --- a/proxy/dashcapture.go +++ b/proxy/dashcapture.go @@ -7,6 +7,7 @@ import ( "time" bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/accountant" "github.com/rossoctl/context-guru/apply" "github.com/rossoctl/context-guru/cheapmodel" "github.com/rossoctl/context-guru/components" @@ -118,6 +119,28 @@ type capture struct { // declares no tools, and nil when the dashboard is off. Read on the request path with // the pristine body, like meta; emitted at finish, where the session id is known. inv *dash.Inventory + // savings is the handler's accountant tracker (nil without a dashboard); accReq is this + // request's record (nil when it was skipped); accDone guards a double finish. + savings *savingsTracker + // observe is true when the tenancy runs in observe mode: the trace is then empty by design + // and the accountant figure must not be booked. + observe bool + // pings holds the keep-alive ping facts the next request consumes (Options.Pings; nil = none). + pings *PingStore + accReq *accountant.Request + // accClient is the body the agent sent, kept for accEnd's envelope diff (envelopeOps). + accClient []byte + // accThread is the thread id accEnd gave the library, the second half of the ping store's key. + accThread string + // accPingWindow is the window of the ping facts accEnd read, so finish drops those and not a + // newer ping that arrived while the request was in flight. + accPingWindow string + accDone bool + // rounds is the number of upstream rounds of the expand loop (noteRounds). The usage is the + // LAST round's, billed on a body the expand loop grew, while the savings record holds the + // first round's body: with rounds > 1 the library's bytes-to-tokens ratio would be wrong, so + // the record is not computed. + rounds int } // newCapture starts a capture for one request, or returns nil when the dashboard @@ -141,6 +164,8 @@ func (h *Handler) newCapture(r *http.Request, provider, route string, tn *Tenanc unique: map[string]int{}, waits: h.waits, terminal: h.opts.TerminalTools, + savings: h.savings, pings: h.opts.Pings, + observe: tn.Mode == components.ModeObserve, } } @@ -599,9 +624,8 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content cgCacheWrite, cgCacheRead := c.llm.CacheTotals() cgModel := c.llm.Model() - // Cache attribution, with a cold start treated as the non-failure it is. BEFORE - // pricing, because whether this is the session's first request is an input to a dollar - // figure and not only to a label — see Event.cachesplitSavedUSD. + // Cache attribution, with a cold start treated as the non-failure it is. The verdict is a + // label on the row; the savings are the accountant library's. // // Per THREAD (#423). Keyed on the session alone, a subagent's requests reset the main // thread's idle gap, so the main thread's TTL expiries fell through to prefix_change — on @@ -679,8 +703,8 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content } e.AttributeCache(seenSession, seenModel, age, cacheAttributionTTLMs(e.Provider, e.Model), cause, prevPromptTokens) e.SessionFirst, e.TailChanged = !seenSession, tailChanged - // The raw gap and what the keep-alive did during it. Both feed a dollar figure rather - // than only a label (see Event.keepaliveSavedUSD), so they have to be set before Price. + // The raw gap and what the keep-alive did during it, recorded on the row. The keep-alive's + // saving is the accountant library's rescue, not computed from these. e.SinceLastMs = sinceMs e.KeepAlivePings, e.KeepAliveRefreshed = c.kaPings, c.kaRefreshed e.KeepAliveStrategyID = c.kaStrategy @@ -690,7 +714,25 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content if c.pricer != nil && c.model != "" { price, priced = c.pricer.Price(context.Background(), c.model) } - e.Price(price, usageOK && priced) + // Run the accountant BEFORE the event is priced. Only with usage: without it the thread + // state stays untouched and the event carries no result. Not after several upstream rounds + // either (see rounds): no Result is attached, so PriceMode books no saving + // (acc_fallback reason no_result). The thread state is left as the + // previous request left it; the next request then sees a longer gap, which the library + // handles (a phase it cannot prove is unknown and falls back too). + if c.accReq != nil && usageOK && !c.accDone { + c.accDone = true + if c.rounds <= 1 { + e.SetAccountant(c.savings.compute(c.accReq, usage)) + // The ping facts were read for this request; now that it is priced they are spent. + c.pings.drop(accPingKey(c.accReq.Meta.Session, c.accThread), c.accPingWindow) + } + } + e.SetAccountantContext(c.trace.OpsDropped, c.observe) + e.PriceMode(price, usageOK && priced) + if c.accReq != nil && e.AccountantFallback() != "" { + logFallback(e) + } if cgIn > 0 || cgOut > 0 || cgCacheWrite > 0 || cgCacheRead > 0 { cgPrice, cgPriced := price, priced if c.pricer != nil && cgModel != "" && cgModel != c.model { @@ -789,3 +831,17 @@ func (c *capture) liveFacts(ttfbMs float64) *LiveFacts { return &LiveFacts{Messages: c.trace.Messages, Tools: c.meta.Tools, SystemBlocks: c.meta.SystemBlocks, MaxTokens: c.meta.MaxTokens, Effort: c.meta.ReasoningEffort, Thinking: c.meta.ThinkingMode, CacheTTL: c.trace.CacheTTL, Agent: c.agent, Stream: c.meta.Stream, TTFBMs: ttfbMs, Arrival: c.predArrival} } + +// noteRounds records how many upstream rounds the request took (1 without an expand). Nil-safe. +func (c *capture) noteRounds(n int) { + if c != nil { + c.rounds = n + } +} + +// accDrop forgets the savings record: the pipeline's rewrite did not go upstream. Nil-safe. +func (c *capture) accDrop() { + if c != nil { + c.accReq = nil + } +} diff --git a/proxy/proxy.go b/proxy/proxy.go index e795cf93..9d3940d9 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -28,6 +28,7 @@ import ( "time" bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/accountant" "github.com/rossoctl/context-guru/apply" "github.com/rossoctl/context-guru/cheapmodel" "github.com/rossoctl/context-guru/components" @@ -153,6 +154,11 @@ type Options struct { // its help text and at startup. Record *sessionRecorder + // Pings is the store the keep-alive emitter writes ping facts to and the next request of the + // same thread reads them from (see PingSavings). It must be the same instance on both sides. + // nil = none, and pings then feed no request. + Pings *PingStore + // PipelineFor builds a pipeline for a per-request override on /compact // (?preset=… or x-context-guru-pipeline: a,b,c). nil = overrides ignored, the // handler always uses the configured pipeline. Supplied by main (which holds @@ -267,6 +273,10 @@ type Handler struct { pool *modes.Pool // observeSeq numbers observations so each request enqueues one. observeSeq atomic.Uint64 + // savings is nil without a dashboard recorder: the Result is attached to the recorded event. + savings *savingsTracker + // accSeq numbers requests for the accountant RequestMeta.RequestID. + accSeq atomic.Uint64 // shadow is observe mode's own state store, separate from the live one. Observe must // not write into the live store — a real request would then replay a decision that was // never enforced — but it also cannot simply discard its writes: offloaders FREEZE a @@ -409,6 +419,10 @@ func New(pipe *components.Pipeline, st store.Store, agg *metrics.Aggregator, opt h := &Handler{pipe: pipe, store: st, agg: agg, opts: opts, client: c, tracker: tr, rec: opts.Dashboard, recorder: opts.Record, sent: newSentStash()} + // The tracker needs a dashboard: the Result is attached to the recorded event. + if opts.Dashboard != nil { + h.savings = newSavingsTracker() + } // The restart grace decides by the thread's own last cache refresh — its last request or // a keep-alive ping that read the cache — and not by this process's start (#766). The // recorder holds that time per thread — seeded from the database at startup @@ -1327,6 +1341,9 @@ func (h *Handler) chat(provider bschemas.ModelProvider, static upstream, pick fu if rec := recover(); rec != nil { lg.Error("context-guru: recovered from panic before forward; sending original", "panic", rec) body = orig + // The original goes upstream, so no operation of the savings record is on the + // wire: drop the record, and the event books no saving. + cp.accDrop() } }() // A client that received our own expand tool_use answered it itself, with an @@ -1414,6 +1431,21 @@ func (h *Handler) chat(provider bschemas.ModelProvider, static upstream, pick fu addedMs := float64(added.Microseconds()) / 1000.0 cp.noteCG(addedMs) cp.noteTrace(tr) + // The savings tracker begins HERE, after applyMode, and not before it: the session + // (tr.Session) is known only once the pipeline has run, and BeginRequest only records + // the client body, so opening the record late loses nothing. `orig` is what the agent + // sent. An empty session means the pipeline did not run (bypassed, observe, no + // messages): there are no operations to price, so the request is skipped. + if cp != nil && cp.savings != nil && tr.Session != "" { + cp.accBegin(accountant.RequestMeta{ + RequestID: "r" + strconv.FormatUint(h.accSeq.Add(1), 10), + Session: tr.Session, + Model: gjson.GetBytes(orig, "model").String(), + At: cp.start, + // The proxy has no subscription detection, so every request is billed as api. + Billing: accountant.Billing{Mode: accountant.BillingAPI}, + }, orig) + } // A real request has arrived on this session: cancel any pending keep-alive // (this request refreshes the entry itself, and a ping concurrent with it would // be a second request against the tenant's budget for nothing), and pick up what @@ -1536,6 +1568,11 @@ func (h *Handler) chat(provider bschemas.ModelProvider, static upstream, pick fu // Only the ultrahigh preset turns it on (toolfilter `defer: auto`). body, di = h.applyToolDefer(r, tn, provider, wire, up, tr.Session, threadID, orig, body) } + // The savings record ends HERE, after both injections, with the body that goes + // upstream: the provider bills the injected tool declarations, so the library's + // bytes-to-tokens ratio must count them too. They are appended to `tools`, so no + // message moves and the operations' positions stay valid. + cp.accEnd(th, body, tr.Ops) }() // Load the request's one INFO line with everything the pipeline decided. serve // emits it in a defer once the response is finished, so a request produces exactly @@ -1937,6 +1974,7 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema } } } + cp.noteRounds(rounds) cp.finish(usage, usageOK, h.captureContentFor(tn), h.contentCap(), h.contentMax()) // THE one line per request. In a defer so every terminal path emits it exactly // once — stream-through, buffered replay, the round-cap exit, an upstream failure diff --git a/proxy/savings_test.go b/proxy/savings_test.go index ff2a960e..479f626a 100644 --- a/proxy/savings_test.go +++ b/proxy/savings_test.go @@ -85,7 +85,7 @@ func sendReal(t *testing.T, srv *httptest.Server, session string) { // with the equivalent Filter, so the surface and the underlying query cannot silently // diverge. func TestStatsSavingsThreeScopesAgreeWithOverview(t *testing.T) { - up := savingsUpstream(t) + up := billedUpstream(t) // a usage that bills the body, so the accountant library books a saving defer up.Close() h, rec := savingsTestHandler(t, up.URL) srv := httptest.NewServer(h.Mux()) From 189c169dbe67f9c5eaa6311b4cccaa24a2211fd1 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 06:46:15 +0200 Subject: [PATCH 27/58] build: Pin accountant to main after the TTL phase and media fixes Pin accountant at 5fa1986. It holds PR 46 (media pricing, faster extent search) and PR 47 (TTL per breakpoint, a billed read beats the clock). dash, proxy, apply, keepalive and components tests pass with it. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 92bd07ee..a734561a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.26.4 require github.com/maximhq/bifrost/core v1.7.0 // accountant prices context operations. This module's tests import it and its shared test cases. -require github.com/rossoctl/accountant v0.1.1-0.20261104153022-d31126bbb49b +require github.com/rossoctl/accountant v0.1.1-0.20261104203351-5fa19864db4c require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 diff --git a/go.sum b/go.sum index 5709e52d..7e368317 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rossoctl/accountant v0.1.1-0.20261104153022-d31126bbb49b h1:DnS0+FiNJt4qIUHjDAQ+G6H6Lmx28Aw8hAa4rciSbvM= -github.com/rossoctl/accountant v0.1.1-0.20261104153022-d31126bbb49b/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261104203351-5fa19864db4c h1:X71YQPS5juEzAUWTVEfuffU2sxW7avZIpyfTW/VkspQ= +github.com/rossoctl/accountant v0.1.1-0.20261104203351-5fa19864db4c/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From b843ee6c87db3ff0676ddc15ee318b91d2930074 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 06:52:51 +0200 Subject: [PATCH 28/58] fix(accountant): Keep measured filter requests and reads on priced rows (#911) The B2 double-count fix dropped a whole request from DeclFilterSavings when the accountant priced its toolfilter removal. That also dropped Requests, Reads and Since, so ToolFilterDocFor lost its Realized block and the item list in adaptToolsSaved could sum to more than the total. Now the row stays in the scan. An EXISTS flag column marks it and accumulateDeclFilterRow skips only the USD. carriedCost does the same for the items, so items and total agree. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/adapttools.go | 17 ++++++++++-- dash/declcredit_test.go | 58 +++++++++++++++++++++++++++++++++++++++-- dash/toolsuggest.go | 45 ++++++++++++++++++-------------- 3 files changed, 96 insertions(+), 24 deletions(-) diff --git a/dash/adapttools.go b/dash/adapttools.go index cd3cba58..91d5f9f9 100644 --- a/dash/adapttools.go +++ b/dash/adapttools.go @@ -253,6 +253,9 @@ type adaptToolsReq struct { tools int cacheRead, cacheWrite int64 cacheWrite1h, fresh int64 + // inBaseline is set (opt.onlyFiltered only) when the accountant priced this request's toolfilter + // removal: its tokens count, its dollars do not, as in DeclFilterSavings (context-guru#911). + inBaseline bool } // adaptToolsSet is one declaration set: its tool count, when it was first seen, and the billed @@ -332,13 +335,15 @@ type carriedRequest struct { func (d *DB) carriedCost(f Filter, price func(string) (modelinfo.Price, bool), keyOf func(session, kind, name, server string) string, opt carriedOpts) (*carriedScan, error) { where, args := f.where() + inBaselineCol := `0` if opt.onlyFiltered { where += ` AND r.filtered_decl_tokens > 0` + inBaselineCol = declFilterInBaseline } out := &carriedScan{Items: map[string]*carried{}, keepRequests: opt.perRequest} reqs := map[string][]adaptToolsReq{} // tenant\x00session -> requests, oldest first rows, err := d.sql.QueryContext(d.readCtx(), `SELECT r.tenant_id, r.session_id, r.ts, r.model, - r.token_accounting, r.decl_digest, r.tools, r.cache_read, r.cache_write, r.cache_write_1h, r.fresh_input + r.token_accounting, r.decl_digest, r.tools, r.cache_read, r.cache_write, r.cache_write_1h, r.fresh_input, `+inBaselineCol+` FROM requests r WHERE `+where+` AND r.tools > 0 ORDER BY r.ts`, args...) if err != nil { return nil, err @@ -346,7 +351,7 @@ func (d *DB) carriedCost(f Filter, price func(string) (modelinfo.Price, bool), for rows.Next() { var t, s string var q adaptToolsReq - if err := rows.Scan(&t, &s, &q.ts, &q.model, &q.accounting, &q.digest, &q.tools, &q.cacheRead, &q.cacheWrite, &q.cacheWrite1h, &q.fresh); err != nil { + if err := rows.Scan(&t, &s, &q.ts, &q.model, &q.accounting, &q.digest, &q.tools, &q.cacheRead, &q.cacheWrite, &q.cacheWrite1h, &q.fresh, &q.inBaseline); err != nil { rows.Close() return nil, err } @@ -465,6 +470,14 @@ func priceCarriedSession(out *carriedScan, byID []*carried, keys []string, list continue } m := sums[st] + if r.inBaseline { + // The accountant booked this request's removal: count the request and the tokens, add no dollars. + for key, v := range m { + byID[key].Requests++ + byID[key].Reads += int64(v[0]) + } + continue + } p, ok := modelinfo.Price{}, false if price != nil && r.model != "" { p, ok = price(r.model) diff --git a/dash/declcredit_test.go b/dash/declcredit_test.go index 74d50934..730f3bd9 100644 --- a/dash/declcredit_test.go +++ b/dash/declcredit_test.go @@ -476,8 +476,8 @@ func TestDeclCreditSkipsRequestsTheAccountantPriced(t *testing.T) { if err != nil { t.Fatal(err) } - if c.FilterRequests != 2 || c.FilterReads != 2000 { - t.Fatalf("filter half = %d requests / %d reads, want 2 / 2000 (the priced request is already in the baseline)", + if c.FilterRequests != 3 || c.FilterReads != 3000 { + t.Fatalf("filter half = %d requests / %d reads, want 3 / 3000 (the priced request keeps its facts and adds no dollars)", c.FilterRequests, c.FilterReads) } // 2,000 cache-read tokens at flatPrice's $0.1/MTok. @@ -485,3 +485,57 @@ func TestDeclCreditSkipsRequestsTheAccountantPriced(t *testing.T) { t.Errorf("filter USD = %g, want 0.0002", c.FilterUSD) } } + +// The accountant's pricing drops only the DOLLARS of a request's filter credit. The requests, the tokens +// the filter did not send and the first date are facts, so they stay (context-guru#911). The items of +// /api/adapt/tools must then agree with the total: both add no dollars for the same rows. +func TestFilterCreditKeepsFactsAndSkipsDollarsForAccountantPricedRows(t *testing.T) { + db := openTestDB(t) + var evs []*Event + for i := 0; i < 10; i++ { + e := mkEvent(day(0)+int64(i), "s1", "claude", 100, 90) + e.TenantID, e.Tools, e.FilteredDeclTokens, e.DeclDigest = "t1", 2, 5000, "d1" + e.Components = append(e.Components, CompRow{Component: "toolfilter", Kind: KindEnvelope, Acted: true, + Mutated: true, PricedBy: PricedByAccountant}) + evs = append(evs, e) + } + if err := db.insertBatch(evs); err != nil { + t.Fatal(err) + } + w := &invWriter{db: db, seen: map[string]*invSession{}} + if err := w.write([]invMsg{{tenant: "t1", session: "s1", ts: day(0), inv: &Inventory{Digest: "d1", + UseFingerprint: 1, Decls: []Decl{{Kind: KindMCPTool, Name: "mcp__srv__q", Server: "srv", Tokens: 5000}}}}}); err != nil { + t.Fatal(err) + } + got, err := db.DeclFilterSavings(Filter{Tenant: "t1"}, flatPrice) + if err != nil { + t.Fatal(err) + } + if got.Requests != 10 || got.Reads != 50000 || got.Since != day(0) { + t.Errorf("facts = %d requests / %d reads / since %d, want 10 / 50000 / %d", got.Requests, got.Reads, got.Since, day(0)) + } + if got.USD != 0 { + t.Errorf("USD = %g, want 0: the accountant booked these tokens", got.USD) + } + doc, err := db.ToolFilterDocFor(Filter{Tenant: "t1"}, flatPrice, ToolFilterState{}) + if err != nil { + t.Fatal(err) + } + if doc.Realized == nil || doc.Realized.Requests != 10 { + t.Errorf("Realized = %+v, want the measured block with 10 requests", doc.Realized) + } + saved, err := db.adaptToolsSaved(Filter{Tenant: "t1"}, flatPrice, []string{"mcp__srv"}) + if err != nil { + t.Fatal(err) + } + if saved.Filter.Requests != 10 || saved.Filter.USD != 0 { + t.Errorf("saved.filter = %d requests / $%g, want 10 / 0", saved.Filter.Requests, saved.Filter.USD) + } + sum := 0.0 + for _, it := range saved.Filter.Items { + sum += it.USD + } + if sum > saved.Filter.USD+1e-12 { + t.Errorf("items sum to $%g, more than the total $%g: %+v", sum, saved.Filter.USD, saved.Filter.Items) + } +} diff --git a/dash/toolsuggest.go b/dash/toolsuggest.go index f39c9fcf..fbfa025f 100644 --- a/dash/toolsuggest.go +++ b/dash/toolsuggest.go @@ -176,11 +176,11 @@ func (d *DB) DeclFilterSavings(f Filter, price func(string) (modelinfo.Price, bo sessions := map[string]bool{} for rows.Next() { var session, model string - var tok, cacheRead, cacheWrite, cacheWrite1h, ts int64 - if err := rows.Scan(&session, &model, &tok, &cacheRead, &cacheWrite, &cacheWrite1h, &ts); err != nil { + var tok, cacheRead, cacheWrite, cacheWrite1h, ts, inBaseline int64 + if err := rows.Scan(&session, &model, &tok, &cacheRead, &cacheWrite, &cacheWrite1h, &ts, &inBaseline); err != nil { return nil, err } - accumulateDeclFilterRow(out, sessions, session, model, tok, cacheRead, cacheWrite, cacheWrite1h, ts, price) + accumulateDeclFilterRow(out, sessions, session, model, tok, cacheRead, cacheWrite, cacheWrite1h, ts, inBaseline != 0, price) } if err := rows.Err(); err != nil { return nil, err @@ -189,14 +189,15 @@ func (d *DB) DeclFilterSavings(f Filter, price func(string) (modelinfo.Price, bo return out, nil } -// declFilterNotPriced keeps the requests whose declaration filtering is not already in the booked baseline. -// The proxy reports each toolfilter removal to the accountant library, so on a request the library priced -// (a toolfilter row with priced_by='accountant') the removal is in baseline_cost_usd and in the net saving. -// Adding the measured credit on top of it counts the same tokens twice (context-guru#768 B2). A request the -// library could not price (priced_by='none') or one written before the switch has nothing in the baseline, so -// the measured credit stays the only place its filter saving appears. -const declFilterNotPriced = `AND NOT EXISTS (SELECT 1 FROM request_components c - WHERE c.request_id = r.id AND c.component = 'toolfilter' AND c.priced_by = 'accountant')` +// declFilterInBaseline is a SELECT column: 1 when the accountant library priced this request's toolfilter +// removal. The proxy reports each removal to the library, so on such a request the removal is already in +// baseline_cost_usd and in the net saving. Adding the measured dollar credit on top counts the same tokens +// twice (context-guru#768 B2), so those rows add no USD. They still add their requests, tokens and dates: +// those are facts about what the filter did, not dollars (context-guru#911). A request the library could +// not price (priced_by='none') or one written before the switch has nothing in the baseline, so its +// measured credit stays the only place its filter saving appears. +const declFilterInBaseline = `EXISTS (SELECT 1 FROM request_components c + WHERE c.request_id = r.id AND c.component = 'toolfilter' AND c.priced_by = '` + PricedByAccountant + `')` // declFilterSavingsQuery builds the SQL and args DeclFilterSavings executes — pulled out for // #502's EXPLAIN-plan test, same reason as cachesplitHistoricalQuery (dash/cachehistory.go). @@ -205,16 +206,17 @@ const declFilterNotPriced = `AND NOT EXISTS (SELECT 1 FROM request_components c func declFilterSavingsQuery(f Filter) (string, []any) { where, args := f.where() return `SELECT r.session_id, r.model, r.filtered_decl_tokens, - r.cache_read, r.cache_write, r.cache_write_1h, r.ts FROM requests r - WHERE ` + where + ` AND r.filtered_decl_tokens > 0 ` + declFilterNotPriced, args + r.cache_read, r.cache_write, r.cache_write_1h, r.ts, ` + declFilterInBaseline + ` FROM requests r + WHERE ` + where + ` AND r.filtered_decl_tokens > 0`, args } // accumulateDeclFilterRow folds one filtered_decl_tokens>0 row into a running total — the one // piece of DeclFilterSavings that has to run in Go rather than SQL, since each row prices at // ITS OWN model's rate and cache tier. Shared with DeclFilterSavingsByTenant so the two never -// drift on how a row is priced. +// drift on how a row is priced. skipUSD keeps the row's requests, tokens and dates and adds no dollars: +// the accountant already booked them (declFilterInBaseline). func accumulateDeclFilterRow(out *DeclFilterSaving, sessions map[string]bool, - session, model string, tok, cacheRead, cacheWrite, cacheWrite1h, ts int64, price func(string) (modelinfo.Price, bool)) { + session, model string, tok, cacheRead, cacheWrite, cacheWrite1h, ts int64, skipUSD bool, price func(string) (modelinfo.Price, bool)) { out.Requests++ sessions[session] = true out.Reads += tok @@ -248,6 +250,9 @@ func accumulateDeclFilterRow(out *DeclFilterSaving, sessions map[string]bool, out.FreshTokens += tok rate = p.Input } + if skipUSD { + return + } if priced { out.USD += float64(tok) * rate } else { @@ -261,8 +266,8 @@ func accumulateDeclFilterRow(out *DeclFilterSaving, sessions map[string]bool, func (d *DB) DeclFilterSavingsByTenant(since int64, price func(string) (modelinfo.Price, bool)) (map[string]*DeclFilterSaving, error) { cond, args := (Filter{Since: since, TenantAll: true}).where() rows, err := d.sql.QueryContext(d.readCtx(), `SELECT r.tenant_id, r.session_id, r.model, r.filtered_decl_tokens, - r.cache_read, r.cache_write, r.cache_write_1h, r.ts FROM requests r - WHERE `+cond+` AND r.filtered_decl_tokens > 0 `+declFilterNotPriced, args...) + r.cache_read, r.cache_write, r.cache_write_1h, r.ts, `+declFilterInBaseline+` FROM requests r + WHERE `+cond+` AND r.filtered_decl_tokens > 0`, args...) if err != nil { return nil, err } @@ -271,8 +276,8 @@ func (d *DB) DeclFilterSavingsByTenant(since int64, price func(string) (modelinf sessions := map[string]map[string]bool{} for rows.Next() { var tenant, session, model string - var tok, cacheRead, cacheWrite, cacheWrite1h, ts int64 - if err := rows.Scan(&tenant, &session, &model, &tok, &cacheRead, &cacheWrite, &cacheWrite1h, &ts); err != nil { + var tok, cacheRead, cacheWrite, cacheWrite1h, ts, inBaseline int64 + if err := rows.Scan(&tenant, &session, &model, &tok, &cacheRead, &cacheWrite, &cacheWrite1h, &ts, &inBaseline); err != nil { return nil, err } s, ok := out[tenant] @@ -281,7 +286,7 @@ func (d *DB) DeclFilterSavingsByTenant(since int64, price func(string) (modelinf out[tenant] = s sessions[tenant] = map[string]bool{} } - accumulateDeclFilterRow(s, sessions[tenant], session, model, tok, cacheRead, cacheWrite, cacheWrite1h, ts, price) + accumulateDeclFilterRow(s, sessions[tenant], session, model, tok, cacheRead, cacheWrite, cacheWrite1h, ts, inBaseline != 0, price) } if err := rows.Err(); err != nil { return nil, err From c00c28d88e52f5441018b9249bafcec663bb5c6f Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 06:54:42 +0200 Subject: [PATCH 29/58] fix(accountant): Report the tail breakpoint mark as an operation (#906) The default tail breakpoint (#795) adds a cache_control mark to the last message after the pipeline ran. A string content also becomes one text block. The message gets longer on the wire, and no operation reported the extra bytes. The sum check failed on every request with a trailing system message (section=messages ops=0 bodies=-63 bytes), so every request fell back with check_sum and booked no saving. apply now appends one rewrite operation for the last message, in wire bytes, when markTail places a mark. Its ChangeID comes from the message before the mark, so it is the same on every request. The operator tailbreakpoint is an envelope operator in dash, so it gets its own row. No accountant check or plausibility rule is weakened. The check now has the operation it was missing. Harness (FAKE_TOKDIV=8, tailx, 7 requests): net_saved_usd 0 -> 0.0817. Truth (control cost - tail on cost): 0.0736. Fallbacks 7 -> 0. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- apply/apply.go | 6 +++ apply/ops.go | 25 ++++++++++ apply/tailbp_test.go | 21 ++++++++ dash/accountant_price.go | 1 + proxy/accountant_fallbacks_test.go | 78 ++++++++++++++++++++++++++++++ 5 files changed, 131 insertions(+) create mode 100644 proxy/accountant_fallbacks_test.go diff --git a/apply/apply.go b/apply/apply.go index 1daa4363..4db6049c 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -411,6 +411,12 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o return } if out, ok := markTail(res.Body); ok { + // The mark makes the last message longer (a string content becomes one block, and + // the block carries the mark). No pipeline component reports that, so the sum check + // would fail on the messages section. Report it here, as the last operation. + if op, ok := tailOp(res.Body, out); ok && res.Trace.OpsDropped == 0 { + res.Trace.Ops = append(res.Trace.Ops, op) + } res.Body, res.Changed = out, true } }() diff --git a/apply/ops.go b/apply/ops.go index 0da12ba0..1da43f48 100644 --- a/apply/ops.go +++ b/apply/ops.go @@ -1,8 +1,11 @@ package apply import ( + "hash/fnv" "strconv" + "github.com/tidwall/gjson" + "github.com/rossoctl/accountant" "github.com/rossoctl/context-guru/components" ) @@ -278,3 +281,25 @@ func chain(ops []accountant.Operation, ks []int, b, a int) { cur = next } } + +// tailOp is the operation for the tail mark (markTail): a rewrite of the last message, from its wire +// bytes before the mark to its wire bytes after. It is the last operation of the request, so a +// pipeline operation on the same message ends where this one starts. The ChangeID comes from the +// message as it was before the mark, so the same message gets the same ChangeID on every request. +func tailOp(before, after []byte) (accountant.Operation, bool) { + b, a := gjson.GetBytes(before, "messages").Array(), gjson.GetBytes(after, "messages").Array() + if len(b) == 0 || len(b) != len(a) { + return accountant.Operation{}, false + } + last := len(b) - 1 + h := fnv.New64a() + h.Write([]byte(b[last].Raw)) + return accountant.Operation{ + Kind: accountant.KindRewrite, + Operator: "context-guru.tailbreakpoint", + At: accountant.Position{Section: accountant.SectionMessages, Index: last}, + SizeBefore: accountant.Size{Bytes: len(b[last].Raw)}, + SizeAfter: accountant.Size{Bytes: len(a[last].Raw)}, + ChangeID: "cg:tailbreakpoint:" + strconv.FormatUint(h.Sum64(), 16), + }, true +} diff --git a/apply/tailbp_test.go b/apply/tailbp_test.go index 9b0a2a3f..406efeaa 100644 --- a/apply/tailbp_test.go +++ b/apply/tailbp_test.go @@ -135,3 +135,24 @@ func TestTheTailBreakpointIsAddedPastTheLookback(t *testing.T) { t.Fatalf("not added past the lookback:\n%s", out) } } + +// The tail mark is reported as an operation on the last message, sized in wire bytes, so the +// accountant's sum check holds (#906). +func TestTailOpCoversTheMarkBytes(t *testing.T) { + body := tailBody(`{"type":"ephemeral"}`, 5000) + out, ok := markTail(body) + if !ok { + t.Fatal("no tail mark") + } + op, ok := tailOp(body, out) + if !ok { + t.Fatal("no operation") + } + last := len(gjson.GetBytes(out, "messages").Array()) - 1 + if op.At.Index != last || op.SizeAfter.Bytes-op.SizeBefore.Bytes != len(out)-len(body) { + t.Fatalf("op %+v, body grew by %d", op, len(out)-len(body)) + } + if again, _ := tailOp(body, out); again.ChangeID != op.ChangeID { + t.Fatal("the ChangeID is not stable") + } +} diff --git a/dash/accountant_price.go b/dash/accountant_price.go index 4b64d455..e20d4b20 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -191,6 +191,7 @@ const EndedChangeComponent = "ended_change" var envelopeComponents = map[string]bool{ "expand": true, "adjudicate": true, "headttl": true, "toolschema": true, "toolfilter": true, "cachesplit": true, + "tailbreakpoint": true, } // AccountantEstimated reports whether the applied figure is an estimate: the library could not rule diff --git a/proxy/accountant_fallbacks_test.go b/proxy/accountant_fallbacks_test.go new file mode 100644 index 00000000..420a6a6d --- /dev/null +++ b/proxy/accountant_fallbacks_test.go @@ -0,0 +1,78 @@ +package proxy + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/rossoctl/context-guru/dash" +) + +// accPostBody sends body to the handler under session and returns the response status. +func accPostBody(t *testing.T, h *Handler, session string, body []byte) { + t.Helper() + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", strings.NewReader(string(body))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-context-guru-session", session) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() +} + +// accNoSumFailure fails the test when an event's Result carries a failed check, or the event fell +// back for a reason that means the operation list is wrong. +func accNoSumFailure(t *testing.T, e dash.Event) { + t.Helper() + res, ok := e.AccountantResult() + if !ok { + t.Fatalf("no accountant result (fallback %q)", e.AccountantFallback()) + } + for _, c := range res.Checks { + t.Errorf("check failed: %s %s", c.Check, c.Detail) + } + if fb := e.AccountantFallback(); fb != "" { + t.Errorf("the override must apply, fell back: %s", fb) + } +} + +// A Claude Code style request ends in a long `system` message after the client's last mark. The +// default tail breakpoint (#795) marks that message, which makes its wire bytes longer. The +// mark is an operation of its own: without it the sum check failed on every request and no saving +// was booked (#906). +func TestTailBreakpointReportsItsBytes(t *testing.T) { + up := billedUpstream(t) + defer up.Close() + h, rec, tap := accHandlerYAML(t, up.URL, "preset: codesafe\ncache:\n tail_breakpoint: true\n") + build := func(n int) []byte { + msgs := []any{map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "text", "text": "reminder"}, + map[string]any{"type": "text", "text": "fix it", "cache_control": map[string]any{"type": "ephemeral"}}, + }}} + for k := 0; k < n; k++ { + msgs = append(msgs, + map[string]any{"role": "assistant", "content": []any{map[string]any{"type": "tool_use", "id": "tu_" + string(rune('a'+k)), "name": "bash", "input": map[string]any{"command": "ls"}}}}, + map[string]any{"role": "user", "content": []any{map[string]any{"type": "tool_result", "tool_use_id": "tu_" + string(rune('a'+k)), "content": bigToolOutput()}}}) + } + // A string content, so the mark also turns it into one text block. + msgs = append(msgs, map[string]any{"role": "system", "content": strings.Repeat("env line. ", 700)}) + b, _ := json.Marshal(map[string]any{"model": "aws/claude-sonnet-5", "max_tokens": 64, "messages": msgs, + "system": []any{map[string]any{"type": "text", "text": "s1", "cache_control": map[string]any{"type": "ephemeral"}}}}) + return b + } + accPostBody(t, h, "sess-tail", build(1)) + accPostBody(t, h, "sess-tail", build(2)) + waitForRows(t, rec, 2) + evs := tap.all() + for _, e := range evs { + accNoSumFailure(t, e) + } + if got := evs[1].AccountantFallback(); got != "" { + t.Fatalf("second request fell back: %s", got) + } +} From 97022309871630afee0d117064bb8a8d9ee1ae81 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:02:19 +0200 Subject: [PATCH 30/58] fix(accountant): Book no saved tokens on a request the provider rejected (#914) A request the provider answered with a non-2xx status billed nothing: cost 0 and baseline 0. Its saved_gross and saved_unique were still booked, so tokens saved disagreed with dollars saved. In the run_m4 repro (turn 1 returns 200, turn 2 returns 400), /api/stats showed saved_gross 28657 and saved_unique 19105 for 2 requests. finish() now calls Event.DropUnbilledSavings. On a status that is not 2xx it sets tokens_after to tokens_before, and zeroes saved_unique, attempted_tokens, and saved_gross, saved_unique and acted on every component row. Status 0 (never reached a provider, such as /compact) and any 2xx (billed, even without usage) are untouched. attempted_tokens goes to 0 so Overview's saved_gross_attempted / attempted_tokens ratio keeps one row population on both sides. No separate attempted counter is added: attempted_tokens already holds what was eligible, and no reader needs the tokens of a request that did not bill. After: saved_gross 9553 and saved_unique 9553 for the same 2 requests (turn 1 only); the 400 row reads tokens_before 19243, tokens_after 19243, saved_unique 0. Dollars do not change. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/event.go | 21 +++++++++ proxy/dashcapture.go | 2 + proxy/rejected_savings_test.go | 80 ++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 proxy/rejected_savings_test.go diff --git a/dash/event.go b/dash/event.go index 64da6617..c2086900 100644 --- a/dash/event.go +++ b/dash/event.go @@ -783,6 +783,27 @@ func (e *Event) FromTrace(tr apply.Trace, uniqueSaved map[string]int) { e.UncompressedReason = uncompressedReason(e, tr) } +// DropUnbilledSavings removes the token saving from a request the provider rejected (issue #914). +// The provider billed none of its tokens, so cost and baseline are 0, and a token saving booked +// beside them would disagree with the dollar saving. It sets TokensAfter to TokensBefore, zeroes +// SavedUnique, and zeroes SavedGross and SavedUnique on every component row. A row keeps Mutated +// (the proxy did rewrite the body) but is no longer Acted. A status of 0 is a request that never +// reached a provider (/compact, an upstream that did not answer), and a 2xx was billed, so +// neither is touched. AttemptedTokens goes to 0 too: Overview divides the gross saving by it over +// the rows that have it, and a rejected row must leave both sides. FrozenTokens describes the +// cache freeze, not a saving, and stays. +func (e *Event) DropUnbilledSavings() { + if e.Status == 0 || (e.Status >= 200 && e.Status < 300) { + return + } + e.TokensAfter = e.TokensBefore + e.SavedUnique, e.AttemptedTokens = 0, 0 + for i := range e.Components { + c := &e.Components[i] + c.SavedGross, c.SavedUnique, c.Acted = 0, 0, false + } +} + // uncompressedReason answers "why didn't you compact this?" from what the trace // shows. Empty means we did compact. func uncompressedReason(e *Event, tr apply.Trace) string { diff --git a/proxy/dashcapture.go b/proxy/dashcapture.go index 2ab23ed9..ad8fdb2e 100644 --- a/proxy/dashcapture.go +++ b/proxy/dashcapture.go @@ -526,6 +526,8 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content e.Project = c.project e.Meta = c.meta e.FromTrace(c.trace, c.unique) + // A request the provider rejected saved nothing: book no tokens beside its zero dollars (#914). + e.DropUnbilledSavings() e.ThreadID = c.thread e.AgentCompaction = c.agentComp // The provider's terminal reason, off the same response bytes the token tiers came diff --git a/proxy/rejected_savings_test.go b/proxy/rejected_savings_test.go new file mode 100644 index 00000000..0f00059e --- /dev/null +++ b/proxy/rejected_savings_test.go @@ -0,0 +1,80 @@ +package proxy + +import ( + "path/filepath" + "testing" + "time" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/dash" +) + +// A request the provider rejected (a 4xx or 5xx) removed no tokens from any bill. Its row must +// book no saved tokens: not on the event, not on its component rows. The row keeps the size +// the proxy received, so tokens_after equals tokens_before. A 2xx row keeps its saving (#914). +func TestFinishBooksNoSavedTokensOnARejectedRequest(t *testing.T) { + for _, tc := range []struct { + name string + status int + usage Usage + usageOK bool + wantSaved int + wantActed bool + wantUnique int + }{ + {"2xx keeps the saving", 200, Usage{FreshInput: 10, CacheWrite: 1000}, true, 900, true, 400}, + {"400 books none", 400, Usage{}, false, 0, false, 0}, + {"500 books none", 500, Usage{}, false, 0, false, 0}, + // A 2xx with no usage was billed by the provider: its saving is real, only unpriced. + {"2xx without usage keeps the saving", 200, Usage{}, false, 900, true, 400}, + // Status 0 is a request that never reached a provider (/compact): not a rejection. + {"status 0 keeps the saving", 0, Usage{}, false, 900, true, 400}, + } { + t.Run(tc.name, func(t *testing.T) { + rec, err := dash.NewRecorder(dash.Options{ + DBPath: filepath.Join(t.TempDir(), "d.db"), BatchSize: 1, FlushInterval: time.Millisecond}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { rec.Close() }) + c := &capture{rec: rec, tenant: "t1", provider: "anthropic", model: "m", route: "/v1/messages", + start: time.Now(), status: tc.status, + trace: apply.Trace{Session: "s", Messages: 4, AttemptedTokens: 900, Run: &components.RunReport{ + TokensBefore: 1000, TokensAfter: 100, + Components: []components.Report{{Component: "mask", Kind: "offload", TokensBefore: 1000, TokensAfter: 100}}, + }}, + unique: map[string]int{"mask": 400}} + c.finish(tc.usage, tc.usageOK, false, 0, 0) + + var got *dash.Event + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) && got == nil { + if pg, err := rec.DB().Requests(dash.Filter{TenantAll: true}, 0, 10); err == nil && len(pg.Requests) == 1 { + got, err = rec.DB().Request(pg.Requests[0].ID, false) + if err != nil { + t.Fatal(err) + } + } + time.Sleep(20 * time.Millisecond) + } + if got == nil { + t.Fatal("no row was stored") + } + if got.Saved() != tc.wantSaved || got.SavedUnique != tc.wantUnique { + t.Errorf("event saved = %d unique = %d, want %d and %d (tokens %d -> %d)", + got.Saved(), got.SavedUnique, tc.wantSaved, tc.wantUnique, got.TokensBefore, got.TokensAfter) + } + if wantAtt := map[bool]int{true: 900, false: 0}[tc.wantSaved > 0]; got.AttemptedTokens != wantAtt { + t.Errorf("attempted_tokens = %d, want %d", got.AttemptedTokens, wantAtt) + } + if len(got.Components) != 1 { + t.Fatalf("components = %d, want 1", len(got.Components)) + } + if c := got.Components[0]; c.SavedGross != tc.wantSaved || c.SavedUnique != tc.wantUnique || c.Acted != tc.wantActed { + t.Errorf("component gross = %d unique = %d acted = %v, want %d, %d, %v", + c.SavedGross, c.SavedUnique, c.Acted, tc.wantSaved, tc.wantUnique, tc.wantActed) + } + }) + } +} From 14a5848a36d85e43d985a285a86383574042270c Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:03:28 +0200 Subject: [PATCH 31/58] fix(accountant): Leave tool-section operations out of the plausibility bound (#907) PriceMode compared the tokens the library saved with e.TokensBefore. TokensBefore counts message text only. Removed tool declarations are not in it. A 121-token conversation that drops 2,188 tokens of declarations looked like a request that removed more than it had, so the event fell back with implausible_size and lost every other component's saving too. countedSavedTokens now skips operations of the operators that act on the tools section (toolfilter, toolschema, expand, adjudicate). The bound is unchanged for every operation on messages. The accountant checks are not touched. The bound cannot be made against the full request size here, because the event has no count of tool and system tokens. Harness (FAKE_TOKDIV=7, MSGMULT=30, tf2, 8 requests): fallbacks 8 implausible_size -> 0. net_saved_usd 0 -> 0.0554. Truth (control cost - treatment cost): 0.0564. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accountant_price.go | 16 ++++++++++++++-- dash/accountant_switch_test.go | 25 +++++++++++++++++++++++++ docs/dashboard.md | 3 ++- proxy/accountant_fallbacks_test.go | 29 +++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/dash/accountant_price.go b/dash/accountant_price.go index e20d4b20..8565e983 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -348,12 +348,24 @@ func inAdded(rows []CompRow, name string) bool { return false } +// toolSectionOperators are the operators whose operations act on the tools section, not on messages: +// the declaration filter and the schema rewrite (proxy/accountant_envelope.go reports them from a body +// diff), and the two declarations the proxy injects. Their tokens are not in TokensBefore, which counts +// message text only. +var toolSectionOperators = map[string]bool{ + "context-guru.toolfilter": true, "context-guru.toolschema": true, + "context-guru.expand": true, "context-guru.adjudicate": true, +} + // countedSavedTokens is the Saved tokens, all four tiers, of the operations that feed -// AccountantFigure.ByComponent: not DryRun, not a spend. +// AccountantFigure.ByComponent: not DryRun, not a spend, and not an operation on the tools section. +// TokensBefore is the size of the messages. A tool declaration is not in it, so 2,188 removed +// declaration tokens would look larger than a 121-token conversation and drop the whole result (#907). +// The bound stays for every operation on messages. func (e *Event) countedSavedTokens() int { n := 0 for _, o := range e.accRes.Ops { - if o.DryRun || o.Call != nil { + if o.DryRun || o.Call != nil || toolSectionOperators[o.Operator] { continue } n += o.Saved.Input + o.Saved.CacheRead + o.Saved.CacheWrite5m + o.Saved.CacheWrite1h diff --git a/dash/accountant_switch_test.go b/dash/accountant_switch_test.go index 801d652d..11589a1e 100644 --- a/dash/accountant_switch_test.go +++ b/dash/accountant_switch_test.go @@ -338,3 +338,28 @@ func TestPriceEstimatedIsStoredAndReadBack(t *testing.T) { t.Fatalf("stored price_estimated = %v, want est true and exact false", got) } } + +// Removed tool declarations are not message text. TokensBefore counts messages only, so a small +// conversation that drops a large tool list must not trip the plausibility bound (#907). +func TestPriceMode_ToolSectionTokensAreNotBoundedByMessageTokens(t *testing.T) { + e := switchEvent() + e.TokensBefore = 121 + e.accRes.Ops = []accountant.PricedOp{{Operator: "context-guru.toolfilter", ChangeID: "cg:toolfilter:x", + Saved: accountant.TierTokens{CacheRead: 2188}}} + e.Components = []CompRow{{Component: "idle"}} + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb != "" { + t.Fatalf("fell back: %q", fb) + } + if got := e.BaselineCostUSD - e.CostUSD; !accNear(got, 2188*0.3e-6) { + t.Fatalf("delta = %v, want the declarations' saving", got) + } + // The bound still holds for messages: the same size on a message operation falls back. + m := switchEvent() + m.TokensBefore = 121 + m.accRes.Ops[0].Saved = accountant.TierTokens{CacheRead: 2188} + m.PriceMode(accTestPrice, true) + if fb := m.AccountantFallback(); fb != FallbackImplausibleSize { + t.Fatalf("message operation: fallback %q, want %q", fb, FallbackImplausibleSize) + } +} diff --git a/docs/dashboard.md b/docs/dashboard.md index e79002dc..2d3318eb 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -181,7 +181,8 @@ Limits: of 10,000 threads (the oldest go first), the next request of the thread is a first request again. - The figures are estimates. The library converts removed bytes to tokens with the request's own billed-token ratio. A figure that implies more removed tokens than the request had is rejected - (`implausible_size`). + (`implausible_size`). The bound counts message tokens, so it leaves out operations on the tool + section (removed or rewritten declarations). - The cost and the provider's cache saving are not the library's. - Rows stored before the switch (`priced_by` empty) keep their legacy figures. The Components tab, the summarize ledger and the replay cap value those rows with the legacy rule at read time, and diff --git a/proxy/accountant_fallbacks_test.go b/proxy/accountant_fallbacks_test.go index 420a6a6d..37c057c0 100644 --- a/proxy/accountant_fallbacks_test.go +++ b/proxy/accountant_fallbacks_test.go @@ -76,3 +76,32 @@ func TestTailBreakpointReportsItsBytes(t *testing.T) { t.Fatalf("second request fell back: %s", got) } } + +// A small conversation that drops a large tool list: the removed declarations are many times the +// size of the messages. The plausibility bound reads message tokens, so it dropped the whole result +// with implausible_size (#907). +func TestRemovedToolDeclarationsDoNotTripThePlausibilityBound(t *testing.T) { + up := billedUpstream(t) + defer up.Close() + h, rec, tap := accHandlerYAML(t, up.URL, + "pipeline: [toolfilter]\ncomponents:\n toolfilter:\n remove: [big_one, big_two]\n") + decl := func(name string, n int) map[string]any { + return map[string]any{"name": name, "description": strings.Repeat("Does a thing. ", n), + "input_schema": map[string]any{"type": "object", "properties": map[string]any{"q": map[string]any{"type": "string"}}}} + } + b, _ := json.Marshal(map[string]any{"model": "aws/claude-sonnet-5", "max_tokens": 64, + "tools": []any{decl("bash", 3), decl("big_one", 400), decl("big_two", 400)}, + "messages": []any{map[string]any{"role": "user", "content": "hi"}}}) + accPostBody(t, h, "sess-tools", b) + accPostBody(t, h, "sess-tools", b) + waitForRows(t, rec, 2) + for i, e := range tap.all() { + accNoSumFailure(t, e) + if e.TokensBefore >= 100 { + t.Fatalf("precondition: messages must be small, TokensBefore = %d", e.TokensBefore) + } + if i == 1 && e.BaselineCostUSD <= e.CostUSD { + t.Errorf("no saving booked for the removed declarations: baseline %v cost %v", e.BaselineCostUSD, e.CostUSD) + } + } +} From 4f2a41759f86a37b5d45773131b3ea1b4f3d65e5 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:03:42 +0200 Subject: [PATCH 32/58] fix(accountant): Trust the library's keep-alive rescue and judge the TTL by the billed tier (#912, #919) kaSavedGate needed a request_components row with priced_by='accountant' to accept a rescue the ping only wrote. A keep-alive-only setup stores no component rows, so the rescue was zeroed again. The request row now stores acc_priced (additive column, default 0), set when the accountant override applied. The gate trusts keepalive_saved_usd of such a row as the library's own decision. It is a plain column read placed after the credited-row guard, so the probe order is unchanged. Rows with acc_priced = 0 keep the SQL gate. The entry TTL given to the library was 1h when every breakpoint asked for it, before the billed tier was known. accBilledTTL now lowers the TTL (and the breakpoint TTLs) to 5m after the usage arrives, when the response billed a 5m write, before Compute runs. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accountant_price.go | 3 +- dash/accountant_switch_test.go | 33 ++++++++++++++++ dash/event.go | 10 ++++- dash/keepalive.go | 9 ++++- dash/keepalivetab_test.go | 69 ++++++++++++++++++++++++++++++++++ dash/query.go | 8 ++-- dash/schema.go | 5 +++ dash/store.go | 6 +-- docs/dashboard.md | 9 +++++ docs/data-dependence.md | 2 +- proxy/accountant_ping_test.go | 35 +++++++++++++++++ proxy/accountant_wire.go | 36 ++++++++++++++++-- proxy/dashcapture.go | 1 + 13 files changed, 210 insertions(+), 16 deletions(-) diff --git a/dash/accountant_price.go b/dash/accountant_price.go index 4b64d455..b0d745a9 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -214,7 +214,7 @@ func (e *Event) AccountantFallback() string { return e.accFallback } // library figure. Otherwise the event books no saving (Price set every saving to 0) and // AccountantFallback says why: a figure nothing measured is not shown. func (e *Event) PriceMode(p modelinfo.Price, accountingComplete bool) { - e.accFallback, e.PriceEstimated = "", false + e.accFallback, e.PriceEstimated, e.AccountantPriced = "", false, false if !e.priceCost(p, accountingComplete) { e.accFallback = FallbackUnpriced return @@ -335,6 +335,7 @@ func (e *Event) accOverride(p modelinfo.Price) string { // the legacy rules had them: every reader of keepalive_saved_usd and cachesplit_saved_usd stays valid. e.KeepAliveSavedUSD, e.CachesplitSavedUSD = fig.KeepAliveSavedUSD, fig.SplitUSD e.PriceEstimated = estimated + e.AccountantPriced = true return "" } diff --git a/dash/accountant_switch_test.go b/dash/accountant_switch_test.go index 801d652d..2d37ff2a 100644 --- a/dash/accountant_switch_test.go +++ b/dash/accountant_switch_test.go @@ -338,3 +338,36 @@ func TestPriceEstimatedIsStoredAndReadBack(t *testing.T) { t.Fatalf("stored price_estimated = %v, want est true and exact false", got) } } + +// The marker the keep-alive gate trusts (context-guru#912): set when the override applied, clear on a +// fallback, and stored with the request. +func TestAccountantPricedIsSetOnOverrideAndStored(t *testing.T) { + db, err := Open(filepath.Join(t.TempDir(), "p.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ok, fell := switchEvent(), switchEvent() + ok.TS, ok.SessionID, fell.TS, fell.SessionID = 1, "ok", 2, "fell" + fell.accRes = nil // no_result: the override cannot apply + ok.PriceMode(accTestPrice, true) + fell.PriceMode(accTestPrice, true) + if !ok.AccountantPriced || fell.AccountantPriced { + t.Fatalf("AccountantPriced = %v on the overridden event and %v on the fallback (%q)", + ok.AccountantPriced, fell.AccountantPriced, fell.AccountantFallback()) + } + if err := db.insertBatch([]*Event{ok, fell}); err != nil { + t.Fatal(err) + } + page, err := db.Requests(Filter{}, 0, 10) + if err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, e := range page.Requests { + got[e.SessionID] = e.AccountantPriced + } + if len(got) != 2 || !got["ok"] || got["fell"] { + t.Fatalf("stored acc_priced = %v, want ok true and fell false", got) + } +} diff --git a/dash/event.go b/dash/event.go index 64da6617..196e5e2a 100644 --- a/dash/event.go +++ b/dash/event.go @@ -444,7 +444,13 @@ type Event struct { // change broke the cache (penalty_unknown): the gross saving is right and the net may be too // high. It is stored, so the dashboard marks such a figure after a restart. False on an exact // figure, on an event that books no saving, and on a row from before the column existed. - PriceEstimated bool `json:"price_estimated,omitempty"` + PriceEstimated bool `json:"price_estimated,omitempty"` + // AccountantPriced is true when the accountant override applied to this request, so its + // keepalive_saved_usd is the library's own rescue decision. Stored (requests.acc_priced), so the + // keep-alive reachability gate trusts that decision with no component row and no ping probe. False + // on a request the library did not price and on every row from before the column existed, which + // keep the SQL gate (context-guru#912, #919). + AccountantPriced bool `json:"acc_priced,omitempty"` CacheMissReason string `json:"cache_miss_reason"` UncompressedReason string `json:"uncompressed_reason"` // CacheMissCause says what changed on a miss that is not hit, cold_start or ttl_expiry. @@ -829,7 +835,7 @@ func uncompressedReason(e *Event, tr apply.Trace) string { // accountingComplete=false leaves every cost at zero and the row is marked partial/missing: a cost we // cannot compute must read as unknown, not as free. func (e *Event) Price(p modelinfo.Price, accountingComplete bool) { - e.accFallback, e.PriceEstimated = "", false + e.accFallback, e.PriceEstimated, e.AccountantPriced = "", false, false e.priceCost(p, accountingComplete) } diff --git a/dash/keepalive.go b/dash/keepalive.go index 1057ae92..996f4510 100644 --- a/dash/keepalive.go +++ b/dash/keepalive.go @@ -218,7 +218,14 @@ func kaSavedGate(alias, pingFloor string) string { // is a second EXISTS inside the same AND, and it runs only for a row the strict probe refused. libraryPriced := `EXISTS (SELECT 1 FROM request_components c WHERE c.request_id = ` + alias + `id AND c.priced_by = '` + PricedByAccountant + `')` - return `(CASE WHEN ` + col + ` > 0 AND (` + probe(`kp.cache_read > 0`) + ` OR (` + libraryPriced + ` AND ` + + // A request the accountant override priced (acc_priced = 1) carries the library's own rescue decision + // in keepalive_saved_usd. It is trusted as stored: the gap, the last ping and the extent were judged by + // the library, with the billed usage, and the SQL model below cannot see them. This also holds when the + // request has no component row, as in a keep-alive-only setup (context-guru#912, #919). The test is a + // plain column read, so the probe never runs for such a row. It stays AFTER col > 0: the credited-row + // guard is first (see above). A row with acc_priced = 0 (legacy, or one the library did not price) keeps + // the SQL model, with its component-row clause. + return `(CASE WHEN ` + col + ` > 0 AND (` + alias + `acc_priced = 1 OR ` + probe(`kp.cache_read > 0`) + ` OR (` + libraryPriced + ` AND ` + probe(`kp.cache_write > 0`) + `)) THEN ` + col + ` ELSE 0 END)` } diff --git a/dash/keepalivetab_test.go b/dash/keepalivetab_test.go index f333eff2..5e5413b5 100644 --- a/dash/keepalivetab_test.go +++ b/dash/keepalivetab_test.go @@ -1333,3 +1333,72 @@ func TestTheGateKeepsTheRescueOfAPingThatWroteTheEntry(t *testing.T) { t.Errorf("net = %.6f, want %.6f", got.KeepAliveNetUSD, want) } } + +// A keep-alive-only setup stores no component rows, so the rescue the library credited has nothing for the +// gate to find and was zeroed by the strict probe: the original B4 symptom, left in the case the keep-alive +// is mostly used for (context-guru#912). The request itself must say the library priced it. +func TestTheGateKeepsTheRescueOfALibraryPricedRequestWithNoComponentRows(t *testing.T) { + rescued := kaCredit(1_100_000, "s1", 0.01045695) + rescued.Components = nil + rescued.AccountantPriced = true + fx := newKAFixture(t, + kaAgent(1_000_000, "s1", 0.10), + kaPing(1_050_000, "s1", 0.01145625, 0, 3051), // the entry had expired: the ping wrote it again + rescued, + ) + got := kaSession(t, fx, "s1") + if math.Abs(got.KeepAliveSavedUSD-0.01045695) > 1e-9 { + t.Errorf("saved = %.6f, want 0.010457: the rescue the library credited was zeroed by the gate", got.KeepAliveSavedUSD) + } + if want := 0.01045695 - 0.01145625; math.Abs(got.KeepAliveNetUSD-want) > 1e-9 { + t.Errorf("net = %.6f, want %.6f", got.KeepAliveNetUSD, want) + } +} + +// The library's decision is trusted for a library-priced request even when the SQL model finds no ping that +// reached the entry, and a row without the marker keeps the SQL gate (context-guru#919). +func TestTheGateTrustsTheLibraryForPricedRowsAndKeepsSQLForLegacyRows(t *testing.T) { + priced := kaCredit(1_100_000, "lib", 0.01) + priced.AccountantPriced = true + legacy := kaCredit(1_100_000, "old", 0.01) + fx := newKAFixture(t, + kaAgent(1_000_000, "lib", 0.10), kaPing(1_050_000, "lib", 0.001, 0, 0), priced, // the ping read and wrote nothing + kaAgent(1_000_000, "old", 0.10), kaPing(1_050_000, "old", 0.001, 0, 0), legacy, + ) + if got := kaSession(t, fx, "lib"); math.Abs(got.KeepAliveSavedUSD-0.01) > 1e-9 { + t.Errorf("library-priced saved = %.6f, want 0.01: the stored decision is trusted", got.KeepAliveSavedUSD) + } + if got := kaSession(t, fx, "old"); got.KeepAliveSavedUSD != 0 { + t.Errorf("legacy saved = %.6f, want 0: the SQL gate still judges a row with no marker", got.KeepAliveSavedUSD) + } +} + +// The credited-row guard must come before every probe (a 197x slowdown, see +// kaSavedGate), so the marker test sits inside the same AND after `col > 0`. +func TestKaSavedGateKeepsTheCreditGuardFirst(t *testing.T) { + g := kaSaved("r.") + if !strings.HasPrefix(g, "(CASE WHEN r.keepalive_saved_usd > 0 AND (r.acc_priced = 1 OR ") { + t.Fatalf("the guard order changed: %.120s", g) + } + db := openTestDB(t) + // The probe must still seek the ping by its session index, and the marker must be a plain column read. + plan := explainPlan(t, db, "SELECT SUM("+g+") FROM requests r WHERE r.ts >= ?", []any{0}) + if !strings.Contains(strings.Join(plan, "\n"), "idx_requests_tenant_session") { + t.Errorf("the ping probe no longer seeks idx_requests_tenant_session:\n%v", plan) + } +} + +func kaSession(t *testing.T, fx *kaFixture, id string) *SessionRow { + t.Helper() + rows, _, err := fx.db.Sessions(Filter{TenantAll: true}, 50, 0) + if err != nil { + t.Fatal(err) + } + for _, r := range rows { + if r.SessionID == id { + return r + } + } + t.Fatalf("session %s is missing from the list", id) + return nil +} diff --git a/dash/query.go b/dash/query.go index d6e82d82..5d4dcec2 100644 --- a/dash/query.go +++ b/dash/query.go @@ -193,13 +193,13 @@ var requestCols = `r.id, r.ts, r.tenant_id, r.session_id, r.model, r.provider, r r.agent_compaction, r.cache_miss_cause, r.manual_ping, r.manual_saved_usd, r.project, r.source, r.sweep_expands, r.sweep_expand_tokens, r.sweep_false_expands, r.sweep_answer_usd, - r.session_alias, r.price_estimated, + r.session_alias, r.price_estimated, r.acc_priced, r.defer_tokens, r.defer_list_tokens, r.defer_searches, r.defer_loaded, r.defer_wrong_guesses, r.defer_gross_usd, r.defer_net_usd, r.defer_dropped_searches` func scanRequest(rows interface{ Scan(...any) error }) (*Event, error) { var e Event - var byp, ca, stream, keepAlive, agentComp, manual, estimated int + var byp, ca, stream, keepAlive, agentComp, manual, estimated, accPriced int var temp, topP sql.NullFloat64 var waitingFor, waitTool, wakeCause sql.NullString var openBackground, openTeammate, monitorExpiryMs sql.NullInt64 @@ -219,10 +219,10 @@ func scanRequest(rows interface{ Scan(...any) error }) (*Event, error) { &agentComp, &e.CacheMissCause, &manual, &e.ManualSavedUSD, &e.Project, &e.Source, &e.SweepExpands, &e.SweepExpandTokens, &e.SweepFalseExpands, &e.SweepAnswerUSD, - &e.SessionAlias, &estimated, + &e.SessionAlias, &estimated, &accPriced, &e.DeferTokens, &e.DeferListTokens, &e.DeferSearches, &e.DeferLoaded, &e.DeferWrongGuesses, &e.DeferGrossUSD, &e.DeferNetUSD, &e.DeferDroppedSearches) - e.PriceEstimated = estimated != 0 + e.PriceEstimated, e.AccountantPriced = estimated != 0, accPriced != 0 e.WaitingFor, e.WaitTool, e.WakeCause = waitingFor.String, waitTool.String, wakeCause.String e.OpenBackground, e.OpenTeammate = int(openBackground.Int64), int(openTeammate.Int64) e.MonitorExpiryMs = monitorExpiryMs.Int64 diff --git a/dash/schema.go b/dash/schema.go index c1e4d5fc..01900ab9 100644 --- a/dash/schema.go +++ b/dash/schema.go @@ -105,6 +105,10 @@ CREATE TABLE IF NOT EXISTS requests ( -- net saving may be too high: the gross is right, the penalty is not known. 0 on an exact figure -- and on every row written before this column existed, which is the claim those rows always made. price_estimated INTEGER NOT NULL DEFAULT 0, + -- 1 when the accountant override applied to the request, so keepalive_saved_usd is the library's own + -- rescue decision and the read-side gate trusts it. 0 on a request the library did not price and on + -- every row written before this column existed, which keep the SQL reachability gate. + acc_priced INTEGER NOT NULL DEFAULT 0, preset TEXT NOT NULL DEFAULT '', mode TEXT NOT NULL DEFAULT '', -- operating mode (active|bypass|observe) route TEXT NOT NULL DEFAULT '', @@ -765,6 +769,7 @@ var additiveColumns = []struct{ table, column, ddl string }{ {"requests", "decl_digest", "TEXT NOT NULL DEFAULT ''"}, {"requests", "session_alias", "TEXT NOT NULL DEFAULT ''"}, {"requests", "price_estimated", "INTEGER NOT NULL DEFAULT 0"}, + {"requests", "acc_priced", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "cache_ttl", "TEXT NOT NULL DEFAULT ''"}, {"requests", "sse_buffered", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "ttfb_ms", "REAL NOT NULL DEFAULT 0"}, diff --git a/dash/store.go b/dash/store.go index 4d016356..ec0f1823 100644 --- a/dash/store.go +++ b/dash/store.go @@ -593,12 +593,12 @@ func (d *DB) insertBatch(evs []*Event) error { waiting_for, wait_tool, wake_cause, open_background, open_teammate, monitor_expiry_ms, agent_compaction, cache_miss_cause, manual_ping, manual_saved_usd, project, source, sweep_expands, sweep_expand_tokens, sweep_false_expands, sweep_answer_usd, session_alias, - price_estimated, + price_estimated, acc_priced, defer_tokens, defer_list_tokens, defer_searches, defer_loaded, defer_wrong_guesses, defer_gross_usd, defer_net_usd, defer_dropped_searches ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, - ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) + ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { return err } @@ -673,7 +673,7 @@ func (d *DB) insertBatch(evs []*Event) error { * safe direction — it claims LESS about the row. */ sourceOf(e.Source), e.SweepExpands, e.SweepExpandTokens, e.SweepFalseExpands, e.SweepAnswerUSD, - e.SessionAlias, boolInt(e.PriceEstimated), + e.SessionAlias, boolInt(e.PriceEstimated), boolInt(e.AccountantPriced), e.DeferTokens, e.DeferListTokens, e.DeferSearches, e.DeferLoaded, e.DeferWrongGuesses, e.DeferGrossUSD, e.DeferNetUSD, e.DeferDroppedSearches, ) diff --git a/docs/dashboard.md b/docs/dashboard.md index e79002dc..f75224a5 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -115,6 +115,15 @@ What the library prices: - **Keep-alive.** `keepalive_saved_usd` is the library's rescue. The proxy passes each ping's usage, and the library derives the extent the ping read from it. A prefix that another thread of the session kept warm inside the TTL is not credited (`EntryFacts.SharedWarm`, accountant R54). + When the override applies, the request stores `requests.acc_priced = 1` (sent as `acc_priced`). The + keep-alive views trust the stored `keepalive_saved_usd` of such a request as the library's own + decision, with no ping probe and with no component row needed, so a keep-alive-only setup keeps its + rescue. A row with `acc_priced = 0` (older rows, and requests the library could not price) keeps the + read-side reachability test. The entry TTL given to the library follows the billed tier: if the + request asked for 1h and the response billed 5m, the library judges the next gap against 5 minutes. +- **Tool-filter credit.** A request whose `toolfilter` row is `priced_by = accountant` adds no dollars + to the measured filter credit (the net saving has them). It still counts in the request count, the + tokens not sent and the first date, so the Realized block and the item list stay. - **Cachesplit.** `cachesplit_saved_usd` is the library's split effect. - **Head TTL.** The 1h premium and the rescue it earns are on the `headttl` row. - **A break nobody owns.** A break that a change left behind when it ended is counted on the diff --git a/docs/data-dependence.md b/docs/data-dependence.md index fc6fe446..9a0d4666 100644 --- a/docs/data-dependence.md +++ b/docs/data-dependence.md @@ -219,7 +219,7 @@ One row for each registered component. The test `components/all/datadep_test.go` | **Keep-alive pings** | content. The host keeps the credential. | ❌ **Should be off. Today it is still on**, and keeps the last body. The proxy binary also keeps the credential. | ❌ Off | ✅ Works | To keep the prompt cache warm, the proxy sends the exact same prompt again with the user's credential. Without the text, it cannot ping. The Ping button (`POST /api/me/cache/ping`) uses the same body and key. With keep-alive off, a request that carries the header `x-context-guru-ping: 1` is still held, for the button only: no ping is sent for it, and the body and key are released 300 seconds after the last request on a 5-minute cache, or 59 minutes after it on a 1-hour cache (a press does not extend it), and one account holds at most 16 such threads. Without the header nothing is held. The install adds the header and tells the user. An older install (from before `--no-ping-header`) gets it at its next session start, with the same notice. Any other install without the header is never changed. It gets a one-time note instead. The cache-state answer also carries the time of the next planned ping (a number, metadata), so the mod can hide the button while a ping fires before the cache expires. | | **Body stash** | content | ❌ **Should be off. Today it is still on**, with 2 full bodies per thread. | ❌ Off. The sweep's question then cannot be asked, and `cache_aware_summarizer` stops. | ✅ Works | To ask the model a cheap question, Context Guru adds the question to the exact text it sent last turn. The provider already cached that text. The sweep keeps **2 more full copies of the conversation per thread**, outside the stash, for a thread the keep-alive can ping (see `extract_llm_sweep` above). | | Observe-mode store | content | ❌ **Should follow the level. Today it is still a real Store.** | It should follow the level | ✅ Works | Observe mode runs the pipeline later on a copy, to estimate savings. The copy should store only what the real run would store. | -| Dashboard database | content and metadata | ❌ Off | 🟡 Works, metadata: numbers, fixed codes, ids, and tool, MCP-server and skill names. **No** before/after text, summaries, declaration text, error text or rejection text. | ✅ Works. It keeps text only when content capture is on. | The dashboard shows what Context Guru removed and what your setup carries. Every number on it needs only counts and prices. Every request row comes from traffic the proxy saw itself. On a local install the dashboard also reads two things from your disk, and stores neither: the working folder recorded in Claude Code's session files, to group folders into projects for the project picker, and each installed skill's description from its `SKILL.md`. The keep-alive calculator and recommendation (#868) replay the per-wait ping plan from columns the proxy already writes for each request: stop reason, wait cause, thread id, cache tier and provider. They read no text. Tool deferral's net-saving panel (#869) adds eight count and price columns per request: tokens deferred, the names list's tokens, searches, dropped searches, tools loaded, wrong guesses, and the gross and net dollars. The tool inventory also records the built-in tools Claude Code defers (by name, with 0 tokens) and counts a tool loaded through search as a use. `save-more tools` reads the same inventory, the tool deferral history and the account's keep and always-defer lists. All of it is tool names, counts and prices, and works at the metadata level. An earlier history import copied whole request rows from those session files; it is removed, and its rows are deleted when the database opens. | +| Dashboard database | content and metadata | ❌ Off | 🟡 Works, metadata: numbers, fixed codes, ids, and tool, MCP-server and skill names. **No** before/after text, summaries, declaration text, error text or rejection text. | ✅ Works. It keeps text only when content capture is on. | The dashboard shows what Context Guru removed and what your setup carries. Every number on it needs only counts and prices. Every request row comes from traffic the proxy saw itself. On a local install the dashboard also reads two things from your disk, and stores neither: the working folder recorded in Claude Code's session files, to group folders into projects for the project picker, and each installed skill's description from its `SKILL.md`. The keep-alive calculator and recommendation (#868) replay the per-wait ping plan from columns the proxy already writes for each request: stop reason, wait cause, thread id, cache tier and provider. They read no text. Tool deferral's net-saving panel (#869) adds eight count and price columns per request: tokens deferred, the names list's tokens, searches, dropped searches, tools loaded, wrong guesses, and the gross and net dollars. The tool inventory also records the built-in tools Claude Code defers (by name, with 0 tokens) and counts a tool loaded through search as a use. `save-more tools` reads the same inventory, the tool deferral history and the account's keep and always-defer lists. All of it is tool names, counts and prices, and works at the metadata level. Each request row also has one flag, `acc_priced` (context-guru#912), set when the accountant library priced the request. The keep-alive views read it to trust the library's stored rescue figure. It is a yes or no, with no text. An earlier history import copied whole request rows from those session files; it is removed, and its rows are deleted when the database opens. | | Dashboard memos | content | ❌ Off, with the dashboard | 🟡 Works, metadata: a hash and a token count. That is all they need. | ✅ Works. They keep declaration and system-prompt text. | They remember a tool list so they do not count its tokens again on every request. A hash and a count are enough. | | Archive | content and metadata | ❌ Off | 🟡 Works, metadata rows only | ✅ Works | It moves old conversations off the local disk. The database stays small, and you can still open old history. | | Insights | metadata | ❌ Off (it needs the dashboard) | 🟡 Works, metadata | 🟡 Works, metadata | A cost report needs only counts, tool names and prices, not the text. | diff --git a/proxy/accountant_ping_test.go b/proxy/accountant_ping_test.go index 49f47f0a..cdd44b97 100644 --- a/proxy/accountant_ping_test.go +++ b/proxy/accountant_ping_test.go @@ -335,3 +335,38 @@ func TestPingAtIsTheStartOfThePing(t *testing.T) { t.Fatalf("a ping with no duration: At = %v, want TS", got) } } + +// A gateway that asks for 1h and bills 5m must not make a 20-minute gap look hot (context-guru#919). +// The entry TTL is lowered to the billed tier after the usage arrives, before the library prices the +// request. A request billed 1h, or one that wrote nothing, keeps the asked 1h. +func TestTheEntryTTLFollowsTheBilledTier(t *testing.T) { + t0 := time.Unix(1_700_000_000, 0) + body := []byte(`{"model":"claude-sonnet-5","system":[{"type":"text","text":"` + strings.Repeat("x", 4000) + + `","cache_control":{"type":"ephemeral","ttl":"1h"}}],"messages":[{"role":"user","content":"hi"}]}`) + phaseAfter := func(u Usage) accountant.CachePhase { + st, pings := newSavingsTracker(), NewPingStore() + step := func(at time.Time, u Usage) accountant.Result { + c := &capture{savings: st, pings: pings, provider: string(bschemas.Anthropic)} + c.accBegin(accountant.RequestMeta{RequestID: at.String(), Session: "s", Model: "claude-sonnet-5", At: at}, body) + c.accEnd(thread.Result{}, body, nil) + if c.accReq.Facts.TTL != time.Hour { + t.Fatalf("asked 1h on every breakpoint, facts TTL = %v", c.accReq.Facts.TTL) + } + c.accBilledTTL(u) + return st.compute(c.accReq, u) + } + step(t0, u) + // The second turn bills a write, not a read: a billed read would beat the clock (accountant R45). + return step(t0.Add(20*time.Minute), Usage{CacheWrite: int64(len(body)), CacheWrite1h: int64(len(body))}).Phase + } + n := int64(len(body)) + if got := phaseAfter(Usage{CacheWrite: n, CacheWrite1h: 0}); got == accountant.PhaseHot { + t.Errorf("billed 5m, gap 20m: phase = %q, want not hot (the entry expired after 5 minutes)", got) + } + if got := phaseAfter(Usage{CacheWrite: n, CacheWrite1h: n}); got != accountant.PhaseHot { + t.Errorf("billed 1h, gap 20m: phase = %q, want hot", got) + } + if got := phaseAfter(Usage{CacheRead: n}); got != accountant.PhaseHot { + t.Errorf("wrote nothing, gap 20m: phase = %q, want hot (no tier billed: the asked 1h stands)", got) + } +} diff --git a/proxy/accountant_wire.go b/proxy/accountant_wire.go index 64f59adc..f62ae177 100644 --- a/proxy/accountant_wire.go +++ b/proxy/accountant_wire.go @@ -61,10 +61,9 @@ func (c *capture) accEnd(th thread.Result, forwarded []byte, ops []accountant.Op // An unknown lifetime cannot prove an expiry, so the library would never rescue after a gap past // the TTL (design §5.2). The TTL is 1h only when EVERY breakpoint of the forwarded body asks for it, // as the keeper decides: a 1h head over a 5m tail leaves a tail that expires in five minutes. The -// keeper also needs the billed tier, which is known only after the response; the facts are written -// before it, so a request that asks for 1h and is billed 5m keeps the 1h here. The library then -// treats a gap of 5 to 60 minutes as hot, which can overstate a rescue (an edge case: a gateway that -// downgrades the tier). +// keeper also needs the billed tier, which is known only after the response. These facts are written +// before it, so they hold the ASKED tier. accBilledTTL corrects it once the usage arrives, before the +// library prices the request (context-guru#919). func entryFacts(provider bschemas.ModelProvider, model string, forwarded []byte, pings *PingStore, session string, t accountant.Thread) accountant.EntryFacts { facts := accountant.EntryFacts{TTL: 5 * time.Minute, Lifetime: accountant.LifetimeUnknown} @@ -86,3 +85,32 @@ func entryFacts(provider bschemas.ModelProvider, model string, forwarded []byte, // accPingKey is the key the ping store keeps one thread's ping facts under: the session and the // thread id as the accountant sees it ("main" for the primary thread). func accPingKey(session, threadID string) string { return session + "\x00" + threadID } + +// accBilledTTL lowers the entry TTL from the tier the request asked for (1h) to the tier the response +// billed, when a gateway downgraded it. It runs after the usage arrives and before Compute, which reads +// the facts' TTL for the verdict and stores it as the entry's TTL. Without it the library treats a gap of +// 5 to 60 minutes as hot and can overstate a rescue by up to 55 minutes of idle time. +// +// The rule is keepalive.ResolveTier's: one hour needs every write billed at the one-hour tier. A request +// that wrote nothing bills no tier, so the asked tier stands. Only an exact lifetime has a TTL that can be +// lowered: a minimum (OpenAI) has no tier. SharedWarm was judged with the asked TTL at accEnd and is not +// redone; it matters only when another thread of the session touched the entry. Nil-safe. +func (c *capture) accBilledTTL(u Usage) { + if c == nil || c.accReq == nil { + return + } + f := &c.accReq.Facts + if f.Lifetime != accountant.LifetimeExact || f.TTL <= 5*time.Minute { + return + } + if u.CacheWrite > 0 && u.CacheWrite1h < u.CacheWrite { + f.TTL = 5 * time.Minute + // The library judges each part of the entry by the TTL of the breakpoint that wrote it, so the + // breakpoints must say 5m too. A copy: the slice may be the one the client hash cache holds. + bps := append([]accountant.Breakpoint(nil), c.accReq.Breakpoints...) + for i := range bps { + bps[i].TTL = "5m" + } + c.accReq.Breakpoints = bps + } +} diff --git a/proxy/dashcapture.go b/proxy/dashcapture.go index 2ab23ed9..086da3dc 100644 --- a/proxy/dashcapture.go +++ b/proxy/dashcapture.go @@ -731,6 +731,7 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content if c.accReq != nil && usageOK && !c.accDone { c.accDone = true if c.rounds <= 1 { + c.accBilledTTL(usage) if res, ok := c.savings.computeSafe(c.accReq, usage); ok { e.SetAccountant(res) } else { From a0a7f4b0b29c4a4ca412a4863316a02478f319c1 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:11:04 +0200 Subject: [PATCH 33/58] fix(accountant): Report the tool deferral names list as an operation (#908) Tool deferral runs after the pipeline reported its operations. It marks declarations defer_loading, adds the search tool, and puts a names list at messages[1]: appended to the system message there, or as a new system message. The tools half is found by the envelope reporter. The names list was reported by nobody, so the sum check failed on the messages section (ops=0 bodies=-308 bytes) and conservation failed with it, on every request of a deferred thread. proxy now diffs the body before and after applyToolDefer and reports the change as a tooldefer operation: a one-message splice for a new message, a rewrite in place for an appended list. tooldefer is an envelope operator in dash, so it gets its own row. No accountant check is weakened. Harness (FAKE_TOKDIV=4, MSGMULT=1500, td, 8 requests): check_sum fallbacks 8 -> 0, failed checks 8+7 -> 0. The Overview still does not credit the deferred declarations: the library sees bytes, and a declaration marked defer_loading keeps its bytes. net_saved_usd is -0.0006 (the names list and search tool as cost), truth is +0.0185. /api/tool-defer still reads 0.0412. Making the two agree needs a product decision, so it is not part of this change. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accountant_price.go | 2 +- proxy/accountant_envelope.go | 36 ++++++++++++++++++++++++ proxy/accountant_envelope_test.go | 22 +++++++++++++++ proxy/accountant_fallbacks_test.go | 45 ++++++++++++++++++++++++++++++ proxy/proxy.go | 6 ++++ 5 files changed, 110 insertions(+), 1 deletion(-) diff --git a/dash/accountant_price.go b/dash/accountant_price.go index 8565e983..1ff05f06 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -191,7 +191,7 @@ const EndedChangeComponent = "ended_change" var envelopeComponents = map[string]bool{ "expand": true, "adjudicate": true, "headttl": true, "toolschema": true, "toolfilter": true, "cachesplit": true, - "tailbreakpoint": true, + "tailbreakpoint": true, "tooldefer": true, } // AccountantEstimated reports whether the applied figure is an estimate: the library could not rule diff --git a/proxy/accountant_envelope.go b/proxy/accountant_envelope.go index 668d6eec..04f4c9c1 100644 --- a/proxy/accountant_envelope.go +++ b/proxy/accountant_envelope.go @@ -281,3 +281,39 @@ func hashOf(s string) string { } func accHex(v uint64) string { return strconv.FormatUint(v, 16) } + +// messageOps explains a change to the messages array that an edit AFTER the pipeline made, and that +// no component reported: tool deferral puts its names list at messages[1], either appended to a system +// message already there or as a new one (tooldefer placeList). before is the body the edit read and +// after the one it produced. A new message is a splice that inserts one; an appended list is a rewrite +// in place of that message. Any other change returns nil, and the sum check says so. The operator is +// the edit's own, so its row and its ChangeID do not collide with a pipeline component's. +func messageOps(operator string, before, after []byte) []accountant.Operation { + b, a := gjson.GetBytes(before, "messages").Array(), gjson.GetBytes(after, "messages").Array() + switch { + case len(a) == len(b)+1: + i := 0 + for i < len(b) && b[i].Raw == a[i].Raw { + i++ + } + for j := i; j < len(b); j++ { + if b[j].Raw != a[j+1].Raw { + return nil // not a single insertion + } + } + op := envelopeOp(operator, accountant.Position{Section: accountant.SectionMessages, Index: i}, + 0, len(a[i].Raw), "msg-inserted:"+strconv.Itoa(i)) + op.MessageSplice = &accountant.MessageSplice{At: i, Removed: 0, Inserted: 1} + return []accountant.Operation{op} + case len(a) == len(b): + var ops []accountant.Operation + for i := range b { + if b[i].Raw != a[i].Raw { + ops = append(ops, envelopeOp(operator, accountant.Position{Section: accountant.SectionMessages, Index: i}, + len(b[i].Raw), len(a[i].Raw), "msg-rewritten:"+strconv.Itoa(i)+":"+hashOf(b[i].Raw))) + } + } + return ops + } + return nil +} diff --git a/proxy/accountant_envelope_test.go b/proxy/accountant_envelope_test.go index 13ae0102..c7b5de8f 100644 --- a/proxy/accountant_envelope_test.go +++ b/proxy/accountant_envelope_test.go @@ -228,3 +228,25 @@ func assertPremiumBooked(t *testing.T, client, fwd []byte, ops []accountant.Oper } t.Fatalf("no ttlPremium effect: ops %+v, effects %+v", ops, res.CacheEffects) } + +func TestMessageOpsExplainTheDeferralList(t *testing.T) { + m0, m1 := `{"role":"user","content":"hi"}`, `{"role":"system","content":"env"}` + m1l := `{"role":"system","content":"env\n\nlist of names"}` + ins := `{"role":"system","content":"list of names"}` + for _, tc := range []struct { + name, before, after string + wantOps int + }{ + {"appended", `{"messages":[` + m0 + `,` + m1 + `]}`, `{"messages":[` + m0 + `,` + m1l + `]}`, 1}, + {"inserted", `{"messages":[` + m0 + `,` + m1 + `]}`, `{"messages":[` + m0 + `,` + ins + `,` + m1 + `]}`, 1}, + {"unchanged", `{"messages":[` + m0 + `]}`, `{"messages":[` + m0 + `]}`, 0}, + } { + ops := messageOps("tooldefer", []byte(tc.before), []byte(tc.after)) + if len(ops) != tc.wantOps { + t.Fatalf("%s: %d ops, want %d", tc.name, len(ops), tc.wantOps) + } + if got, want := opsDelta(ops, accountant.SectionMessages), sectionDelta([]byte(tc.before), []byte(tc.after), accountant.SectionMessages); got != want { + t.Errorf("%s: ops explain %d bytes of messages, the bodies differ by %d", tc.name, got, want) + } + } +} diff --git a/proxy/accountant_fallbacks_test.go b/proxy/accountant_fallbacks_test.go index 37c057c0..301aaaaa 100644 --- a/proxy/accountant_fallbacks_test.go +++ b/proxy/accountant_fallbacks_test.go @@ -4,10 +4,13 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "path/filepath" "strings" "testing" + "time" "github.com/rossoctl/context-guru/dash" + "github.com/rossoctl/context-guru/tooldefer" ) // accPostBody sends body to the handler under session and returns the response status. @@ -105,3 +108,45 @@ func TestRemovedToolDeclarationsDoNotTripThePlausibilityBound(t *testing.T) { } } } + +// Tool deferral (`defer: auto`) marks declarations defer_loading, adds the search tool and puts a +// names list at messages[1], all after the pipeline reported its operations. The tools half is the +// envelope reporter's; the names list is its own operation. Without it every request of a deferred +// thread failed check_sum and conservation (#908). +func TestToolDeferReportsItsNamesList(t *testing.T) { + up := billedUpstream(t) + defer up.Close() + for _, tc := range []struct { + name string + msgs []any + }{ + // Claude Code's shape: a system message already at messages[1], the list is appended to it. + {"appended", []any{map[string]any{"role": "user", "content": "hi"}, + map[string]any{"role": "system", "content": "# Environment\nwd: /w"}}}, + // No message at index 1: the list is a new system message. + {"inserted", []any{map[string]any{"role": "user", "content": "hi"}, + map[string]any{"role": "assistant", "content": "hello"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + h, rec, tap := accHandlerOpts(t, up.URL, + "pipeline: [toolfilter]\ncomponents:\n toolfilter:\n defer: auto\n always_defer: [mcp__big]\n", + func(o *Options) { o.ToolDeferPath = filepath.Join(t.TempDir(), tooldefer.FileName) }) + // The provider check of deferral runs on the first request; seed it as passed. + h.toolDefer().prober.Seed(tooldefer.ProbeKey{Upstream: up.URL, Model: "claude-sonnet-5-5", Route: "/v1/messages"}, + tooldefer.Pass, time.Now()) + decl := func(name string, n int) map[string]any { + return map[string]any{"name": name, "description": strings.Repeat("Does a thing. ", n), + "input_schema": map[string]any{"type": "object", "properties": map[string]any{"q": map[string]any{"type": "string"}}}} + } + b, _ := json.Marshal(map[string]any{"model": "claude-sonnet-5-5", "max_tokens": 64, + "tools": []any{decl("bash", 3), decl("mcp__big", 300)}, + "messages": tc.msgs}) + accPostBody(t, h, "sess-defer", b) + accPostBody(t, h, "sess-defer", b) + waitForRows(t, rec, 2) + for _, e := range tap.all() { + accNoSumFailure(t, e) + } + }) + } +} diff --git a/proxy/proxy.go b/proxy/proxy.go index e9fe1abb..e9130f31 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -1588,7 +1588,13 @@ func (h *Handler) chat(provider bschemas.ModelProvider, static upstream, pick fu // Tool deferral (#485), LAST, on the final tools array: the proxy's own tools are on // the never-defer list, and the names list must see the tools as they go upstream. // Only the ultrahigh preset turns it on (toolfilter `defer: auto`). + preDefer := body body, di = h.applyToolDefer(r, tn, provider, wire, up, tr.Session, threadID, orig, body) + // Deferral adds its names list to the messages after the pipeline reported its + // operations. Report it as an operation of its own, after theirs (#908). + if di != nil && !bytes.Equal(preDefer, body) { + tr.Ops = append(tr.Ops[:len(tr.Ops):len(tr.Ops)], messageOps("tooldefer", preDefer, body)...) + } } // The savings record ends HERE, after both injections, with the body that goes // upstream: the provider bills the injected tool declarations, so the library's From ff140deb17a7c6ecf26cb538583a8b6a38958a24 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:17:44 +0200 Subject: [PATCH 34/58] fix(accountant): Hold a no-history estimate out of the totals (#915) The first request on a thread the proxy has no history for (a restart with a missing or stale state file, or a session begun before the proxy) is priced by the library as if the provider would have written every removed token. If the cache was in fact warm, that is about 2x too high. It was flagged estimated, but still went into baseline_cost_usd, net_saved_usd and total_saved_usd. In run_m5, request 1 booked baseline 0.0424785 (saving 0.042015); the hand figure is 0.021569. Choice: option (a), not (b). The proxy cannot price the unseen prefix at the read tier because the size of that prefix is the very thing the missing history held. The usage only shows how far the compacted body matched, not how much the client had cached. A hand figure needs that size, and the proxy does not have it. So the proxy books no saving on an estimated request, and keeps the figure. Changes: - PriceMode: on an estimated request baseline_cost_usd equals cost_usd and every component saved_usd, saved_first_usd and saved_replay_usd is 0. The held figure goes to the new requests.estimated_saved_usd (Event.EstimatedSavedUSD, additive column, 0 on old rows). The keep-alive rescue and the split saving do not depend on a break and stay booked. - Overview and the lean path sum it as estimated_saved_usd, beside estimated_requests. It is in no total, so net saved = total saved = sum of components = keep-alive tab, with no filter. - UI: the Preset card says how many requests are not counted and shows the held figure. The request detail shows it on its own line marked with an approximate sign. - docs/dashboard.md describes the stored field and the rule. Numbers (run_m5, same binary and harness): net_saved_usd 0.0911627 before, 0.0491477 after. Request 1 baseline 0.0424785 before, 0.0004635 (equal to cost) after, with 0.042015 held in estimated_saved_usd. Requests 2 and 3 are unchanged (0.0243054, 0.0257421). Golden files: stats.json, stats_lean.json and stats_windowed.json gain estimated_saved_usd: 0. No golden fixture holds an estimated row, so no other number moves. Test changes: TestSavingsRecordEndsWithTheInjectedForwardedBody and TestDashboardBooksTheAccountantSaving sent the first request of a thread and read its booked saving. They now assert the estimate is held out; the second one sends a second request and asserts the booked saving there. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accountant_price.go | 18 +++++++- dash/accountant_switch_test.go | 17 +++++++ dash/accounting_test.go | 57 ++++++++++++++++++++++++ dash/event.go | 14 ++++-- dash/overview.go | 9 +++- dash/overview_lean.go | 3 +- dash/query.go | 4 +- dash/schema.go | 4 ++ dash/store.go | 6 +-- dash/testdata/golden/stats.json | 1 + dash/testdata/golden/stats_lean.json | 1 + dash/testdata/golden/stats_windowed.json | 1 + dash/ui/app.js | 11 +++-- docs/dashboard.md | 18 ++++++-- proxy/accountant_inject_test.go | 15 ++++++- proxy/dash_test.go | 25 ++++++++++- 16 files changed, 181 insertions(+), 23 deletions(-) diff --git a/dash/accountant_price.go b/dash/accountant_price.go index 4b64d455..dbb76784 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -214,7 +214,7 @@ func (e *Event) AccountantFallback() string { return e.accFallback } // library figure. Otherwise the event books no saving (Price set every saving to 0) and // AccountantFallback says why: a figure nothing measured is not shown. func (e *Event) PriceMode(p modelinfo.Price, accountingComplete bool) { - e.accFallback, e.PriceEstimated = "", false + e.accFallback, e.PriceEstimated, e.EstimatedSavedUSD = "", false, 0 if !e.priceCost(p, accountingComplete) { e.accFallback = FallbackUnpriced return @@ -331,6 +331,22 @@ func (e *Event) accOverride(p modelinfo.Price) string { c.SavedFirstUSD, c.SavedReplayUSD = fig.ByComponentFirst[c.Component], fig.ByComponentReplay[c.Component] } e.BaselineCostUSD = e.CostUSD + fig.DeltaUSD + if estimated { + // The library could not rule out a break: it had no history for the thread, so it priced + // each removal as a write the provider would have made. When the cache was in fact warm + // (a restart that lost the state file, or a session begun before the proxy) the figure is + // about 2x too high, and the proxy cannot say by how much: the size of the unseen warm + // prefix is the thing the missing history held. A figure nothing measured is not booked + // (issue #915). The request books no break-affected saving, and the figure is kept in + // EstimatedSavedUSD, outside every total, so the Overview, the components table, the + // keep-alive tab and the API still agree. The keep-alive rescue and the split saving do not + // depend on a break, so they stay booked below. + e.EstimatedSavedUSD = fig.DeltaUSD + e.BaselineCostUSD = e.CostUSD + for i := range e.Components { + e.Components[i].SavedUSD, e.Components[i].SavedFirstUSD, e.Components[i].SavedReplayUSD = 0, 0, 0 + } + } // The keep-alive rescue and the split saving keep their own columns, outside the baseline, as // the legacy rules had them: every reader of keepalive_saved_usd and cachesplit_saved_usd stays valid. e.KeepAliveSavedUSD, e.CachesplitSavedUSD = fig.KeepAliveSavedUSD, fig.SplitUSD diff --git a/dash/accountant_switch_test.go b/dash/accountant_switch_test.go index 801d652d..88a071bc 100644 --- a/dash/accountant_switch_test.go +++ b/dash/accountant_switch_test.go @@ -236,6 +236,18 @@ func TestPriceMode_PenaltyUnknownIsAnEstimate(t *testing.T) { if fb := e.AccountantFallback(); fb != "" || !e.AccountantEstimated() { t.Fatalf("fallback %q estimated %v, want the override applied as an estimate", fb, e.AccountantEstimated()) } + // Issue #915: the estimate is held apart. The books get no saving from it. + if e.BaselineCostUSD != e.CostUSD { + t.Errorf("an estimated request booked a baseline delta of %v, want 0", e.BaselineCostUSD-e.CostUSD) + } + for _, c := range e.Components { + if c.SavedUSD != 0 || c.SavedFirstUSD != 0 || c.SavedReplayUSD != 0 { + t.Errorf("component %s booked %v from an estimated request, want 0", c.Component, c.SavedUSD) + } + } + if want := 1000 * 0.3e-6; !accNear(e.EstimatedSavedUSD, want) { + t.Errorf("EstimatedSavedUSD = %v, want the held figure %v", e.EstimatedSavedUSD, want) + } e.Price(accTestPrice, true) if e.AccountantEstimated() { t.Fatal("Price left the estimate flag set") @@ -331,8 +343,13 @@ func TestPriceEstimatedIsStoredAndReadBack(t *testing.T) { t.Fatal(err) } got := map[string]bool{} + held := map[string]float64{} for _, e := range page.Requests { got[e.SessionID] = e.PriceEstimated + held[e.SessionID] = e.EstimatedSavedUSD + } + if !accNear(held["est"], est.EstimatedSavedUSD) || est.EstimatedSavedUSD == 0 || held["exact"] != 0 { + t.Fatalf("stored estimated_saved_usd = %v, want %v on est and 0 on exact", held, est.EstimatedSavedUSD) } if len(got) != 2 || !got["est"] || got["exact"] { t.Fatalf("stored price_estimated = %v, want est true and exact false", got) diff --git a/dash/accounting_test.go b/dash/accounting_test.go index b3b18dfd..0449dad2 100644 --- a/dash/accounting_test.go +++ b/dash/accounting_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/rossoctl/accountant" "github.com/rossoctl/context-guru/internal/modelinfo" ) @@ -1177,3 +1178,59 @@ func TestOverviewCountsEstimatedRequests(t *testing.T) { t.Fatalf("estimated requests: full %d, lean %d, want 2 and 2 (of %d)", full.EstimatedRequests, lean.EstimatedRequests, full.Requests) } } + +// Issue #915. An estimated request's saving is held OUT of every total. The Overview's net and total +// saved, the components table and the API all read the same booked rows, so they agree, and the held +// figure is shown on its own beside them. The full and the lean path report the same figures. +func TestOverviewKeepsEstimatedSavingsOutOfTheTotals(t *testing.T) { + db := openTestDB(t) + now := time.Now().UnixMilli() + price := modelinfo.Price{Input: 3e-6, CacheRead: 0.3e-6, CacheWrite: 3.75e-6, Output: 15e-6} + for i, est := range []bool{true, false} { + e := &Event{TS: now - int64(i)*1000, SessionID: "s1", Model: "m1", TenantID: "t1", + TokensBefore: 5000, TokensAfter: 4000, FreshInput: 100, CacheRead: 1000, OutputTokens: 10, + Components: []CompRow{{Component: "mask", Acted: true, Mutated: true, SavedGross: 1000, SavedUnique: 1000}}} + e.SetAccountant(accountant.Result{Ops: []accountant.PricedOp{{Operator: "context-guru.mask", ChangeID: "c", + Saved: accountant.TierTokens{CacheWrite5m: 1000}}}}) + if est { + e.accRes.Unpriced = []accountant.Unpriced{{Reason: accountant.ReasonPenaltyUnknown}} + } + e.PriceMode(price, true) + insertReq(t, db, e) + } + f := Filter{TenantAll: true, Since: now - 60_000} + exact := 1000 * 3.75e-6 + for name, get := range map[string]func() (*Overview, error){ + "full": func() (*Overview, error) { return db.Overview(f) }, + "lean": func() (*Overview, error) { return db.OverviewLean(f, nil, nil) }, + } { + o, err := get() + if err != nil { + t.Fatal(err) + } + if o.EstimatedRequests != 1 || !accNear(o.EstimatedSavedUSD, exact) { + t.Errorf("%s: estimated requests %d, held %v, want 1 and %v", name, o.EstimatedRequests, o.EstimatedSavedUSD, exact) + } + if got := o.BaselineCostUSD - o.CostUSD; !accNear(got, exact) { + t.Errorf("%s: baseline delta %v, want only the exact request's %v", name, got, exact) + } + } + o, err := db.Overview(f) + if err != nil { + t.Fatal(err) + } + if !accNear(o.NetSavedUSD, exact) || !accNear(o.TotalSavedUSD, exact) { + t.Errorf("net %v total %v, want both %v: the estimate must not be in either", o.NetSavedUSD, o.TotalSavedUSD, exact) + } + comps, err := db.Components(f) + if err != nil { + t.Fatal(err) + } + var sum float64 + for _, c := range comps { + sum += c.SavedUSD + } + if !accNear(sum, o.NetSavedUSD) { + t.Errorf("components sum %v != net saved %v", sum, o.NetSavedUSD) + } +} diff --git a/dash/event.go b/dash/event.go index c2086900..910b3d9b 100644 --- a/dash/event.go +++ b/dash/event.go @@ -444,9 +444,17 @@ type Event struct { // change broke the cache (penalty_unknown): the gross saving is right and the net may be too // high. It is stored, so the dashboard marks such a figure after a restart. False on an exact // figure, on an event that books no saving, and on a row from before the column existed. - PriceEstimated bool `json:"price_estimated,omitempty"` - CacheMissReason string `json:"cache_miss_reason"` - UncompressedReason string `json:"uncompressed_reason"` + PriceEstimated bool `json:"price_estimated,omitempty"` + // EstimatedSavedUSD is the saving the library priced on an estimated request, held apart from + // the books (issue #915). With no history the library prices every removal as if the provider + // would have written it, so after a restart that lost the state, or on a session that began on a + // cache the proxy never saw, the figure can be about 2x too high. Such a request books + // BaselineCostUSD == CostUSD and every component SavedUSD 0, so every total, the components table + // and the keep-alive tab agree without a filter. This field keeps the figure that was not booked. + // 0 on an exact request and on a row from before the column existed. + EstimatedSavedUSD float64 `json:"estimated_saved_usd,omitempty"` + CacheMissReason string `json:"cache_miss_reason"` + UncompressedReason string `json:"uncompressed_reason"` // CacheMissCause says what changed on a miss that is not hit, cold_start or ttl_expiry. // Set only from a measured comparison (Cause* constants). "" when nothing was measured. CacheMissCause string `json:"cache_miss_cause,omitempty"` diff --git a/dash/overview.go b/dash/overview.go index 17e0022c..55c77cfa 100644 --- a/dash/overview.go +++ b/dash/overview.go @@ -421,6 +421,11 @@ type Overview struct { // a component broke the cache (requests.price_estimated): the first request of a thread, and the first one // after a restart that lost the state. Their net can be too high. Overview.Requests is the denominator. EstimatedRequests int64 `json:"estimated_requests"` + // EstimatedSavedUSD is what those requests would have saved, held OUT of every total (#915): + // NetSavedUSD, TotalSavedUSD, the components and the keep-alive tab never include it. It is + // what the library priced without knowing whether the cache was already warm, and that can be + // about 2x too high. Shown beside the totals, never added to them. + EstimatedSavedUSD float64 `json:"estimated_saved_usd"` // BilledInputTokens is fresh + cache reads + cache writes: the input the PROVIDER counted. // It is here to be compared with TokensBefore, which is what our own tokenizer counted // over message text only — a different unit, roughly a third the size, and the reason no @@ -673,7 +678,7 @@ func (d *DB) Overview(f Filter) (*Overview, error) { COALESCE(SUM(CASE WHEN r.billed_token_factor > 0 AND r.baseline_cost_usd > r.cost_usd THEN 1 ELSE 0 END),0), `+deferNetSum+`, - COALESCE(SUM(r.price_estimated),0) + COALESCE(SUM(r.price_estimated),0), COALESCE(SUM(r.estimated_saved_usd),0) FROM requests r WHERE `+cond, args...).Scan( &o.Requests, &o.Sessions, &o.TokensBefore, &o.TokensAfter, &o.SavedUnique, &o.AttemptedTokens, &o.FrozenTokens, &o.SavedGrossAttempted, &o.AttemptedRequests, @@ -691,7 +696,7 @@ func (d *DB) Overview(f Filter) (*Overview, error) { &o.SSERecorded, &o.SSEStreamRows, &o.CacheTTLRecorded, &o.KeepAliveSavedUSD, &o.KeepAliveMissesAvoided, &o.CacheWrite1h, &o.PreCorrectionSavingRows, &o.PreCorrectionSavedUSD, &o.CorrectedSavingRows, - &o.DeferNetUSD, &o.EstimatedRequests) + &o.DeferNetUSD, &o.EstimatedRequests, &o.EstimatedSavedUSD) if err != nil { return nil, err } diff --git a/dash/overview_lean.go b/dash/overview_lean.go index 27479efb..b4544cde 100644 --- a/dash/overview_lean.go +++ b/dash/overview_lean.go @@ -68,7 +68,7 @@ func (d *DB) overviewLeanWindow(f Filter) (*Overview, error) { &o.Requests, &o.TokensBefore, &o.TokensAfter, &o.SavedUnique, &o.CostUSD, &o.BaselineCostUSD, &o.CGLLMCostUSD, &o.CacheSavedUSD, &o.CachesplitSavedUSD, &o.KeepAliveSavedUSD, &o.KeepAliveMissesAvoided, &upAvg, &cgAvg, &o.KeepAlivePingUSD, &o.KeepAlivePings, - &o.EstimatedRequests}, + &o.EstimatedRequests, &o.EstimatedSavedUSD}, append(manual.dest(), &o.DeferNetUSD)...)...); err != nil { return nil, err } @@ -117,6 +117,7 @@ func leanWindowMainQuery(f Filter) (string, []any) { COALESCE(SUM(CASE WHEN ` + autoPing + ` THEN r.cost_usd ELSE 0 END),0), COALESCE(SUM(CASE WHEN ` + autoPing + ` THEN 1 ELSE 0 END),0), COALESCE(SUM(CASE WHEN ` + agent + ` THEN r.price_estimated ELSE 0 END),0), + COALESCE(SUM(CASE WHEN ` + agent + ` THEN r.estimated_saved_usd ELSE 0 END),0), ` + manualSelect + `, COALESCE(SUM(CASE WHEN ` + agent + ` THEN r.defer_net_usd ELSE 0 END),0) FROM requests r WHERE ` + cond, args diff --git a/dash/query.go b/dash/query.go index d6e82d82..439e2b05 100644 --- a/dash/query.go +++ b/dash/query.go @@ -193,7 +193,7 @@ var requestCols = `r.id, r.ts, r.tenant_id, r.session_id, r.model, r.provider, r r.agent_compaction, r.cache_miss_cause, r.manual_ping, r.manual_saved_usd, r.project, r.source, r.sweep_expands, r.sweep_expand_tokens, r.sweep_false_expands, r.sweep_answer_usd, - r.session_alias, r.price_estimated, + r.session_alias, r.price_estimated, r.estimated_saved_usd, r.defer_tokens, r.defer_list_tokens, r.defer_searches, r.defer_loaded, r.defer_wrong_guesses, r.defer_gross_usd, r.defer_net_usd, r.defer_dropped_searches` @@ -219,7 +219,7 @@ func scanRequest(rows interface{ Scan(...any) error }) (*Event, error) { &agentComp, &e.CacheMissCause, &manual, &e.ManualSavedUSD, &e.Project, &e.Source, &e.SweepExpands, &e.SweepExpandTokens, &e.SweepFalseExpands, &e.SweepAnswerUSD, - &e.SessionAlias, &estimated, + &e.SessionAlias, &estimated, &e.EstimatedSavedUSD, &e.DeferTokens, &e.DeferListTokens, &e.DeferSearches, &e.DeferLoaded, &e.DeferWrongGuesses, &e.DeferGrossUSD, &e.DeferNetUSD, &e.DeferDroppedSearches) e.PriceEstimated = estimated != 0 diff --git a/dash/schema.go b/dash/schema.go index c1e4d5fc..1172c5f0 100644 --- a/dash/schema.go +++ b/dash/schema.go @@ -105,6 +105,9 @@ CREATE TABLE IF NOT EXISTS requests ( -- net saving may be too high: the gross is right, the penalty is not known. 0 on an exact figure -- and on every row written before this column existed, which is the claim those rows always made. price_estimated INTEGER NOT NULL DEFAULT 0, + -- the saving the accountant library priced on an estimated request, kept OUT of baseline_cost_usd + -- and every total (#915). 0 on an exact request and on every row written before this column. + estimated_saved_usd REAL NOT NULL DEFAULT 0, preset TEXT NOT NULL DEFAULT '', mode TEXT NOT NULL DEFAULT '', -- operating mode (active|bypass|observe) route TEXT NOT NULL DEFAULT '', @@ -765,6 +768,7 @@ var additiveColumns = []struct{ table, column, ddl string }{ {"requests", "decl_digest", "TEXT NOT NULL DEFAULT ''"}, {"requests", "session_alias", "TEXT NOT NULL DEFAULT ''"}, {"requests", "price_estimated", "INTEGER NOT NULL DEFAULT 0"}, + {"requests", "estimated_saved_usd", "REAL NOT NULL DEFAULT 0"}, {"requests", "cache_ttl", "TEXT NOT NULL DEFAULT ''"}, {"requests", "sse_buffered", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "ttfb_ms", "REAL NOT NULL DEFAULT 0"}, diff --git a/dash/store.go b/dash/store.go index 4d016356..0b13a7c7 100644 --- a/dash/store.go +++ b/dash/store.go @@ -593,12 +593,12 @@ func (d *DB) insertBatch(evs []*Event) error { waiting_for, wait_tool, wake_cause, open_background, open_teammate, monitor_expiry_ms, agent_compaction, cache_miss_cause, manual_ping, manual_saved_usd, project, source, sweep_expands, sweep_expand_tokens, sweep_false_expands, sweep_answer_usd, session_alias, - price_estimated, + price_estimated, estimated_saved_usd, defer_tokens, defer_list_tokens, defer_searches, defer_loaded, defer_wrong_guesses, defer_gross_usd, defer_net_usd, defer_dropped_searches ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, - ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) + ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { return err } @@ -673,7 +673,7 @@ func (d *DB) insertBatch(evs []*Event) error { * safe direction — it claims LESS about the row. */ sourceOf(e.Source), e.SweepExpands, e.SweepExpandTokens, e.SweepFalseExpands, e.SweepAnswerUSD, - e.SessionAlias, boolInt(e.PriceEstimated), + e.SessionAlias, boolInt(e.PriceEstimated), e.EstimatedSavedUSD, e.DeferTokens, e.DeferListTokens, e.DeferSearches, e.DeferLoaded, e.DeferWrongGuesses, e.DeferGrossUSD, e.DeferNetUSD, e.DeferDroppedSearches, ) diff --git a/dash/testdata/golden/stats.json b/dash/testdata/golden/stats.json index 559c6774..90ed407e 100644 --- a/dash/testdata/golden/stats.json +++ b/dash/testdata/golden/stats.json @@ -85,6 +85,7 @@ } ], "estimated_requests": 0, + "estimated_saved_usd": 0, "estimator_divergence": 1.001123848055743, "estimator_divergence_compacted": 1.0011973896904747, "estimator_divergence_compacted_rows": 197, diff --git a/dash/testdata/golden/stats_lean.json b/dash/testdata/golden/stats_lean.json index 559c6774..90ed407e 100644 --- a/dash/testdata/golden/stats_lean.json +++ b/dash/testdata/golden/stats_lean.json @@ -85,6 +85,7 @@ } ], "estimated_requests": 0, + "estimated_saved_usd": 0, "estimator_divergence": 1.001123848055743, "estimator_divergence_compacted": 1.0011973896904747, "estimator_divergence_compacted_rows": 197, diff --git a/dash/testdata/golden/stats_windowed.json b/dash/testdata/golden/stats_windowed.json index 33583bff..ec10ebc1 100644 --- a/dash/testdata/golden/stats_windowed.json +++ b/dash/testdata/golden/stats_windowed.json @@ -85,6 +85,7 @@ } ], "estimated_requests": 0, + "estimated_saved_usd": 0, "estimator_divergence": 1.0011168816663873, "estimator_divergence_compacted": 1.0012067457085105, "estimator_divergence_compacted_rows": 79, diff --git a/dash/ui/app.js b/dash/ui/app.js index f698f745..3db6c42e 100644 --- a/dash/ui/app.js +++ b/dash/ui/app.js @@ -1825,8 +1825,9 @@ function renderSavedLedger(o, ka, partial) { 'of ' + compact(inputWithout) + ' tokens', share(presetTok, inputWithout)) : null, capNote(o, usd) ? el('p', { class: 'ov-none ov-small', 'data-testid': 'replay-capped', text: capNote(o, usd) }) : null, o.estimated_requests > 0 ? el('p', { class: 'ov-none ov-small', 'data-testid': 'estimated-requests', - text: 'Includes ' + num(o.estimated_requests) + ' of ' + num(o.requests) + ' requests priced as an estimate: the first ' - + 'request of a thread, or the first after a restart that lost its state. Their net saving can be too high.' }) : null, + text: num(o.estimated_requests) + ' of ' + num(o.requests) + ' requests are not counted above: the first ' + + 'request of a thread, or the first after a restart that lost its state. The proxy cannot tell if the cache was already warm. ' + + 'Estimated saving, not in any total: ' + usd(o.estimated_saved_usd || 0) + '.' }) : null, ])); // Tools and skills filter. @@ -3707,8 +3708,10 @@ async function openRequest(id, fromURL) { // thread yet), so the gross saving is right and the net may be too high. Marked "≈", the // same way this page marks every other estimate. kv(term('Net saving', 'Baseline cost minus actual cost minus context-guru model cost.' - + (e.price_estimated ? ' An estimate: there was no history to tell whether a change broke the cache, so the net may be too high.' : '')), - priced ? (e.price_estimated ? '≈ ' : '') + usd(e.baseline_cost_usd - e.cost_usd - e.cg_llm_cost_usd) : '—'), + + (e.price_estimated ? ' Not counted: there was no history to tell whether the cache was already warm, so no saving is booked. The estimate is shown on the next line.' : '')), + priced ? usd(e.baseline_cost_usd - e.cost_usd - e.cg_llm_cost_usd) : '—'), + e.price_estimated ? kv(term('Estimated saving (not counted)', 'What the accountant would have booked with no history. It can be about 2x too high when the cache was already warm, so it is in no total.'), + priced ? '≈ ' + usd(e.estimated_saved_usd || 0) : '—') : null, // On a turn whose cache HIT this is usually the largest money figure on the row, // and it was not reported anywhere: the cache reads this request was billed for, // against the fresh rate they would have cost without the cache. diff --git a/docs/dashboard.md b/docs/dashboard.md index e79002dc..9359474d 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -141,10 +141,20 @@ Limits: provider's lifetime is unknown (no explicit breakpoints). The gross saving is right, and the net can be too high if a component broke the cache on that request (accountant design §6.4). The event carries an estimate flag, stored in `requests.price_estimated` and sent as `price_estimated` - in the request JSON. The request detail marks the net saving with "≈" and says why in its tooltip. - Overview counts them (`estimated_requests`, on the full and the lean path) and the Preset card says how many of - the window's requests they are. The totals and charts add such a figure like any other: they do not mark it. A row written before - the column existed reads as exact. + in the request JSON. The library prices each removal as a write the provider would have made. If the cache was + in fact already warm (a restart that lost the state file, or a session begun before the proxy), the figure is + about 2x too high, and the proxy cannot say by how much, because the size of the unseen warm prefix is what the + missing history held. So the proxy books no saving on such a request: `baseline_cost_usd` equals `cost_usd` and + every component `saved_usd` is 0. Net saved, total saved, the Components table and the keep-alive tab therefore + agree. The figure that was not booked is stored in `requests.estimated_saved_usd` and sent as `estimated_saved_usd` + (request JSON) and as `estimated_saved_usd` (Overview, summed over the window). The request detail shows it on its + own line, marked "≈". The Preset card says how many requests are not counted and shows the estimated figure beside + them (`estimated_requests`, on the full and the lean path). A row written before the column existed reads as exact and + keeps the figure it was booked with. The keep-alive rescue and the split saving do not depend on a break and stay booked. +- A request the provider rejected (any status that is not 2xx) books no saved tokens: `tokens_after` equals + `tokens_before`, `saved_unique` and `attempted_tokens` are 0, and each component row has `saved_gross` 0, + `saved_unique` 0 and `acted` false. The provider billed none of it, so tokens and dollars agree. A status of 0 + (the request never reached a provider, such as `/compact`) and any 2xx are not changed. - The library keeps each thread's state in memory. The proxy saves it to `.accountant-state.json` every minute and at a clean shutdown, and restores it at start, so a restart does not drop the active changes of a live session. The file holds hashes, sizes and ids, no message text, and is mode 0600. diff --git a/proxy/accountant_inject_test.go b/proxy/accountant_inject_test.go index 1cb8be8d..6d2900d6 100644 --- a/proxy/accountant_inject_test.go +++ b/proxy/accountant_inject_test.go @@ -15,6 +15,7 @@ import ( "github.com/rossoctl/context-guru/dash" "github.com/rossoctl/context-guru/expand" "github.com/rossoctl/context-guru/internal/adjudicate" + "github.com/rossoctl/context-guru/internal/modelinfo" ) // M4: the savings record ends with the body that goes upstream, AFTER expand.Inject and @@ -90,9 +91,19 @@ func TestSavingsRecordEndsWithTheInjectedForwardedBody(t *testing.T) { row = &e.Components[i] } } - if row == nil || row.Kind != dash.KindEnvelope || row.SavedUSD >= 0 { - t.Fatalf("%s: row %+v, want an envelope row with a cost", name, row) + // This is the first request of its thread, so the library priced it as an estimate and the + // books hold it out (issue #915): the row exists and books 0. The library's own figure for + // the row is the cost. + if row == nil || row.Kind != dash.KindEnvelope || row.SavedUSD != 0 { + t.Fatalf("%s: row %+v, want an envelope row that books nothing on an estimated request", name, row) } + price := modelinfo.Price{Input: 3e-6, CacheRead: 0.3e-6, CacheWrite: 3.75e-6, Output: 15e-6} + if fig := e.AccountantFigure(price); fig.ByComponent[name] >= 0 { + t.Fatalf("%s: the library's figure is %v, want a cost", name, fig.ByComponent[name]) + } + } + if !e.PriceEstimated || e.EstimatedSavedUSD == 0 { + t.Fatalf("the first request of a thread must be held as an estimate: estimated %v held %v", e.PriceEstimated, e.EstimatedSavedUSD) } var sum float64 for _, c := range e.Components { diff --git a/proxy/dash_test.go b/proxy/dash_test.go index 179b04c1..b256dfea 100644 --- a/proxy/dash_test.go +++ b/proxy/dash_test.go @@ -694,11 +694,34 @@ func TestDashboardBooksTheAccountantSaving(t *testing.T) { } resp.Body.Close() waitForRows(t, rec, 1) + // The first request of a thread has no history, so its saving is an estimate and is held out of + // the books (issue #915). It is booked from the second request on. + first, err := rec.DB().Requests(dash.Filter{}, 0, 10) + if err != nil { + t.Fatal(err) + } + if f := first.Requests[0]; !f.PriceEstimated || f.BaselineCostUSD != f.CostUSD || f.EstimatedSavedUSD <= 0 { + t.Fatalf("first request: estimated %v baseline %v cost %v held %v, want an estimate held out of the books", + f.PriceEstimated, f.BaselineCostUSD, f.CostUSD, f.EstimatedSavedUSD) + } + req2, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", + strings.NewReader(string(anthropicRequest("aws/claude-sonnet-5")))) + req2.Header.Set("Content-Type", "application/json") + req2.Header.Set("x-context-guru-session", "sess-acc") + resp2, err := srv.Client().Do(req2) + if err != nil { + t.Fatal(err) + } + resp2.Body.Close() + waitForRows(t, rec, 2) page, err := rec.DB().Requests(dash.Filter{}, 0, 10) if err != nil { t.Fatal(err) } - e := page.Requests[0] + e := page.Requests[0] // newest first + if e.PriceEstimated { + t.Fatalf("the second request of the thread is still an estimate") + } if e.TokensBefore <= e.TokensAfter { t.Fatalf("the pipeline did not compact (%d -> %d)", e.TokensBefore, e.TokensAfter) } From 2da3cb0400ad14007137c0d10a4207f8ffad71f6 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:44:47 +0200 Subject: [PATCH 35/58] fix(dash): Keep since and until exact in the stats cache key (#916) The key rounded since and until down to 30 s, so a narrower window got the wider window's cached totals. The key now holds the exact values. The UI rounds its own now to 30 s so relative ranges still share a key. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/api.go | 18 ++++-------------- dash/componentscache_test.go | 16 ++++++++++++++++ dash/ui/app.js | 4 +++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/dash/api.go b/dash/api.go index 3ff691f8..8a21b369 100644 --- a/dash/api.go +++ b/dash/api.go @@ -531,21 +531,11 @@ func (a *API) refreshJSON(c *jsonCache, key string, compute func(db *DB) ([]byte // cacheKey scopes a cache entry to the actual caller, not just the URL: a manager and a // tenant hitting the same query string must never share a cached body. // -// since/until are rounded to the cache TTL bucket rather than compared verbatim. The UI's -// relative ranges ("last 24h") re-stamp since/until with Date.now() on every single call — -// so the raw query string is different every time and this cache would otherwise never hit -// for the single most common view on the page. +// since and until stay exact in the key. An earlier version rounded them down to the TTL bucket, +// so a narrower window got the wider window's cached totals (issue #916). The UI keeps its relative +// ranges cheap by rounding `now` itself (resolveTime in ui/app.js), so one repaint burst shares a key. func cacheKey(p Principal, r *http.Request) string { - q := r.URL.Query() - bucket := dashCacheTTL.Milliseconds() - for _, param := range []string{"since", "until"} { - if v := q.Get(param); v != "" { - if ms, err := strconv.ParseInt(v, 10, 64); err == nil { - q.Set(param, strconv.FormatInt(ms/bucket*bucket, 10)) - } - } - } - return p.TenantID + "\x00" + strconv.FormatBool(p.Manager) + "\x00" + q.Encode() + return p.TenantID + "\x00" + strconv.FormatBool(p.Manager) + "\x00" + r.URL.Query().Encode() } // NewAPI builds the HTTP surface for a recorder. Malformed CIDRs are skipped with diff --git a/dash/componentscache_test.go b/dash/componentscache_test.go index a40a6be0..de31a17f 100644 --- a/dash/componentscache_test.go +++ b/dash/componentscache_test.go @@ -101,3 +101,19 @@ func TestComponentsCached(t *testing.T) { t.Errorf("past dashCacheStale: got %d component rows, want 3 (recomputed)", len(rows)) } } + +// A narrower window must not share a key with a wider one (issue #916). The key used to round +// since/until down to 30 s, so since=T+1s and since=T hit one cached body. +func TestCacheKeyKeepsSinceAndUntilExact(t *testing.T) { + p := Principal{Manager: true} + k := func(u string) string { return cacheKey(p, httptest.NewRequest(http.MethodGet, u, nil)) } + if k("/api/stats?since=1700000001000") == k("/api/stats?since=1700000000000") { + t.Error("since 1 s apart gives one key; a narrower window would get the wider window's totals") + } + if k("/api/stats?since=1&until=1700000001000") == k("/api/stats?since=1&until=1700000000000") { + t.Error("until 1 s apart gives one key") + } + if k("/api/stats?since=5") != k("/api/stats?since=5") { + t.Error("the same query must give the same key") + } +} diff --git a/dash/ui/app.js b/dash/ui/app.js index 3db6c42e..6c8366e2 100644 --- a/dash/ui/app.js +++ b/dash/ui/app.js @@ -274,7 +274,9 @@ function resolveTime(v, nowMs) { if (v === 'now') return nowMs; if (typeof v === 'string') { const m = /^now-(\d+)([smhdw])$/.exec(v); - if (m) return nowMs - Number(m[1]) * RANGE_UNIT_MS[m[2]]; + // Round `now` down to 30 s so a relative range gives the same since for 30 s and the server cache can hit. + // The server keeps since exact (issue #916), so this rounding is the UI's own choice. + if (m) return Math.floor(nowMs / 30000) * 30000 - Number(m[1]) * RANGE_UNIT_MS[m[2]]; const n = Number(v); return Number.isFinite(n) && n > 0 ? n : 0; } From 6138171b41b3be8a0cb45004bd12b3b1b7dfc8a3 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:46:37 +0200 Subject: [PATCH 36/58] fix(accountant): Version, sync and de-duplicate the state file; hash past nested cache marks (#917) Wrap the snapshot in a proxy version and refuse another one. Sync before the rename, skip an unchanged write and remove stale temp files at start. Strip cache_control at any depth before hashing a body element. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- docs/dashboard.md | 4 ++ docs/data-dependence.md | 2 +- proxy/accountant_savings.go | 2 + proxy/accountant_savings_body.go | 61 ++++++++++++++++----- proxy/accountant_savings_body_test.go | 19 +++++++ proxy/accountant_state.go | 70 ++++++++++++++++++++++-- proxy/accountant_state_test.go | 76 +++++++++++++++++++++++++++ 7 files changed, 217 insertions(+), 17 deletions(-) diff --git a/docs/dashboard.md b/docs/dashboard.md index 1b382c98..403de2b5 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -167,6 +167,10 @@ Limits: - The library keeps each thread's state in memory. The proxy saves it to `.accountant-state.json` every minute and at a clean shutdown, and restores it at start, so a restart does not drop the active changes of a live session. The file holds hashes, sizes and ids, no message text, and is mode 0600. + The file carries a proxy version (`cgStateVersion`) around the library snapshot. A file with another version is + ignored, because the hash of a body changed between versions. The save syncs the file before the rename, skips the + write when nothing changed, and the start removes temp files a crash left. The hash leaves out every `cache_control` + mark, nested ones too. A file older than one hour is ignored (no cache entry lives longer), and so is a missing or unreadable one: the proxy then starts empty, as before. A crash (kill -9) loses up to one minute of state. A request that follows a lost state is priced as a first request and carries the estimate flag. The diff --git a/docs/data-dependence.md b/docs/data-dependence.md index 9a0d4666..8188528c 100644 --- a/docs/data-dependence.md +++ b/docs/data-dependence.md @@ -226,7 +226,7 @@ One row for each registered component. The test `components/all/datadep_test.go` | Thread tracker | metadata | 🟡 Works, metadata | 🟡 Works, metadata | 🟡 Works, metadata | It tells the main agent from its subagents. It uses hashes to check if a new request continues an earlier one. | | Session tracker | metadata | 🟡 Works, metadata | 🟡 Works, metadata | 🟡 Works, metadata | It needs a stable label for each conversation. When the client sends none, it hashes the start of the conversation. | | Cache tracker | metadata | 🟡 Works, metadata | 🟡 Works, metadata | 🟡 Works, metadata | It must know how much of the conversation the provider already cached, so components never edit that part. After a restart it asks the dashboard (when on) for the time each thread's cached prefix was last refreshed — by its own last request or a keep-alive ping that read the cache, under either session id the conversation uses (the row stores the content-derived alias id, a hash like the session id itself); timestamps and ids, metadata only — and treats a thread refreshed inside the request's cache lifetime as cached, not as new, until that thread's first request. Without the dashboard — and for a tenant running observe mode, whose rows carry no session id — it uses the proxy's start time instead. | -| Accountant savings (per-component figures from the accountant library) | metadata | ❌ Off. It needs the dashboard, because the figure is attached to a dashboard event. | 🟡 Works, metadata | 🟡 Works, metadata | To price a saving it must know which part of the conversation a rewrite changed and whether the provider had that part cached. It reads each request once and keeps only a hash and a size for each part, plus the cache breakpoints. The text is dropped as soon as it is hashed, and the hashes are held in memory, one set per conversation thread, plus each thread's last set for the sibling check (SharedWarm). So a restart does not lose them, it also saves them to one file beside the dashboard database (`.accountant-state.json`, readable by its owner only), every minute and when it stops, and loads them again when it starts. The file holds hashes, sizes, positions, and the ids and names the state needs (session and thread ids, tool and component names, model names, times). It holds no message text. It is ignored when it is more than one hour old. It is not written when there is no dashboard. It reads the tools and system sections the same way, as names, hashes and sizes, to report the envelope changes. The result it writes to the dashboard is dollar amounts for each component, with their split by tier. No level changes what it does. | +| Accountant savings (per-component figures from the accountant library) | metadata | ❌ Off. It needs the dashboard, because the figure is attached to a dashboard event. | 🟡 Works, metadata | 🟡 Works, metadata | To price a saving it must know which part of the conversation a rewrite changed and whether the provider had that part cached. It reads each request once and keeps only a hash and a size for each part, plus the cache breakpoints. The text is dropped as soon as it is hashed, and the hashes are held in memory, one set per conversation thread, plus each thread's last set for the sibling check (SharedWarm). So a restart does not lose them, it also saves them to one file beside the dashboard database (`.accountant-state.json`, readable by its owner only), every minute and when it stops, and loads them again when it starts. The file holds hashes, sizes, positions, and the ids and names the state needs (session and thread ids, tool and component names, model names, times). It holds no message text. It also holds one proxy version number, and it is ignored when that number differs or when it is more than one hour old. It is not written when there is no dashboard. It reads the tools and system sections the same way, as names, hashes and sizes, to report the envelope changes. The result it writes to the dashboard is dollar amounts for each component, with their split by tier. No level changes what it does. | | Wait-cause and lifecycle signals | metadata | ❌ Off (it needs the dashboard) | 🟡 Works, metadata | 🟡 Works, metadata | To tell why the agent paused, it reads the request once. It keeps only counts and ids. Since #857 it also reads the **tool name** of the last tool call, to tell that an agent has finished (`done`). The keep-alive then sends no pings for that thread. A tool name is metadata, and the rule works at every storage level. Since #858 the keep-alive also uses the **wait cause** (from the stop reason, the tool names and the open-task counts), whether the thread id starts with `a:`, the token count of the prefix and the time of day, to pick the ping count, to ignore a strategy's time windows for a wait that is not for a person, and to skip the sweep question during a tool wait. It also keeps one flag per thread: "has a tool call been seen". All of it is metadata, and it works at every storage level. | | Costly settings check (`/context-guru:save-more settings`, `GET /api/adapt/settings`) | metadata | ❌ Off (the cost needs the dashboard). The plugin still finds the settings and says "cost not measured yet". | 🟡 Works, metadata | 🟡 Works, metadata | It counts and prices dashboard rows by request shape: model name, tool count, `max_tokens` and the stream flag. One check also uses the upstream status (400) of a Haiku row in the same session. It reads no request text and keeps nothing new. The plugin reads only the named keys of the user's own settings files (the betas setting, the recap setting, `ANTHROPIC_SMALL_FAST_MODEL` and `ANTHROPIC_DEFAULT_HAIKU_MODEL`) on the user's machine, and sends none of them to the proxy (#615). | | Beep (`beep.sh`, the `beep` plugin option) | none | 🟡 Works, no data | 🟡 Works, no data | 🟡 Works, no data | It plays a sound when Claude finishes or waits for you. It reads no stored data and never reads the hook payload. It sends nothing to the proxy or anywhere else. The level changes nothing. | diff --git a/proxy/accountant_savings.go b/proxy/accountant_savings.go index 4220307e..6786d53e 100644 --- a/proxy/accountant_savings.go +++ b/proxy/accountant_savings.go @@ -17,6 +17,8 @@ type savingsTracker struct { clients sync.Map // *accountant.Request -> clientHash // onEnd, when set, is called with each record after EndRequest. Tests only. onEnd func(*accountant.Request) + // written is the last state file's bytes, for save to skip an unchanged write. + written lastSaved } // clientHash is what begin built from the client body, kept until end so a request that was forwarded unchanged does diff --git a/proxy/accountant_savings_body.go b/proxy/accountant_savings_body.go index 00f86a56..dc87dd2b 100644 --- a/proxy/accountant_savings_body.go +++ b/proxy/accountant_savings_body.go @@ -50,32 +50,69 @@ func accBodies(raw []byte) accountant.Body { return accountant.Body{Start: built.Start, Pos: built.Pos} } -// withoutCacheMarks is one element's JSON without any cache_control: the element's own, and the ones on -// its content blocks. An element with none is returned as it is, byte for byte, so the common case costs -// no allocation. The result is only hashed. It is never sent anywhere. +// withoutCacheMarks is one element's JSON without any cache_control: the element's own, the ones on its content +// blocks, and the ones nested deeper, such as a mark inside a tool_result block's own content array (issue #917). +// An element with none is returned as it is, byte for byte, so the common case costs no allocation. The result is +// only hashed. It is never sent anywhere. func withoutCacheMarks(raw string) []byte { if !strings.Contains(raw, "cache_control") { return []byte(raw) } out := raw - if gjson.Get(out, "cache_control").Exists() { - if next, err := sjson.Delete(out, "cache_control"); err == nil { - out = next + // Each delete removes one mark and the walk starts again, so the paths stay right after the edit. A body has + // at most four marks on the wire, so the loop is short. + for n := 0; n < 64; n++ { + path := findCacheMark(gjson.Parse(out), "") + if path == "" { + break } + next, err := sjson.Delete(out, path) + if err != nil { + break + } + out = next + } + return []byte(out) +} + +// findCacheMark returns the gjson path of the first cache_control key under v, or "" when there is none. It looks +// at objects and arrays only, and does not look inside a string. +func findCacheMark(v gjson.Result, prefix string) string { + var found string + join := func(k string) string { + if prefix == "" { + return k + } + return prefix + "." + k } - if content := gjson.Get(out, "content"); content.IsArray() { + switch { + case v.IsObject(): + v.ForEach(func(k, c gjson.Result) bool { + if k.String() == "cache_control" { + found = join(strings.NewReplacer(".", "\\.", "*", "\\*", "?", "\\?").Replace(k.String())) + return false + } + if c.IsObject() || c.IsArray() { + ek := strings.NewReplacer(".", "\\.", "*", "\\*", "?", "\\?", "|", "\\|", "#", "\\#", "@", "\\@").Replace(k.String()) + if found = findCacheMark(c, join(ek)); found != "" { + return false + } + } + return true + }) + case v.IsArray(): i := 0 - content.ForEach(func(_, blk gjson.Result) bool { - if blk.Get("cache_control").Exists() { - if next, err := sjson.Delete(out, "content."+strconv.Itoa(i)+".cache_control"); err == nil { - out = next + v.ForEach(func(_, c gjson.Result) bool { + if c.IsObject() || c.IsArray() { + if found = findCacheMark(c, join(strconv.Itoa(i))); found != "" { + return false } } i++ return true }) } - return []byte(out) + return found } // accBreakpoints lists the cache_control marks, as positions in the same space as accBodies, one per diff --git a/proxy/accountant_savings_body_test.go b/proxy/accountant_savings_body_test.go index e90b4172..11924199 100644 --- a/proxy/accountant_savings_body_test.go +++ b/proxy/accountant_savings_body_test.go @@ -140,3 +140,22 @@ func TestAccBodies_ARealEditStillChangesTheHash(t *testing.T) { t.Fatal("an edited text kept the same hash") } } + +// A cache mark inside a tool_result block's own content array must not move the hash (issue #917). +func TestNestedCacheMarkDoesNotMoveTheHash(t *testing.T) { + with := `{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral"}}]}]}]}` + without := `{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":[{"type":"text","text":"a"}]}]}]}` + a, b := accBodies([]byte(with)), accBodies([]byte(without)) + if len(a.Pos) != 1 || len(b.Pos) != 1 { + t.Fatalf("positions %d and %d, want 1 and 1", len(a.Pos), len(b.Pos)) + } + if a.Pos[0].Hash != b.Pos[0].Hash { + t.Error("a nested cache_control moved the hash") + } + // A cache_control text inside a string is data, and still counts. + s1 := `{"messages":[{"role":"user","content":"say cache_control now"}]}` + s2 := `{"messages":[{"role":"user","content":"say now"}]}` + if accBodies([]byte(s1)).Pos[0].Hash == accBodies([]byte(s2)).Pos[0].Hash { + t.Error("text mentioning cache_control was stripped") + } +} diff --git a/proxy/accountant_state.go b/proxy/accountant_state.go index 0e48952c..f0d4048d 100644 --- a/proxy/accountant_state.go +++ b/proxy/accountant_state.go @@ -1,10 +1,14 @@ package proxy import ( + "bytes" + "encoding/json" "errors" "log/slog" "os" "path/filepath" + "strconv" + "sync" "time" ) @@ -17,21 +21,52 @@ const ( accStateMaxAge = time.Hour // accStateEvery is how often the state is saved while running, so a kill -9 or a crash loses minutes, not all. accStateEvery = time.Minute + // accStateVersion is the version of the proxy's own part of the state: how accBodies hashes a body. Change it + // whenever that hash changes. The library snapshot has its own version inside. A file with another version is + // ignored (issue #917). Version 1 was the unwrapped library snapshot, so a file from before the wrap fails the + // check too. Version 2 is the wrap with the hash that leaves out cache marks, nested ones included. + accStateVersion = 2 ) +// accStateFile is the file on disk: the proxy version around the library snapshot. +type accStateFile struct { + Version int `json:"cgStateVersion"` + Snapshot json.RawMessage `json:"snapshot"` +} + +// lastSaved holds the bytes of the last file written, so a save with no change skips the write. +type lastSaved struct { + mu sync.Mutex + data []byte +} + // save writes the snapshot to path through a temp file and a rename, so a crash mid-write leaves the old file. +// It syncs the temp file before the rename, so a power loss cannot leave an empty file under the final name. +// It writes nothing when the bytes equal the last file written. func (s *savingsTracker) save(path string) error { - data, err := s.trackers.Snapshot() + snap, err := s.trackers.Snapshot() + if err != nil { + return err + } + data, err := json.Marshal(accStateFile{Version: accStateVersion, Snapshot: snap}) if err != nil { return err } + s.written.mu.Lock() + defer s.written.mu.Unlock() + if s.written.data != nil && bytes.Equal(s.written.data, data) { + if _, err := os.Stat(path); err == nil { + return nil + } + } tmp, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".tmp*") if err != nil { return err } _, werr := tmp.Write(data) + serr := tmp.Sync() cerr := tmp.Close() - if err := errors.Join(werr, cerr); err != nil { + if err := errors.Join(werr, serr, cerr); err != nil { os.Remove(tmp.Name()) return err } @@ -39,12 +74,31 @@ func (s *savingsTracker) save(path string) error { os.Remove(tmp.Name()) return err } - return os.Rename(tmp.Name(), path) + if err := os.Rename(tmp.Name(), path); err != nil { + os.Remove(tmp.Name()) + return err + } + s.written.data = data + return nil +} + +// removeStaleTemps deletes the temp files a crash between CreateTemp and Rename left beside the state file. +func removeStaleTemps(path string) { + old, err := filepath.Glob(path + ".tmp*") + if err != nil { + return + } + for _, f := range old { + if err := os.Remove(f); err == nil { + slog.Info("accountant: removed a stale temp file", "path", f) + } + } } // restore loads a snapshot saved by save. It fails open: a missing, stale or unreadable file is logged // and the proxy starts with empty state, as before. It reports whether it restored anything. func (s *savingsTracker) restore(path string, now time.Time) bool { + removeStaleTemps(path) fi, err := os.Stat(path) if err != nil { if !errors.Is(err, os.ErrNotExist) { @@ -58,7 +112,15 @@ func (s *savingsTracker) restore(path string, now time.Time) bool { } data, err := os.ReadFile(path) if err == nil { - err = s.trackers.Restore(data) + var f accStateFile + switch { + case json.Unmarshal(data, &f) != nil: + err = errors.New("not a state file") + case f.Version != accStateVersion: + err = errors.New("state file has proxy version " + strconv.Itoa(f.Version) + ", this proxy uses " + strconv.Itoa(accStateVersion)) + default: + err = s.trackers.Restore(f.Snapshot) + } } if err != nil { slog.Warn("accountant: state not restored; starting empty", "path", path, "err", err) diff --git a/proxy/accountant_state_test.go b/proxy/accountant_state_test.go index 4e192fdd..68c396fd 100644 --- a/proxy/accountant_state_test.go +++ b/proxy/accountant_state_test.go @@ -1,6 +1,7 @@ package proxy import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -159,3 +160,78 @@ func TestAccDropForgetsTheClientHash(t *testing.T) { } (*capture)(nil).accDrop() // nil-safe, as before } + +// A file from another proxy version is ignored (fail open). A file without the wrap is the old format (issue #917). +func TestSavingsStateRefusesAnotherProxyVersion(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + twoTurns(t, path) + if !newSavingsTracker().restore(path, time.Now()) { + t.Fatal("a file of the current version was not restored") + } + raw, _ := os.ReadFile(path) + old := strings.Replace(string(raw), fmt.Sprintf(`"cgStateVersion":%d`, accStateVersion), `"cgStateVersion":1`, 1) + if old == string(raw) { + t.Fatal("the version field is not in the file") + } + os.WriteFile(path, []byte(old), 0o600) + if newSavingsTracker().restore(path, time.Now()) { + t.Error("a file with an old proxy version was restored") + } + // The bare library snapshot, as the first release wrote it. + var f accStateFile + json.Unmarshal(raw, &f) + os.WriteFile(path, f.Snapshot, 0o600) + if newSavingsTracker().restore(path, time.Now()) { + t.Error("an unwrapped snapshot was restored") + } +} + +// restore removes the temp files a crash left, and only those. +func TestSavingsStateRemovesStaleTemps(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + stale := filepath.Join(dir, "state.json.tmp123") + other := filepath.Join(dir, "other.tmp1") + os.WriteFile(stale, []byte("x"), 0o600) + os.WriteFile(other, []byte("x"), 0o600) + newSavingsTracker().restore(path, time.Now()) + if _, err := os.Stat(stale); err == nil { + t.Error("the stale temp file stayed") + } + if _, err := os.Stat(other); err != nil { + t.Error("restore removed a file that is not ours") + } +} + +// save writes nothing when the state did not change, and writes again when it did or when the file is gone. +func TestSavingsStateSkipsAnUnchangedWrite(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + c, _, _ := twoTurns(t, path) // saved once through a different tracker + _ = c + s := newSavingsTracker() + if err := s.save(path); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-time.Hour) + os.Chtimes(path, old, old) + if err := s.save(path); err != nil { + t.Fatal(err) + } + fi, _ := os.Stat(path) + if fi.ModTime().After(old.Add(time.Minute)) { + t.Error("an unchanged state was written again") + } + os.Remove(path) + if err := s.save(path); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); err != nil { + t.Error("a removed file was not written again") + } + entries, _ := os.ReadDir(filepath.Dir(path)) + for _, e := range entries { + if strings.Contains(e.Name(), ".tmp") { + t.Errorf("save left %s", e.Name()) + } + } +} From a1b89cf1c41859a726b32ef24c8e564b56a138bd Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:49:11 +0200 Subject: [PATCH 37/58] fix(accountant): Recover panics in ping pricing and in accBegin (#918) The keeper's ping row and PingSavings now recover and log, so a library panic cannot end the process. accBegin recovers and drops its record, so a panic on a hostile body no longer reaches the pre-forward recover. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- .../dashemitter_waitcause_test.go | 18 +++++++++ cmd/context-guru-proxy/main.go | 7 ++++ proxy/accountant_ping.go | 22 ++++++++-- proxy/accountant_ping_test.go | 23 +++++++++++ proxy/accountant_wire.go | 10 +++++ proxy/accountant_wire_test.go | 40 +++++++++++++++++++ 6 files changed, 117 insertions(+), 3 deletions(-) diff --git a/cmd/context-guru-proxy/dashemitter_waitcause_test.go b/cmd/context-guru-proxy/dashemitter_waitcause_test.go index 33a341ec..65b9178f 100644 --- a/cmd/context-guru-proxy/dashemitter_waitcause_test.go +++ b/cmd/context-guru-proxy/dashemitter_waitcause_test.go @@ -1,7 +1,9 @@ package main import ( + "context" "fmt" + "github.com/rossoctl/context-guru/internal/modelinfo" "path/filepath" "testing" "time" @@ -117,3 +119,19 @@ func waitForRow(t *testing.T, rec *dash.Recorder) (*dash.Page, error) { time.Sleep(5 * time.Millisecond) } } + +// A panic while a ping row is built must not end the process: the keeper calls KeepAlivePing from its own goroutine +// (issue #918). A nil Pricer inside the interface value panics in Price. +type panicPricer struct{ modelinfo.Pricer } + +func (panicPricer) Price(context.Context, string) (modelinfo.Price, bool) { panic("price lookup") } + +func TestDashEmitterKeepAlivePingSurvivesAPanic(t *testing.T) { + rec, err := dash.NewRecorder(dash.Options{DBPath: filepath.Join(t.TempDir(), "dash.db"), BatchSize: 1, FlushInterval: time.Millisecond}) + if err != nil { + t.Fatal(err) + } + defer rec.Close() + em := &dashEmitter{rec: rec, prices: panicPricer{}} + em.KeepAlivePing(components.KeepAliveReport{Tenant: "t1", Session: "s1", Model: "m", Status: 200, CacheRead: 10, TS: time.Now().UnixMilli()}) +} diff --git a/cmd/context-guru-proxy/main.go b/cmd/context-guru-proxy/main.go index 343d8750..cba38513 100644 --- a/cmd/context-guru-proxy/main.go +++ b/cmd/context-guru-proxy/main.go @@ -973,6 +973,13 @@ func (d *dashEmitter) KeepAlivePing(r components.KeepAliveReport) { if d.rec == nil { return } + // The keeper calls this from its own goroutine. A panic here (a price lookup, the row build) would end the + // whole process, so log it and carry on without this row (issue #918). + defer func() { + if p := recover(); p != nil { + slog.Error("context-guru: keep-alive ping row panicked; the row is dropped", "panic", p) + } + }() var price modelinfo.Price priced := false if d.prices != nil && r.Model != "" { diff --git a/proxy/accountant_ping.go b/proxy/accountant_ping.go index 04d19ef4..f708e0ca 100644 --- a/proxy/accountant_ping.go +++ b/proxy/accountant_ping.go @@ -1,6 +1,7 @@ package proxy import ( + "log/slog" "sync" "time" @@ -101,15 +102,30 @@ func pingStart(r components.KeepAliveReport) time.Time { // is already booked on its own row as ManualSavedUSD, from the user's estimate. Storing the facts too // would let the next real request credit the same write again as a library rescue, and the total would // count it twice (context-guru#768 B3: $0.0200 booked, about $0.0096 true). -func PingSavings(r components.KeepAliveReport, st *PingStore) (accountant.Result, bool) { +// +// A panic in the library is recovered and logged, and the ping is reported as not converted. The keeper +// runs this in its own goroutine, so an unrecovered panic would end the process (issue #918). +func PingSavings(r components.KeepAliveReport, st *PingStore) (res accountant.Result, ok bool) { + defer func() { + if p := recover(); p != nil { + slog.Error("accountant: ping pricing panicked; the ping is recorded with no saving", "panic", p) + res, ok = accountant.Result{}, false + } + }() if r.Window.IsZero() || r.Status < 200 || r.Status > 299 || r.FreshInput+r.CacheRead+r.CacheWrite+r.CacheWrite1h+r.Output == 0 { return accountant.Result{}, false } - ops, facts := accountant.PingOps(pingFromReport(r), accountant.Billing{Mode: accountant.BillingAPI}) - res := accountant.ComputeStandalone(ops) + ops, facts := pingOps(pingFromReport(r), accountant.Billing{Mode: accountant.BillingAPI}) + res = accountantStandalone(ops) if !r.Manual { st.put(accPingKey(r.Session, accThreadID(r.Thread)), facts) } return res, true } + +// pingOps and accountantStandalone are the two library calls PingSavings makes. Tests replace them to make them panic. +var ( + pingOps = accountant.PingOps + accountantStandalone = accountant.ComputeStandalone +) diff --git a/proxy/accountant_ping_test.go b/proxy/accountant_ping_test.go index cdd44b97..9bdec4b0 100644 --- a/proxy/accountant_ping_test.go +++ b/proxy/accountant_ping_test.go @@ -370,3 +370,26 @@ func TestTheEntryTTLFollowsTheBilledTier(t *testing.T) { t.Errorf("wrote nothing, gap 20m: phase = %q, want hot (no tier billed: the asked 1h stands)", got) } } + +// A panic in the library while a ping is priced is recovered: the keeper's goroutine must live (issue #918). +func TestPingSavingsRecoversFromALibraryPanic(t *testing.T) { + origOps, origStd := pingOps, accountantStandalone + defer func() { pingOps, accountantStandalone = origOps, origStd }() + pings := NewPingStore() + + pingOps = func(accountant.Ping, accountant.Billing) (accountant.RequestOps, accountant.PingFacts) { + panic("boom in PingOps") + } + if _, ok := PingSavings(okReport(), pings); ok { + t.Error("a panicking PingOps reported a figure") + } + pingOps = origOps + accountantStandalone = func(accountant.RequestOps) accountant.Result { panic("boom in ComputeStandalone") } + if _, ok := PingSavings(okReport(), pings); ok { + t.Error("a panicking ComputeStandalone reported a figure") + } + accountantStandalone = origStd + if _, ok := PingSavings(okReport(), pings); !ok { + t.Error("after the panics a normal ping was not priced") + } +} diff --git a/proxy/accountant_wire.go b/proxy/accountant_wire.go index f62ae177..772d1c7e 100644 --- a/proxy/accountant_wire.go +++ b/proxy/accountant_wire.go @@ -1,6 +1,7 @@ package proxy import ( + "log/slog" "time" bschemas "github.com/maximhq/bifrost/core/schemas" @@ -23,6 +24,15 @@ func (c *capture) accBegin(meta accountant.RequestMeta, client []byte) { if c == nil || c.savings == nil { return } + // The savings record is a side figure. A panic while hashing a hostile body must cost that figure only, and + // must not reach the pre-forward recover, which would throw away the pipeline's rewrite of the request + // (issue #918). The record is dropped, and the event books no saving. + defer func() { + if p := recover(); p != nil { + slog.Error("accountant: BeginRequest panicked; the request is recorded with no saving", "panic", p) + c.accDrop() + } + }() c.accReq = c.savings.begin(meta, client) c.accClient = client } diff --git a/proxy/accountant_wire_test.go b/proxy/accountant_wire_test.go index 9cf1b96a..db76a1bf 100644 --- a/proxy/accountant_wire_test.go +++ b/proxy/accountant_wire_test.go @@ -3,6 +3,7 @@ package proxy import ( "context" "fmt" + "github.com/rossoctl/accountant" "io" "math" "net/http" @@ -293,3 +294,42 @@ func TestResponsesWireBooksNoSavingAndSaysWhy(t *testing.T) { } } } + +// A panic in the savings begin (a hostile body) is recovered: the record is dropped, the figure is lost, and the +// caller goes on (issue #918). +func TestAccBeginRecoversAndDrops(t *testing.T) { + s := newSavingsTracker() + s.trackers = nil // BeginRequest on a nil Trackers panics + c := &capture{savings: s} + c.accBegin(accountant.RequestMeta{RequestID: "r1", Session: "s"}, benchBody()) + if c.accReq != nil { + t.Error("a panicked begin left a record") + } +} + +// The panic must not lose the request: the handler forwards it, and the event books no saving. +func TestAHostileBodyPanicInAccBeginStillForwardsTheRequest(t *testing.T) { + var got int + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got++ + io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"id":"m","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],"usage":{"input_tokens":1,"output_tokens":1}}`)) + })) + defer up.Close() + h, _, _ := accHandler(t, up.URL) + h.savings.trackers = nil + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", strings.NewReader(string(anthropicRequest("aws/claude-sonnet-5")))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-context-guru-session", "sess-panic") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != 200 || got != 1 { + t.Errorf("status %d, upstream calls %d, want 200 and 1", resp.StatusCode, got) + } +} From ab085e8e4833f461084adf1506c9c13bff11e553 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:52:25 +0200 Subject: [PATCH 38/58] test(dash): Pin first-request pricing; the $0 row is an unloaded price map (#920) A realistic first turn books its cost when the price is known. The reported rows had no price: the price map loads in the background and the first requests arrived before it finished. No code change. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/firstrequest_test.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 dash/firstrequest_test.go diff --git a/dash/firstrequest_test.go b/dash/firstrequest_test.go new file mode 100644 index 00000000..fd2aceae --- /dev/null +++ b/dash/firstrequest_test.go @@ -0,0 +1,35 @@ +package dash + +import ( + "testing" + + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +// A first request of a session with a realistic cold write books its cost when the price is known +// (issue #920). The reported $0 row had no price: the proxy's price map had not loaded yet, so Price +// was called with accountingComplete true and p.Zero() true. tokens_before=49 plays no part. +func TestFirstRequestBooksCostWhenThePriceIsKnown(t *testing.T) { + p := modelinfo.Price{Input: 3e-06, Output: 1.5e-05, CacheRead: 3e-07, CacheWrite: 3.75e-06} + for _, tokensBefore := range []int{49, 29650} { + e := &Event{Model: "claude-sonnet-4-5", Status: 200, TokensBefore: tokensBefore, CacheWrite: 29650, OutputTokens: 50} + e.PriceMode(p, true) + if e.TokenAccounting != AccountingComplete { + t.Errorf("tokens_before=%d: token_accounting = %q, want complete", tokensBefore, e.TokenAccounting) + } + want := 29650*3.75e-06 + 50*1.5e-05 + if e.CostUSD < want*0.999 || e.CostUSD > want*1.001 { + t.Errorf("tokens_before=%d: cost = %v, want %v", tokensBefore, e.CostUSD, want) + } + } +} + +// Without a price the row is partial at $0. That is the honest answer for an unknown price. The fix for +// #920 is to make the price known before the first request, not to guess one here. +func TestFirstRequestWithoutAPriceIsPartial(t *testing.T) { + e := &Event{Model: "claude-sonnet-4-5", Status: 200, TokensBefore: 49, CacheWrite: 29650} + e.PriceMode(modelinfo.Price{}, true) + if e.TokenAccounting != AccountingPartial || e.CostUSD != 0 { + t.Errorf("got %q cost %v, want partial and 0", e.TokenAccounting, e.CostUSD) + } +} From dcd1b31fad33458759b191b5fce06e27ef923331 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 08:07:30 +0200 Subject: [PATCH 39/58] fix(proxy): Book no dollars on a rejected request that reported usage A 4xx or 5xx body can carry a usage object. The accountant then priced a saving beside the zeroed tokens. Skip the compute for a status that is not 0 and not 2xx, as PingSavings does. The row falls back with no_result. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/dashcapture.go | 20 ++++++++++-- proxy/rejected_savings_test.go | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/proxy/dashcapture.go b/proxy/dashcapture.go index 0cf9083f..a7df3dfa 100644 --- a/proxy/dashcapture.go +++ b/proxy/dashcapture.go @@ -732,7 +732,15 @@ func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, content defer c.savings.drop(c.accReq) if c.accReq != nil && usageOK && !c.accDone { c.accDone = true - if c.rounds <= 1 { + // A rejected request (a status that is not 0 and not 2xx) books no saving, in tokens or in + // dollars. The response body of a 4xx or 5xx can still carry a usage object, but the provider + // billed none of the request, so the library must not price a saving on it. DropUnbilledSavings + // zeroed the tokens above; skipping the compute leaves the event with no Result, so PriceMode + // books no dollars either (acc_fallback no_result, reason rejected_status). PingSavings refuses + // a non-2xx ping the same way. A status of 0 (the request never reached a provider) is not a rejection. + if c.rejected() { + c.accNoResult = "rejected_status" + } else if c.rounds <= 1 { c.accBilledTTL(usage) if res, ok := c.savings.computeSafe(c.accReq, usage); ok { e.SetAccountant(res) @@ -867,16 +875,22 @@ func (c *capture) accDrop() { // noResultWhy names the branch that left an event with no accountant Result, for the acc_fallback line. The reason // no_result covers four unrelated things, and the log line used to say only that. +// rejected is true when the provider answered with a status that is not 2xx. Status 0 means no upstream +// answered (such as /compact), and is not a rejection. +func (c *capture) rejected() bool { + return c.status != 0 && (c.status < 200 || c.status > 299) +} + func (c *capture) noResultWhy(usageOK bool) string { switch { case c.accReq == nil: return "no_record" // no session, or the pipeline did not run: accBegin never opened a record case !usageOK: return "no_usage" - case c.rounds > 1: - return "upstream_rounds=" + strconv.Itoa(c.rounds) case c.accNoResult != "": return c.accNoResult + case c.rounds > 1: + return "upstream_rounds=" + strconv.Itoa(c.rounds) } return "" } diff --git a/proxy/rejected_savings_test.go b/proxy/rejected_savings_test.go index 0f00059e..84fd288e 100644 --- a/proxy/rejected_savings_test.go +++ b/proxy/rejected_savings_test.go @@ -1,7 +1,13 @@ package proxy import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" "path/filepath" + "strings" "testing" "time" @@ -78,3 +84,56 @@ func TestFinishBooksNoSavedTokensOnARejectedRequest(t *testing.T) { }) } } + +// Review 927 round 1, finding 3. A rejected request whose error body still carries a usage object +// must book no dollars beside its zero tokens. The 200 control shows the same two requests do book a +// saving, so the zero on the 429 and the 400 comes from the status and not from the fixture. +func TestFinishBooksNoDollarsOnARejectedRequestThatReportedUsage(t *testing.T) { + decl := func(name string, n int) map[string]any { + return map[string]any{"name": name, "description": strings.Repeat("Does a thing. ", n), + "input_schema": map[string]any{"type": "object", "properties": map[string]any{"q": map[string]any{"type": "string"}}}} + } + b, _ := json.Marshal(map[string]any{"model": "aws/claude-sonnet-5", "max_tokens": 64, + "tools": []any{decl("bash", 3), decl("big_one", 400), decl("big_two", 400)}, + "messages": []any{map[string]any{"role": "user", "content": "hi"}}}) + run := func(status int) dash.Event { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + n := len(body) / 4 + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + fmt.Fprintf(w, `{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],`+ + `"stop_reason":"end_turn","usage":{"input_tokens":12,"output_tokens":34,`+ + `"cache_read_input_tokens":%d,"cache_creation_input_tokens":%d}}`, n*8/10, n-n*8/10-12) + })) + defer up.Close() + h, rec, tap := accHandlerYAML(t, up.URL, + "pipeline: [toolfilter]\ncomponents:\n toolfilter:\n remove: [big_one, big_two]\n") + accPostBody(t, h, "sess-rej", b) + accPostBody(t, h, "sess-rej", b) + waitForRows(t, rec, 2) + return tap.all()[1] + } + ctl := run(200) + if ctl.BaselineCostUSD <= ctl.CostUSD { + t.Fatalf("control (200) booked no saving: baseline %v cost %v", ctl.BaselineCostUSD, ctl.CostUSD) + } + for _, status := range []int{429, 400} { + e := run(status) + if e.Status != status { + t.Fatalf("status = %d, want %d", e.Status, status) + } + if e.Saved() != 0 || e.SavedUnique != 0 || e.BaselineCostUSD != e.CostUSD { + t.Errorf("status %d: saved tokens %d, unique %d, baseline-cost delta %v, want all 0", + status, e.Saved(), e.SavedUnique, e.BaselineCostUSD-e.CostUSD) + } + for _, c := range e.Components { + if c.SavedUSD != 0 || c.SavedGross != 0 { + t.Errorf("status %d: component %s saved %d tokens / $%v, want 0", status, c.Component, c.SavedGross, c.SavedUSD) + } + } + if e.AccountantFallback() != "no_result" { + t.Errorf("status %d: fallback %q, want no_result", status, e.AccountantFallback()) + } + } +} From 359a770d2432b6acc9b3bc2a4b23968f5532dfeb Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 08:07:31 +0200 Subject: [PATCH 40/58] docs(proxy): Correct the messageOps comment on equal-length changes Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/accountant_envelope.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxy/accountant_envelope.go b/proxy/accountant_envelope.go index 04f4c9c1..d5d596c0 100644 --- a/proxy/accountant_envelope.go +++ b/proxy/accountant_envelope.go @@ -286,7 +286,8 @@ func accHex(v uint64) string { return strconv.FormatUint(v, 16) } // no component reported: tool deferral puts its names list at messages[1], either appended to a system // message already there or as a new one (tooldefer placeList). before is the body the edit read and // after the one it produced. A new message is a splice that inserts one; an appended list is a rewrite -// in place of that message. Any other change returns nil, and the sum check says so. The operator is +// in place of that message. When the two arrays are the same length, it returns one rewrite op per changed +// message. Any other change (a different length by more than one, or a non-insertion) returns nil, and the sum check says so. The operator is // the edit's own, so its row and its ChangeID do not collide with a pipeline component's. func messageOps(operator string, before, after []byte) []accountant.Operation { b, a := gjson.GetBytes(before, "messages").Array(), gjson.GetBytes(after, "messages").Array() From 0a397f9dd0117fa1577bbec399785ca45ae99a10 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 08:07:32 +0200 Subject: [PATCH 41/58] fix(dash): Give a held estimated request one owner and stop counting old rows as held - The measured filter credit owns the filter dollars of a request whose figure was held out (declFilterInBaseline is false when estimate_held). - New column requests.estimate_held marks the hold-out. Rows from before keep their legacy meaning and are not counted as not counted. - Count a baseline-priced filtered row as priced in adapttools. - Price resets EstimatedSavedUSD and EstimateHeld. - Name what the hold-out covers (ttl legs, negative figure) in code, UI, docs. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accountant_price.go | 15 ++++++-- dash/accountant_switch_test.go | 7 ++-- dash/accounting_test.go | 9 +++-- dash/adapttools.go | 2 ++ dash/declcredit_test.go | 65 ++++++++++++++++++++++++++++++++++ dash/event.go | 12 +++++-- dash/overview.go | 7 ++-- dash/overview_lean.go | 2 +- dash/query.go | 7 ++-- dash/schema.go | 5 +++ dash/store.go | 6 ++-- dash/toolsuggest.go | 9 +++-- dash/ui/app.js | 10 +++--- docs/dashboard.md | 21 ++++++++--- docs/data-dependence.md | 2 +- 15 files changed, 148 insertions(+), 31 deletions(-) diff --git a/dash/accountant_price.go b/dash/accountant_price.go index 4d6220fe..c17bd093 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -215,7 +215,7 @@ func (e *Event) AccountantFallback() string { return e.accFallback } // library figure. Otherwise the event books no saving (Price set every saving to 0) and // AccountantFallback says why: a figure nothing measured is not shown. func (e *Event) PriceMode(p modelinfo.Price, accountingComplete bool) { - e.accFallback, e.PriceEstimated, e.AccountantPriced, e.EstimatedSavedUSD = "", false, false, 0 + e.accFallback, e.PriceEstimated, e.AccountantPriced, e.EstimatedSavedUSD, e.EstimateHeld = "", false, false, 0, false if !e.priceCost(p, accountingComplete) { e.accFallback = FallbackUnpriced return @@ -340,9 +340,18 @@ func (e *Event) accOverride(p modelinfo.Price) string { // prefix is the thing the missing history held. A figure nothing measured is not booked // (issue #915). The request books no break-affected saving, and the figure is kept in // EstimatedSavedUSD, outside every total, so the Overview, the components table, the - // keep-alive tab and the API still agree. The keep-alive rescue and the split saving do not - // depend on a break, so they stay booked below. + // keep-alive tab and the API still agree. + // + // What is held out: fig.DeltaUSD, which is every component's figure, the head-TTL premium and + // rescue legs (the headttl row), the ended-change break, and the filter's part. The figure can + // be negative when the request's net effect is a cost (a tooldefer names list, an injected + // expand or adjudicate declaration): the cost is held out with the rest, and the readers render + // a negative figure as a cost. What stays booked, each in its own column outside the baseline: + // the keep-alive rescue (keepalive_saved_usd) and the split saving (cachesplit_saved_usd), which + // do not depend on a break. The filter's dollars are owned by the measured credit + // (declFilterInBaseline is false for a held row), not by this figure. e.EstimatedSavedUSD = fig.DeltaUSD + e.EstimateHeld = true e.BaselineCostUSD = e.CostUSD for i := range e.Components { e.Components[i].SavedUSD, e.Components[i].SavedFirstUSD, e.Components[i].SavedReplayUSD = 0, 0, 0 diff --git a/dash/accountant_switch_test.go b/dash/accountant_switch_test.go index 00f23f6e..c7e08e7b 100644 --- a/dash/accountant_switch_test.go +++ b/dash/accountant_switch_test.go @@ -248,9 +248,12 @@ func TestPriceMode_PenaltyUnknownIsAnEstimate(t *testing.T) { if want := 1000 * 0.3e-6; !accNear(e.EstimatedSavedUSD, want) { t.Errorf("EstimatedSavedUSD = %v, want the held figure %v", e.EstimatedSavedUSD, want) } + if !e.EstimateHeld { + t.Error("EstimateHeld = false on a held request") + } e.Price(accTestPrice, true) - if e.AccountantEstimated() { - t.Fatal("Price left the estimate flag set") + if e.AccountantEstimated() || e.EstimatedSavedUSD != 0 || e.EstimateHeld { + t.Fatal("Price left the estimate state set") } } diff --git a/dash/accounting_test.go b/dash/accounting_test.go index 0449dad2..95971e3e 100644 --- a/dash/accounting_test.go +++ b/dash/accounting_test.go @@ -1162,9 +1162,14 @@ func TestOverviewCountsEstimatedRequests(t *testing.T) { for i, est := range []bool{true, false, true, false, false} { e := &Event{TS: now - int64(i)*1000, SessionID: "s1", Model: "m1", TenantID: "t1", TokensBefore: 100, TokensAfter: 90, FreshInput: 5, CacheRead: 40, OutputTokens: 5, - TokenAccounting: AccountingComplete, PriceEstimated: est} + TokenAccounting: AccountingComplete, PriceEstimated: est, EstimateHeld: est} insertReq(t, db, e) } + // A row written before #915: estimated, and it booked its figure. It is not "not counted". + old := &Event{TS: now - 9000, SessionID: "s1", Model: "m1", TenantID: "t1", + TokensBefore: 100, TokensAfter: 90, FreshInput: 5, CacheRead: 40, OutputTokens: 5, + TokenAccounting: AccountingComplete, PriceEstimated: true} + insertReq(t, db, old) f := Filter{TenantAll: true, Since: now - 60_000} full, err := db.Overview(f) if err != nil { @@ -1175,7 +1180,7 @@ func TestOverviewCountsEstimatedRequests(t *testing.T) { t.Fatal(err) } if full.EstimatedRequests != 2 || lean.EstimatedRequests != 2 { - t.Fatalf("estimated requests: full %d, lean %d, want 2 and 2 (of %d)", full.EstimatedRequests, lean.EstimatedRequests, full.Requests) + t.Fatalf("estimated requests: full %d, lean %d, want 2 and 2 (of %d); the old-meaning row must not count", full.EstimatedRequests, lean.EstimatedRequests, full.Requests) } } diff --git a/dash/adapttools.go b/dash/adapttools.go index 91d5f9f9..f0d2f56c 100644 --- a/dash/adapttools.go +++ b/dash/adapttools.go @@ -476,6 +476,8 @@ func priceCarriedSession(out *carriedScan, byID []*carried, keys []string, list byID[key].Requests++ byID[key].Reads += int64(v[0]) } + // A deliberate $0, not a missing measurement: the request is priced. + out.pricedRequests++ continue } p, ok := modelinfo.Price{}, false diff --git a/dash/declcredit_test.go b/dash/declcredit_test.go index 730f3bd9..c4145f19 100644 --- a/dash/declcredit_test.go +++ b/dash/declcredit_test.go @@ -539,3 +539,68 @@ func TestFilterCreditKeepsFactsAndSkipsDollarsForAccountantPricedRows(t *testing t.Errorf("items sum to $%g, more than the total $%g: %+v", sum, saved.Filter.USD, saved.Filter.Items) } } + +// Review 927 round 1, finding 1. A request whose figure was held out (estimate_held = 1, #915) books +// nothing in the baseline, filter part included. The measured credit owns that request's filter dollars: +// it adds them, and the totals do not count the request. A request the accountant booked still adds none. +func TestFilterCreditKeepsDollarsForAHeldEstimatedRequest(t *testing.T) { + db := openTestDB(t) + var evs []*Event + for i := 0; i < 10; i++ { + e := mkEvent(day(0)+int64(i), "s1", "claude", 100, 90) + e.TenantID, e.Tools, e.FilteredDeclTokens, e.DeclDigest = "t1", 2, 5000, "d1" + e.Components = append(e.Components, CompRow{Component: "toolfilter", Kind: KindEnvelope, Acted: true, + Mutated: true, PricedBy: PricedByAccountant}) + e.BaselineCostUSD = e.CostUSD + if i < 4 { // the first four are estimated requests whose figure was held out + e.PriceEstimated, e.EstimateHeld, e.EstimatedSavedUSD = true, true, 0.05 + e.BaselineCostUSD = e.CostUSD + } + evs = append(evs, e) + } + if err := db.insertBatch(evs); err != nil { + t.Fatal(err) + } + w := &invWriter{db: db, seen: map[string]*invSession{}} + if err := w.write([]invMsg{{tenant: "t1", session: "s1", ts: day(0), inv: &Inventory{Digest: "d1", + UseFingerprint: 1, Decls: []Decl{{Kind: KindMCPTool, Name: "mcp__srv__q", Server: "srv", Tokens: 5000}}}}}); err != nil { + t.Fatal(err) + } + one, ok := flatPrice("claude") + if !ok { + t.Fatal("flatPrice has no price") + } + want := 4 * 5000 * one.CacheRead // mkEvent rows read the cache + got, err := db.DeclFilterSavings(Filter{Tenant: "t1"}, flatPrice) + if err != nil { + t.Fatal(err) + } + if got.Requests != 10 || got.Reads != 50000 { + t.Errorf("facts = %d requests / %d reads, want 10 / 50000", got.Requests, got.Reads) + } + if !accNear(got.USD, want) || want == 0 { + t.Errorf("USD = %g, want %g: only the 4 held requests add dollars", got.USD, want) + } + doc, err := db.ToolFilterDocFor(Filter{Tenant: "t1"}, flatPrice, ToolFilterState{}) + if err != nil { + t.Fatal(err) + } + if doc.Realized == nil || doc.Realized.Requests != 10 || !accNear(doc.Realized.USD, want) { + t.Errorf("Realized = %+v, want 10 requests and $%g", doc.Realized, want) + } + saved, err := db.adaptToolsSaved(Filter{Tenant: "t1"}, flatPrice, []string{"mcp__srv"}) + if err != nil { + t.Fatal(err) + } + if saved.Filter.Requests != 10 || !accNear(saved.Filter.USD, want) { + t.Errorf("saved.filter = %d requests / $%g, want 10 / $%g", saved.Filter.Requests, saved.Filter.USD, want) + } + // The totals hold the request out: no baseline delta on any of these rows. + o, err := db.Overview(Filter{TenantAll: true}) + if err != nil { + t.Fatal(err) + } + if d := o.BaselineCostUSD - o.CostUSD; !accNear(d, 0) || o.EstimatedRequests != 4 { + t.Errorf("baseline delta %g, held requests %d, want 0 and 4", d, o.EstimatedRequests) + } +} diff --git a/dash/event.go b/dash/event.go index ccacf13b..0aa7de59 100644 --- a/dash/event.go +++ b/dash/event.go @@ -458,9 +458,14 @@ type Event struct { // BaselineCostUSD == CostUSD and every component SavedUSD 0, so every total, the components table // and the keep-alive tab agree without a filter. This field keeps the figure that was not booked. // 0 on an exact request and on a row from before the column existed. - EstimatedSavedUSD float64 `json:"estimated_saved_usd,omitempty"` - CacheMissReason string `json:"cache_miss_reason"` - UncompressedReason string `json:"uncompressed_reason"` + EstimatedSavedUSD float64 `json:"estimated_saved_usd,omitempty"` + // EstimateHeld is true when the hold-out above applied to this row: the request is estimated AND its + // removal figure was kept out of every total. It is the marker readers use for "not counted". It is + // not the same as PriceEstimated: a row written before #915 has price_estimated = 1 and DID book its + // figure, and it reads here as false (the column defaults to 0, as acc_priced does). Stored. + EstimateHeld bool `json:"estimate_held,omitempty"` + CacheMissReason string `json:"cache_miss_reason"` + UncompressedReason string `json:"uncompressed_reason"` // CacheMissCause says what changed on a miss that is not hit, cold_start or ttl_expiry. // Set only from a measured comparison (Cause* constants). "" when nothing was measured. CacheMissCause string `json:"cache_miss_cause,omitempty"` @@ -865,6 +870,7 @@ func uncompressedReason(e *Event, tr apply.Trace) string { // cannot compute must read as unknown, not as free. func (e *Event) Price(p modelinfo.Price, accountingComplete bool) { e.accFallback, e.PriceEstimated, e.AccountantPriced = "", false, false + e.EstimatedSavedUSD, e.EstimateHeld = 0, false e.priceCost(p, accountingComplete) } diff --git a/dash/overview.go b/dash/overview.go index 55c77cfa..dffa6f7a 100644 --- a/dash/overview.go +++ b/dash/overview.go @@ -418,8 +418,9 @@ type Overview struct { PreCorrectionSavedUSD float64 `json:"pre_correction_saved_usd"` CorrectedSavingRows int64 `json:"corrected_saving_rows"` // EstimatedRequests counts the requests whose saving the accountant library priced without knowing whether - // a component broke the cache (requests.price_estimated): the first request of a thread, and the first one - // after a restart that lost the state. Their net can be too high. Overview.Requests is the denominator. + // a component broke the cache AND that the proxy held out of the books (requests.estimate_held): the first + // request of a thread, and the first one after a restart that lost the state. It does not count a row + // written before #915, which has price_estimated = 1 but booked its figure. Overview.Requests is the denominator. EstimatedRequests int64 `json:"estimated_requests"` // EstimatedSavedUSD is what those requests would have saved, held OUT of every total (#915): // NetSavedUSD, TotalSavedUSD, the components and the keep-alive tab never include it. It is @@ -678,7 +679,7 @@ func (d *DB) Overview(f Filter) (*Overview, error) { COALESCE(SUM(CASE WHEN r.billed_token_factor > 0 AND r.baseline_cost_usd > r.cost_usd THEN 1 ELSE 0 END),0), `+deferNetSum+`, - COALESCE(SUM(r.price_estimated),0), COALESCE(SUM(r.estimated_saved_usd),0) + COALESCE(SUM(r.estimate_held),0), COALESCE(SUM(r.estimated_saved_usd),0) FROM requests r WHERE `+cond, args...).Scan( &o.Requests, &o.Sessions, &o.TokensBefore, &o.TokensAfter, &o.SavedUnique, &o.AttemptedTokens, &o.FrozenTokens, &o.SavedGrossAttempted, &o.AttemptedRequests, diff --git a/dash/overview_lean.go b/dash/overview_lean.go index b4544cde..2592e743 100644 --- a/dash/overview_lean.go +++ b/dash/overview_lean.go @@ -116,7 +116,7 @@ func leanWindowMainQuery(f Filter) (string, []any) { AVG(CASE WHEN ` + agent + ` THEN r.cg_latency_ms END), COALESCE(SUM(CASE WHEN ` + autoPing + ` THEN r.cost_usd ELSE 0 END),0), COALESCE(SUM(CASE WHEN ` + autoPing + ` THEN 1 ELSE 0 END),0), - COALESCE(SUM(CASE WHEN ` + agent + ` THEN r.price_estimated ELSE 0 END),0), + COALESCE(SUM(CASE WHEN ` + agent + ` THEN r.estimate_held ELSE 0 END),0), COALESCE(SUM(CASE WHEN ` + agent + ` THEN r.estimated_saved_usd ELSE 0 END),0), ` + manualSelect + `, COALESCE(SUM(CASE WHEN ` + agent + ` THEN r.defer_net_usd ELSE 0 END),0) diff --git a/dash/query.go b/dash/query.go index 973d5784..8ff3cda3 100644 --- a/dash/query.go +++ b/dash/query.go @@ -193,13 +193,13 @@ var requestCols = `r.id, r.ts, r.tenant_id, r.session_id, r.model, r.provider, r r.agent_compaction, r.cache_miss_cause, r.manual_ping, r.manual_saved_usd, r.project, r.source, r.sweep_expands, r.sweep_expand_tokens, r.sweep_false_expands, r.sweep_answer_usd, - r.session_alias, r.price_estimated, r.acc_priced, r.estimated_saved_usd, + r.session_alias, r.price_estimated, r.acc_priced, r.estimated_saved_usd, r.estimate_held, r.defer_tokens, r.defer_list_tokens, r.defer_searches, r.defer_loaded, r.defer_wrong_guesses, r.defer_gross_usd, r.defer_net_usd, r.defer_dropped_searches` func scanRequest(rows interface{ Scan(...any) error }) (*Event, error) { var e Event - var byp, ca, stream, keepAlive, agentComp, manual, estimated, accPriced int + var byp, ca, stream, keepAlive, agentComp, manual, estimated, accPriced, held int var temp, topP sql.NullFloat64 var waitingFor, waitTool, wakeCause sql.NullString var openBackground, openTeammate, monitorExpiryMs sql.NullInt64 @@ -219,10 +219,11 @@ func scanRequest(rows interface{ Scan(...any) error }) (*Event, error) { &agentComp, &e.CacheMissCause, &manual, &e.ManualSavedUSD, &e.Project, &e.Source, &e.SweepExpands, &e.SweepExpandTokens, &e.SweepFalseExpands, &e.SweepAnswerUSD, - &e.SessionAlias, &estimated, &accPriced, &e.EstimatedSavedUSD, + &e.SessionAlias, &estimated, &accPriced, &e.EstimatedSavedUSD, &held, &e.DeferTokens, &e.DeferListTokens, &e.DeferSearches, &e.DeferLoaded, &e.DeferWrongGuesses, &e.DeferGrossUSD, &e.DeferNetUSD, &e.DeferDroppedSearches) e.PriceEstimated, e.AccountantPriced = estimated != 0, accPriced != 0 + e.EstimateHeld = held != 0 e.WaitingFor, e.WaitTool, e.WakeCause = waitingFor.String, waitTool.String, wakeCause.String e.OpenBackground, e.OpenTeammate = int(openBackground.Int64), int(openTeammate.Int64) e.MonitorExpiryMs = monitorExpiryMs.Int64 diff --git a/dash/schema.go b/dash/schema.go index 426a37b5..4a7d1af2 100644 --- a/dash/schema.go +++ b/dash/schema.go @@ -112,6 +112,10 @@ CREATE TABLE IF NOT EXISTS requests ( -- the saving the accountant library priced on an estimated request, kept OUT of baseline_cost_usd -- and every total (#915). 0 on an exact request and on every row written before this column. estimated_saved_usd REAL NOT NULL DEFAULT 0, + -- 1 when the hold-out of #915 applied: the request is estimated and its figure was kept out of the books. + -- 0 on every row written before this column existed, including an old estimated row that DID book its + -- figure (price_estimated alone does not say which meaning a row has). + estimate_held INTEGER NOT NULL DEFAULT 0, preset TEXT NOT NULL DEFAULT '', mode TEXT NOT NULL DEFAULT '', -- operating mode (active|bypass|observe) route TEXT NOT NULL DEFAULT '', @@ -774,6 +778,7 @@ var additiveColumns = []struct{ table, column, ddl string }{ {"requests", "price_estimated", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "acc_priced", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "estimated_saved_usd", "REAL NOT NULL DEFAULT 0"}, + {"requests", "estimate_held", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "cache_ttl", "TEXT NOT NULL DEFAULT ''"}, {"requests", "sse_buffered", "INTEGER NOT NULL DEFAULT 0"}, {"requests", "ttfb_ms", "REAL NOT NULL DEFAULT 0"}, diff --git a/dash/store.go b/dash/store.go index 6b048cb7..c1c263a3 100644 --- a/dash/store.go +++ b/dash/store.go @@ -593,12 +593,12 @@ func (d *DB) insertBatch(evs []*Event) error { waiting_for, wait_tool, wake_cause, open_background, open_teammate, monitor_expiry_ms, agent_compaction, cache_miss_cause, manual_ping, manual_saved_usd, project, source, sweep_expands, sweep_expand_tokens, sweep_false_expands, sweep_answer_usd, session_alias, - price_estimated, acc_priced, estimated_saved_usd, + price_estimated, acc_priced, estimated_saved_usd, estimate_held, defer_tokens, defer_list_tokens, defer_searches, defer_loaded, defer_wrong_guesses, defer_gross_usd, defer_net_usd, defer_dropped_searches ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, - ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) + ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { return err } @@ -673,7 +673,7 @@ func (d *DB) insertBatch(evs []*Event) error { * safe direction — it claims LESS about the row. */ sourceOf(e.Source), e.SweepExpands, e.SweepExpandTokens, e.SweepFalseExpands, e.SweepAnswerUSD, - e.SessionAlias, boolInt(e.PriceEstimated), boolInt(e.AccountantPriced), e.EstimatedSavedUSD, + e.SessionAlias, boolInt(e.PriceEstimated), boolInt(e.AccountantPriced), e.EstimatedSavedUSD, boolInt(e.EstimateHeld), e.DeferTokens, e.DeferListTokens, e.DeferSearches, e.DeferLoaded, e.DeferWrongGuesses, e.DeferGrossUSD, e.DeferNetUSD, e.DeferDroppedSearches, ) diff --git a/dash/toolsuggest.go b/dash/toolsuggest.go index fbfa025f..5958c919 100644 --- a/dash/toolsuggest.go +++ b/dash/toolsuggest.go @@ -196,8 +196,13 @@ func (d *DB) DeclFilterSavings(f Filter, price func(string) (modelinfo.Price, bo // those are facts about what the filter did, not dollars (context-guru#911). A request the library could // not price (priced_by='none') or one written before the switch has nothing in the baseline, so its // measured credit stays the only place its filter saving appears. -const declFilterInBaseline = `EXISTS (SELECT 1 FROM request_components c - WHERE c.request_id = r.id AND c.component = 'toolfilter' AND c.priced_by = '` + PricedByAccountant + `')` +// +// A request whose figure was held out (r.estimate_held = 1, #915) is NOT in the baseline: the books hold the +// whole estimated figure out, the filter part included, so the measured credit is the one place that +// request's filter saving appears, in dollars. The measured credit owns it (it is measured from the rate +// card and the tier, and it does not depend on the unknown cache state), and the totals never count it. +const declFilterInBaseline = `(r.estimate_held = 0 AND EXISTS (SELECT 1 FROM request_components c + WHERE c.request_id = r.id AND c.component = 'toolfilter' AND c.priced_by = '` + PricedByAccountant + `'))` // declFilterSavingsQuery builds the SQL and args DeclFilterSavings executes — pulled out for // #502's EXPLAIN-plan test, same reason as cachesplitHistoricalQuery (dash/cachehistory.go). diff --git a/dash/ui/app.js b/dash/ui/app.js index 3db6c42e..0fda533c 100644 --- a/dash/ui/app.js +++ b/dash/ui/app.js @@ -102,6 +102,8 @@ function usd(v) { if (a < 1000) return (v < 0 ? '-' : '') + '$' + a.toFixed(2); return (v < 0 ? '-' : '') + '$' + nf.format(Math.round(a)); } +// estUsd renders a held-out estimate: a negative figure is a cost, so it reads as one. +function estUsd(v) { return v < 0 ? 'a cost of ' + usd(-v) : usd(v); } function pct(v, digits = 1) { return v === null || v === undefined ? '—' : v.toFixed(digits) + '%'; } /** * usdOrNA renders a dollar figure, or `n/a` WITH ITS REASON when the figure is not a @@ -1827,7 +1829,7 @@ function renderSavedLedger(o, ka, partial) { o.estimated_requests > 0 ? el('p', { class: 'ov-none ov-small', 'data-testid': 'estimated-requests', text: num(o.estimated_requests) + ' of ' + num(o.requests) + ' requests are not counted above: the first ' + 'request of a thread, or the first after a restart that lost its state. The proxy cannot tell if the cache was already warm. ' - + 'Estimated saving, not in any total: ' + usd(o.estimated_saved_usd || 0) + '.' }) : null, + + 'Estimated saving, not in any total: ' + estUsd(o.estimated_saved_usd || 0) + '.' }) : null, ])); // Tools and skills filter. @@ -3708,10 +3710,10 @@ async function openRequest(id, fromURL) { // thread yet), so the gross saving is right and the net may be too high. Marked "≈", the // same way this page marks every other estimate. kv(term('Net saving', 'Baseline cost minus actual cost minus context-guru model cost.' - + (e.price_estimated ? ' Not counted: there was no history to tell whether the cache was already warm, so no saving is booked. The estimate is shown on the next line.' : '')), + + (e.estimate_held ? ' Not counted: there was no history to tell whether the cache was already warm, so no saving is booked. The estimate is shown on the next line.' : '')), priced ? usd(e.baseline_cost_usd - e.cost_usd - e.cg_llm_cost_usd) : '—'), - e.price_estimated ? kv(term('Estimated saving (not counted)', 'What the accountant would have booked with no history. It can be about 2x too high when the cache was already warm, so it is in no total.'), - priced ? '≈ ' + usd(e.estimated_saved_usd || 0) : '—') : null, + e.estimate_held ? kv(term('Estimated saving (not counted)', 'What the accountant would have booked with no history. It can be about 2x too high when the cache was already warm, so it is in no total. A negative figure is a cost.'), + priced ? (e.estimated_saved_usd < 0 ? '' : '≈ ') + estUsd(e.estimated_saved_usd || 0) : '—') : null, // On a turn whose cache HIT this is usually the largest money figure on the row, // and it was not reported anywhere: the cache reads this request was billed for, // against the fresh rate they would have cost without the cache. diff --git a/docs/dashboard.md b/docs/dashboard.md index 1b382c98..8f2ffb3d 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -122,7 +122,8 @@ What the library prices: read-side reachability test. The entry TTL given to the library follows the billed tier: if the request asked for 1h and the response billed 5m, the library judges the next gap against 5 minutes. - **Tool-filter credit.** A request whose `toolfilter` row is `priced_by = accountant` adds no dollars - to the measured filter credit (the net saving has them). It still counts in the request count, the + to the measured filter credit (the net saving has them). The exception is a request whose figure was held out + (`estimate_held = 1`): the net saving has nothing for it, so the measured credit keeps its dollars. It still counts in the request count, the tokens not sent and the first date, so the Realized block and the item list stay. - **Cachesplit.** `cachesplit_saved_usd` is the library's split effect. - **Head TTL.** The 1h premium and the rescue it earns are on the `headttl` row. @@ -158,11 +159,23 @@ Limits: agree. The figure that was not booked is stored in `requests.estimated_saved_usd` and sent as `estimated_saved_usd` (request JSON) and as `estimated_saved_usd` (Overview, summed over the window). The request detail shows it on its own line, marked "≈". The Preset card says how many requests are not counted and shows the estimated figure beside - them (`estimated_requests`, on the full and the lean path). A row written before the column existed reads as exact and - keeps the figure it was booked with. The keep-alive rescue and the split saving do not depend on a break and stay booked. + them (`estimated_requests`, on the full and the lean path). The count uses a separate marker, + `requests.estimate_held` (sent as `estimate_held`), set only when the hold-out applied. `price_estimated` alone is not + enough: it existed before the hold-out, and a row written then has `price_estimated = 1` and DID book its figure. + Such a row has `estimate_held = 0` (the column defaults to 0), keeps the figure it was booked with, and is not in + `estimated_requests`. The hold-out covers the whole figure: every component, the head-TTL premium and rescue legs + (so the 1h premium of a thread's first request shows as $0 there), the ended-change break and the filter's part. + The figure can be negative when the request's net effect is a cost (a tool-names list, an injected expand or + adjudicate declaration). The UI then shows "a cost of $x". The keep-alive rescue (`keepalive_saved_usd`) and the + split saving do not depend on a break and stay booked, each in its own column. + The filter's dollars on a held request are owned by the measured tool-filter credit, not by the held figure: + the credit keeps its dollars for such a request, so no total counts it twice and none leaves it out. - A request the provider rejected (any status that is not 2xx) books no saved tokens: `tokens_after` equals `tokens_before`, `saved_unique` and `attempted_tokens` are 0, and each component row has `saved_gross` 0, - `saved_unique` 0 and `acted` false. The provider billed none of it, so tokens and dollars agree. A status of 0 + `saved_unique` 0 and `acted` false. The provider billed none of it, so tokens and dollars agree. This holds even when the + error body carries a `usage` object: the proxy then skips the accountant for that request (`acc_fallback` + reason `rejected_status`), so the row books 0 dollars too: `baseline_cost_usd` equals `cost_usd` and every + component `saved_usd` is 0. A status of 0 (the request never reached a provider, such as `/compact`) and any 2xx are not changed. - The library keeps each thread's state in memory. The proxy saves it to `.accountant-state.json` every minute and at a clean shutdown, and restores it at start, so a restart does not drop the active diff --git a/docs/data-dependence.md b/docs/data-dependence.md index 9a0d4666..a1766777 100644 --- a/docs/data-dependence.md +++ b/docs/data-dependence.md @@ -219,7 +219,7 @@ One row for each registered component. The test `components/all/datadep_test.go` | **Keep-alive pings** | content. The host keeps the credential. | ❌ **Should be off. Today it is still on**, and keeps the last body. The proxy binary also keeps the credential. | ❌ Off | ✅ Works | To keep the prompt cache warm, the proxy sends the exact same prompt again with the user's credential. Without the text, it cannot ping. The Ping button (`POST /api/me/cache/ping`) uses the same body and key. With keep-alive off, a request that carries the header `x-context-guru-ping: 1` is still held, for the button only: no ping is sent for it, and the body and key are released 300 seconds after the last request on a 5-minute cache, or 59 minutes after it on a 1-hour cache (a press does not extend it), and one account holds at most 16 such threads. Without the header nothing is held. The install adds the header and tells the user. An older install (from before `--no-ping-header`) gets it at its next session start, with the same notice. Any other install without the header is never changed. It gets a one-time note instead. The cache-state answer also carries the time of the next planned ping (a number, metadata), so the mod can hide the button while a ping fires before the cache expires. | | **Body stash** | content | ❌ **Should be off. Today it is still on**, with 2 full bodies per thread. | ❌ Off. The sweep's question then cannot be asked, and `cache_aware_summarizer` stops. | ✅ Works | To ask the model a cheap question, Context Guru adds the question to the exact text it sent last turn. The provider already cached that text. The sweep keeps **2 more full copies of the conversation per thread**, outside the stash, for a thread the keep-alive can ping (see `extract_llm_sweep` above). | | Observe-mode store | content | ❌ **Should follow the level. Today it is still a real Store.** | It should follow the level | ✅ Works | Observe mode runs the pipeline later on a copy, to estimate savings. The copy should store only what the real run would store. | -| Dashboard database | content and metadata | ❌ Off | 🟡 Works, metadata: numbers, fixed codes, ids, and tool, MCP-server and skill names. **No** before/after text, summaries, declaration text, error text or rejection text. | ✅ Works. It keeps text only when content capture is on. | The dashboard shows what Context Guru removed and what your setup carries. Every number on it needs only counts and prices. Every request row comes from traffic the proxy saw itself. On a local install the dashboard also reads two things from your disk, and stores neither: the working folder recorded in Claude Code's session files, to group folders into projects for the project picker, and each installed skill's description from its `SKILL.md`. The keep-alive calculator and recommendation (#868) replay the per-wait ping plan from columns the proxy already writes for each request: stop reason, wait cause, thread id, cache tier and provider. They read no text. Tool deferral's net-saving panel (#869) adds eight count and price columns per request: tokens deferred, the names list's tokens, searches, dropped searches, tools loaded, wrong guesses, and the gross and net dollars. The tool inventory also records the built-in tools Claude Code defers (by name, with 0 tokens) and counts a tool loaded through search as a use. `save-more tools` reads the same inventory, the tool deferral history and the account's keep and always-defer lists. All of it is tool names, counts and prices, and works at the metadata level. Each request row also has one flag, `acc_priced` (context-guru#912), set when the accountant library priced the request. The keep-alive views read it to trust the library's stored rescue figure. It is a yes or no, with no text. An earlier history import copied whole request rows from those session files; it is removed, and its rows are deleted when the database opens. | +| Dashboard database | content and metadata | ❌ Off | 🟡 Works, metadata: numbers, fixed codes, ids, and tool, MCP-server and skill names. **No** before/after text, summaries, declaration text, error text or rejection text. | ✅ Works. It keeps text only when content capture is on. | The dashboard shows what Context Guru removed and what your setup carries. Every number on it needs only counts and prices. Every request row comes from traffic the proxy saw itself. On a local install the dashboard also reads two things from your disk, and stores neither: the working folder recorded in Claude Code's session files, to group folders into projects for the project picker, and each installed skill's description from its `SKILL.md`. The keep-alive calculator and recommendation (#868) replay the per-wait ping plan from columns the proxy already writes for each request: stop reason, wait cause, thread id, cache tier and provider. They read no text. Tool deferral's net-saving panel (#869) adds eight count and price columns per request: tokens deferred, the names list's tokens, searches, dropped searches, tools loaded, wrong guesses, and the gross and net dollars. The tool inventory also records the built-in tools Claude Code defers (by name, with 0 tokens) and counts a tool loaded through search as a use. `save-more tools` reads the same inventory, the tool deferral history and the account's keep and always-defer lists. All of it is tool names, counts and prices, and works at the metadata level. Each request row also has one flag, `acc_priced` (context-guru#912), set when the accountant library priced the request. The keep-alive views read it to trust the library's stored rescue figure. It is a yes or no, with no text. Two more flags and one number sit beside it: `estimate_held` (a yes or no: the request's estimated figure was kept out of the totals) and `estimated_saved_usd` (that figure, in dollars). They hold no text. An earlier history import copied whole request rows from those session files; it is removed, and its rows are deleted when the database opens. | | Dashboard memos | content | ❌ Off, with the dashboard | 🟡 Works, metadata: a hash and a token count. That is all they need. | ✅ Works. They keep declaration and system-prompt text. | They remember a tool list so they do not count its tokens again on every request. A hash and a count are enough. | | Archive | content and metadata | ❌ Off | 🟡 Works, metadata rows only | ✅ Works | It moves old conversations off the local disk. The database stays small, and you can still open old history. | | Insights | metadata | ❌ Off (it needs the dashboard) | 🟡 Works, metadata | 🟡 Works, metadata | A cost report needs only counts, tool names and prices, not the text. | From 578380b1d586e51e6634b284fcdd3000405727d0 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 07:46:06 +0200 Subject: [PATCH 42/58] fix(accountant): Bump accountant to main 5d9e4f7 The new library adds a per-breakpoint TTL phase, media pricing and validation bounds. The suite for dash, proxy, apply, keepalive and components passes unchanged, so no figure changes. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a734561a..b02848ae 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.26.4 require github.com/maximhq/bifrost/core v1.7.0 // accountant prices context operations. This module's tests import it and its shared test cases. -require github.com/rossoctl/accountant v0.1.1-0.20261104203351-5fa19864db4c +require github.com/rossoctl/accountant v0.1.1-0.20261105050047-5d9e4f715cce require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 diff --git a/go.sum b/go.sum index 7e368317..c1884d84 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rossoctl/accountant v0.1.1-0.20261104203351-5fa19864db4c h1:X71YQPS5juEzAUWTVEfuffU2sxW7avZIpyfTW/VkspQ= -github.com/rossoctl/accountant v0.1.1-0.20261104203351-5fa19864db4c/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261105050047-5d9e4f715cce h1:rLl5aMat+hzoCUZQ7qb8X6rcwsZ7j/C2naFUmn3ekWU= +github.com/rossoctl/accountant v0.1.1-0.20261105050047-5d9e4f715cce/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 79c676e8dff1f496e8e21abff43d083aa41f0804 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 08:01:45 +0200 Subject: [PATCH 43/58] fix(accountant): Credit the cachesplit saving on a short conversation (#909) The live path booked no split credit. The cause was in the library, not in the proxy path. On a short conversation the stable half the provider reads is within the library's read tolerance (128 tokens plus 2%) of the whole previous entry. The library took that read as proof that nothing broke, moved B* past the changed system block and dropped the split effect. The proxy sent the right facts and the right split operation. The fix is accountant fix/split-short-turn (23e2ed1). This commit pins that version and adds a proxy test with a short conversation, which fails on the old pin and passes on the new one. Before (fake upstream, 4 requests, tail moves each turn): cachesplit_saved_usd 0, split_credited 0. After: 0.04433, split_credited 3. Truth: 3 x 4325 x (3.75 - 0.30) / 1M = 0.04477. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- go.mod | 2 +- go.sum | 4 +-- proxy/accountant_savings_test.go | 50 ++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b02848ae..e8b4c23e 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.26.4 require github.com/maximhq/bifrost/core v1.7.0 // accountant prices context operations. This module's tests import it and its shared test cases. -require github.com/rossoctl/accountant v0.1.1-0.20261105050047-5d9e4f715cce +require github.com/rossoctl/accountant v0.1.1-0.20261105060029-23e2ed1c3cc1 require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 diff --git a/go.sum b/go.sum index c1884d84..3d7741d1 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rossoctl/accountant v0.1.1-0.20261105050047-5d9e4f715cce h1:rLl5aMat+hzoCUZQ7qb8X6rcwsZ7j/C2naFUmn3ekWU= -github.com/rossoctl/accountant v0.1.1-0.20261105050047-5d9e4f715cce/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261105060029-23e2ed1c3cc1 h1:BUnCM/hVdPgTHrNncJT31g1kFn3PTWYIBzdiIXmAzns= +github.com/rossoctl/accountant v0.1.1-0.20261105060029-23e2ed1c3cc1/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/proxy/accountant_savings_test.go b/proxy/accountant_savings_test.go index 667e5778..da978e1c 100644 --- a/proxy/accountant_savings_test.go +++ b/proxy/accountant_savings_test.go @@ -378,3 +378,53 @@ func TestSavings_ASpliceOverAMarkedMessageKeepsTheSumCheck(t *testing.T) { t.Errorf("check %s failed: %+v", ch.Check, ch) } } + +// context-guru#909: the live path booked no split credit. A short conversation (a few hundred tokens of messages +// beside a 4,300-token system block) makes the stable half the provider reads fall within the library's read +// tolerance of the whole previous entry. The library then took the read as proof that nothing broke, put B* past the +// changed system block, and dropped the split effect. The earlier tests used long messages, so they missed it. +// The fix is in accountant (fix/split-short-turn); this test pins the host side: the envelope the proxy reports +// for a split, the facts it passes, and the effect the library books for a tail that moves every turn. +func TestSavings_ASplitBooksItsEffectOnAShortConversationWhenTheTailMoves(t *testing.T) { + c := accRun{newSavingsTracker(), accountant.Thread{ID: "main"}} + stable := strings.Repeat("Follow the rules carefully and be concise. ", 400) + turn := func(n int, sys string) []byte { + var msgs []string + for i := 0; i < n; i++ { + role := "user" + if i%2 == 1 { + role = "assistant" + } + mark := "" + if i == n-1 { + mark = `,"cache_control":{"type":"ephemeral"}` + } + msgs = append(msgs, fmt.Sprintf(`{"role":%q,"content":[{"type":"text","text":%q%s}]}`, role, fmt.Sprintf("message %d %s", i, strings.Repeat("w ", 20)), mark)) + } + return []byte(`{` + sys + `,"messages":[` + strings.Join(msgs, ",") + `]}`) + } + for n := 1; n <= 4; n++ { + vol := fmt.Sprintf("\nCurrent branch: main\nRecent commits:\n%08d fix thing\n", n) + client := turn(2*n-1, fmt.Sprintf(`"system":[{"type":"text","text":%q,"cache_control":{"type":"ephemeral"}}]`, stable+vol)) + fwd := turn(2*n-1, fmt.Sprintf(`"system":[{"type":"text","text":%q,"cache_control":{"type":"ephemeral"}},{"type":"text","text":%q}]`, stable, vol)) + total, half := (len(fwd)+3)/4, len(stable)/4 + u := Usage{CacheRead: int64(half), CacheWrite: int64(total - half)} + if n == 1 { + u = Usage{CacheWrite: int64(total)} + } + ops := envelopeOps(client, fwd) + res := c.do(n, client, fwd, u, ops...) + got := 0 + for _, e := range res.CacheEffects { + if e.Kind == accountant.EffectSplit { + got += e.Saved.CacheWrite5m + } + } + if n == 1 && got != 0 { + t.Fatalf("turn 1 has no previous request, yet a split effect of %d tokens", got) + } + if n > 1 && got != half { + t.Fatalf("turn %d: split effect %d tokens, want the stable half %d (checks %v)", n, got, half, res.Checks) + } + } +} From 7cbc5493d304ff032dbbd2899c16964a9bc7a5a1 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 08:07:34 +0200 Subject: [PATCH 44/58] fix(accountant): Report the summarize splice when sibling text repeats (#910) Summarize hit ops_dropped on most requests, so it booked its model cost and no saving. The splice itself was reported. The rebuild after a count change broke the sizes. It matched a kept message to the first unused slot with the same bytes. A user message that holds a tool_result and a text block (for example "continue") has the same bytes on every turn. A kept message took the slot of a removed turn, so the forwarded body held that turn's orphan tool_result and not the kept message. The byte sizes then failed wireSizes and the library dropped all operations. A byte match now prefers a slot of a wire message that a kept tool message already names by its tool_call_id. Before (fake upstream, 10 requests): net_saved_usd -0.00274, 6 requests ops_dropped. After: +0.00404, 0 fallbacks. Truth from the fake's own log (control 0.0721437 minus treatment 0.0681045): +0.00404. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- apply/apply.go | 26 ++++++++++++++++++-- apply/ops_splice_test.go | 53 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/apply/apply.go b/apply/apply.go index 4db6049c..9bb13a7a 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -1982,6 +1982,22 @@ func rebuildCountChanged(body []byte, orig []gjson.Result, slots []slot, out []b toolSlotByID[id] = k } } + // keptWire is the wire messages a kept tool message names, by its tool_call_id. A text block that sits + // beside a tool_result in one wire message has the same bytes on every turn ("continue"), so matching + // it by bytes alone takes the first turn's slot, which a splice may have removed. It would emit that + // turn's wire message, with its orphan tool_result, and leave the survivor's message out (#910). + // A byte match therefore prefers a slot of a wire message a kept tool message already names. + keptWire := map[int]bool{} + for i := range out { + if out[i].Role != bschemas.ChatMessageRoleTool || out[i].ChatToolMessage == nil || out[i].ChatToolMessage.ToolCallID == nil { + continue + } + if k, ok := toolSlotByID[*out[i].ChatToolMessage.ToolCallID]; ok { + if bi, _, ok := splitSlotPath(slots[k].path); ok { + keptWire[bi] = true + } + } + } var parts [][]byte // emitted guards against emitting one body message TWICE: several normalized // messages can share a body index (an Anthropic user message with several @@ -2005,10 +2021,16 @@ func rebuildCountChanged(body []byte, orig []gjson.Result, slots []slot, out []b } if matched < 0 { for k := range slots { - if !used[k] && bytes.Equal(mb, slots[k].pre) { - matched = k + if used[k] || !bytes.Equal(mb, slots[k].pre) { + continue + } + if bi, _, ok := splitSlotPath(slots[k].path); ok && keptWire[bi] { + matched = k // the survivor of a kept wire message break } + if matched < 0 { + matched = k + } } } if matched < 0 { diff --git a/apply/ops_splice_test.go b/apply/ops_splice_test.go index 4d0a01e1..00fad58b 100644 --- a/apply/ops_splice_test.go +++ b/apply/ops_splice_test.go @@ -57,3 +57,56 @@ func TestBodyOptsCountChangeEmitsSplice(t *testing.T) { t.Fatalf("splice = %+v op = %+v", s, res.Trace.Ops[0]) } } + +// dropFirstExchange replaces Input[1..3] with a summary: the first tool exchange of the transcript below. +type dropFirstExchange struct{} + +func (dropFirstExchange) Name() string { return "dropfirst" } +func (dropFirstExchange) Enabled(*components.Ctx) bool { return true } +func (dropFirstExchange) Reformat(req *bschemas.BifrostChatRequest, _ *components.Report, _ *components.Ctx) error { + if len(req.Input) < 6 { + return nil + } + sum := req.Input[1] + schema.SetMessageText(&sum, "s") + out := []bschemas.ChatMessage{req.Input[0], sum} + req.Input = append(out, req.Input[4:]...) + return nil +} + +func init() { + components.Register("dropfirst", func([]byte) (components.Component, error) { return dropFirstExchange{}, nil }) +} + +// context-guru#910: summarize hit ops_dropped on most requests. A user message that holds a tool_result and a +// text block ("CALL_TOOL again") has the same text on every turn. The rebuild matched survivors by bytes and took +// the FIRST unused slot, so the text of a survivor matched the text of a removed turn. It then emitted that +// turn's wire message, which holds an orphan tool_result, in the wrong place. The sizes did not add up, and +// every operation was dropped. A survivor must keep its own wire message, and the splice must be reported. +func TestSpliceKeepsSurvivorsWithIdenticalSiblingText(t *testing.T) { + tu := func(id string) string { + return `{"role":"assistant","content":[{"type":"tool_use","id":"` + id + `","name":"Bash","input":{"command":"` + id + `"}}]}` + } + tr := func(id, out string) string { + return `{"role":"user","content":[{"type":"tool_result","tool_use_id":"` + id + `","content":"` + out + `"},{"type":"text","text":"CALL_TOOL again"}]}` + } + body := `{"model":"claude-x","messages":[ + {"role":"user","content":"first"},` + tu("a") + `,` + tr("a", "out a") + `,` + tu("b") + `,` + tr("b", "out b") + `,` + tu("c") + `,` + tr("c", "out c") + `]}` + res := traceOf(t, "dropfirst", bschemas.Anthropic, body) + if !res.Changed { + t.Fatal("precondition: the count-changing rewrite must be written back") + } + out := gjson.GetBytes(res.Body, "messages").Array() + var ids []string + for _, m := range out { + ids = append(ids, gjson.Get(m.Raw, "content.0.id").String()+gjson.Get(m.Raw, "content.0.tool_use_id").String()) + } + // first, summary, then the exchanges b and c in order. Nothing of a. + want := []string{"", "", "b", "b", "c", "c"} + if strings.Join(ids, ",") != strings.Join(want, ",") { + t.Fatalf("forwarded messages carry ids %q, want %q", ids, want) + } + if res.Trace.OpsDropped != 0 || len(res.Trace.Ops) != 1 || res.Trace.Ops[0].MessageSplice == nil { + t.Fatalf("ops = %+v dropped = %d: the splice was not reported", res.Trace.Ops, res.Trace.OpsDropped) + } +} From f15a7b2edf0a5e70767ec1fbb35a7763db8759d4 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 08:09:35 +0200 Subject: [PATCH 45/58] fix(accountant): Pin the request after a manual ping as hot and mark the manual figure an estimate (#913) The issue feared that a manual ping, which stores no facts, would leave the next request cold. The scenario was run with the pinned library: request at t0 writes 100,000 tokens, manual ping at t0+4m reads them, request at t0+8m reads them again. The gap is 8 minutes, past the 5 minute TTL. The phase is hot, not cold. The library takes a billed read that covers the previous extent as proof that the entry was alive, so it needs no LastPing. The same request after an automatic ping is hot too and books a rescue. The manual run books none, so the write is not counted twice. No code change was needed for the phase. The test pins it. Storing the facts with the rescue disabled, as the issue suggested, would add state and a row flag for no change in any figure, so it is not done. ManualSavedUSD is the user's estimate of the avoided write. The Overview now labels the line 'estimate' and the Cache info text and docs/dashboard.md say it is not a measurement. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/ui/app.js | 4 +-- docs/dashboard.md | 2 +- proxy/accountant_ping_test.go | 63 +++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/dash/ui/app.js b/dash/ui/app.js index 0fda533c..e0801f34 100644 --- a/dash/ui/app.js +++ b/dash/ui/app.js @@ -1781,7 +1781,7 @@ function renderSavedLedger(o, ka, partial) { const cacheRan = o.keepalive_pings > 0 || !!o.keepalive_net_usd || parts.reorder !== 0 || (mp.count || 0) > 0; const brk = el('ul', { class: 'ov-split', 'data-testid': 'saved-cache-split' }, el('li', {}, el('span', { text: 'Automatic pings' }), el('strong', { text: usd(parts.keepalive) })), - el('li', { 'data-testid': 'saved-manual' }, el('span', { text: 'Manual pings' + (mp.count ? ' (' + num(mp.count) + ')' : '') }), + el('li', { 'data-testid': 'saved-manual' }, el('span', { text: 'Manual pings' + (mp.count ? ' (' + num(mp.count) + ')' : '') + ' · estimate' }), el('strong', { text: usd(parts.manual) })), parts.reorder !== 0 ? el('li', {}, el('span', { text: 'Cache reorder' }), el('strong', { text: usd(parts.reorder) })) : null); // The strategy's name, as the plugin calls it, beside "Cache" the way the preset's name sits beside @@ -1813,7 +1813,7 @@ function renderSavedLedger(o, ka, partial) { ], infoButton('cache', 'Cache', [ 'Automatic pings: context-guru sends a small request before the cache expires. Saved = the cache writes it prevented, less the cost of the pings.', - 'Manual pings: the same, when you press Ping.', + 'Manual pings: the same, when you press Ping. This figure is an estimate. It is the write the ping avoided, taken from the cache lifetime and the size of the prefix. No request read it back. The other figures in this total are measured.', 'This is the most it could have saved. Another session can keep the same cache warm. The Cache strategy tab says if it pays off.', ]))); diff --git a/docs/dashboard.md b/docs/dashboard.md index 8f2ffb3d..38f6c762 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -304,7 +304,7 @@ per line when there are several headers). The proxy never forwards it upstream. Its row is a keep-alive row with `manual_ping` set. `GET /api/stats` reports these rows as `manual_pings` (`count`, `ping_usd`, `saved_usd` net of the pings' cost, and `writes_avoided`: the pings that found the cache warm). They are a savings category of their own: they join `total_saved_usd`, and no cache strategy's ledger counts them. -The figure is an upper bound. A press needs the proxy to hold the thread for keep-alive, so with +The figure is an estimate, not a measurement: it is the write the ping avoided, taken from the cache lifetime and the prefix size, and no request read it back. The Overview marks it "estimate". The figure is an upper bound. A press needs the proxy to hold the thread for keep-alive, so with automatic pings on it can overlap the keep-alive credit for the same avoided write. `saved_usd` is `avoided - ping` summed over all presses, so a ping that missed shows as a loss; the per-press `saved_usd` in the reply is never below 0, so the two need not match. A press is refused with diff --git a/proxy/accountant_ping_test.go b/proxy/accountant_ping_test.go index cdd44b97..1723d7a1 100644 --- a/proxy/accountant_ping_test.go +++ b/proxy/accountant_ping_test.go @@ -370,3 +370,66 @@ func TestTheEntryTTLFollowsTheBilledTier(t *testing.T) { t.Errorf("wrote nothing, gap 20m: phase = %q, want hot (no tier billed: the asked 1h stands)", got) } } + +// context-guru#913: a manual ping stores no facts. The next real request then has no LastPing, and the gap +// from the request before the ping is longer than the TTL. The issue feared a cold phase, with the request priced +// at the write tier although the usage shows a cache read. The library takes a billed read as proof that the +// entry was alive (design §5.2, R45), so the request is hot. The arithmetic: request at t0 writes 100,000 +// tokens, a manual ping at t0+4m reads them, a request at t0+8m reads them again. The gap of 8 minutes is past +// the 5 minute TTL, and the read of 100,000 tokens covers the whole extent of the first request. +// +// The control is the same sequence with an automatic ping. It books a rescue; the manual one books none, since +// its avoided write is already on its own row as ManualSavedUSD (an estimate by the user's ask). The two phases +// must agree, and the manual run must carry no unpriced entry and no check failure. +func TestTheRequestAfterAManualPingIsHotNotCold(t *testing.T) { + const prefix = 100_000 + run := func(manual bool) accountant.Result { + st, pings := newSavingsTracker(), NewPingStore() + t0 := time.Unix(1_700_000_000, 0) + body := func(n int) []byte { + // 4 bytes a token: the first message is the 100,000-token prefix. + msgs := `[{"role":"user","content":"` + strings.Repeat("x", prefix*4) + `","cache_control":{"type":"ephemeral"}}` + for i := 1; i < n; i++ { + msgs += `,{"role":"assistant","content":"ok"},{"role":"user","content":"more","cache_control":{"type":"ephemeral"}}` + } + return []byte(`{"model":"m","system":[{"type":"text","text":"sys"}],"messages":` + msgs + `]}`) + } + turn := func(at time.Time, b []byte, u Usage) accountant.Result { + req := st.begin(accountant.RequestMeta{RequestID: at.String(), Session: "s", Model: "m", At: at}, b) + th := accountant.Thread{ID: "main"} + st.end(req, th, b, nil, entryFacts(bschemas.Anthropic, "m", b, pings, "s", th)) + return st.compute(req, u) + } + turn(t0, body(1), Usage{CacheWrite: prefix}) + r := okReport() + r.Session, r.Thread, r.TS, r.Window, r.Manual = "s", "", t0.Add(4*time.Minute).UnixMilli(), t0, manual + r.FreshInput, r.CacheRead, r.CacheWrite, r.CacheWrite1h, r.Output = 0, prefix, 0, 0, 1 + if _, ok := PingSavings(r, pings); !ok { + t.Fatal("the ping was not converted") + } + return turn(t0.Add(8*time.Minute), body(2), Usage{CacheRead: prefix, CacheWrite: 20}) + } + auto, manual := run(false), run(true) + for name, res := range map[string]accountant.Result{"automatic": auto, "manual": manual} { + if res.Phase != accountant.PhaseHot { + t.Errorf("%s ping: the next request is %q, want hot", name, res.Phase) + } + if len(res.Unpriced) != 0 || len(res.Checks) != 0 { + t.Errorf("%s ping: unpriced %v checks %v", name, res.Unpriced, res.Checks) + } + } + rescue := func(res accountant.Result) (n int) { + for _, e := range res.CacheEffects { + if e.Kind == accountant.EffectRescue { + n += e.Saved.CacheWrite5m + } + } + return n + } + if rescue(auto) == 0 { + t.Error("the automatic ping booked no rescue: the control is broken") + } + if n := rescue(manual); n != 0 { + t.Errorf("the manual ping's write is booked a second time as a rescue of %d tokens", n) + } +} From 382dcbcbb9990facb1b18a16a233c6a733b0042a Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 08:40:18 +0200 Subject: [PATCH 46/58] build: Pin accountant to main after the short-turn split fix Pin accountant at c1f0f7a. It holds PR 48: a delivered split on a short conversation is no longer dropped, and a full read does not book a phantom split effect. The pricing branch pinned the PR branch commit 23e2ed1 before. proxy, dash, apply, keepalive, components and tooldefer tests pass with it. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e8b4c23e..25a26601 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.26.4 require github.com/maximhq/bifrost/core v1.7.0 // accountant prices context operations. This module's tests import it and its shared test cases. -require github.com/rossoctl/accountant v0.1.1-0.20261105060029-23e2ed1c3cc1 +require github.com/rossoctl/accountant v0.1.1-0.20261105063645-c1f0f7a052db require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 diff --git a/go.sum b/go.sum index 3d7741d1..26bf71a5 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rossoctl/accountant v0.1.1-0.20261105060029-23e2ed1c3cc1 h1:BUnCM/hVdPgTHrNncJT31g1kFn3PTWYIBzdiIXmAzns= -github.com/rossoctl/accountant v0.1.1-0.20261105060029-23e2ed1c3cc1/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261105063645-c1f0f7a052db h1:v/nXeG3UNGlkktPIF7AxhRMEJ2kkMfB8F4RI4qXxz2c= +github.com/rossoctl/accountant v0.1.1-0.20261105063645-c1f0f7a052db/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 883f834cf75a815b3f40d2481c20bf4da263d9de Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 08:57:24 +0200 Subject: [PATCH 47/58] fix(apply): Keep the kept-wire preference off standalone messages The #910 fix preferred a slot of a wire message that a kept tool message names. A standalone user text has the same bytes as a sibling text, so it could take the sibling's wire and put a tool_result before its tool_use. Apply the preference only to a sibling-text slot whose wire is named by the adjacent kept tool message. Check the order of the kept wire messages at the end of the rebuild, and forward the original body if it is broken. Also hide the "estimate" suffix when there is no manual ping, and split a long sentence in docs/dashboard.md. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- apply/apply.go | 86 +++++++++++--- apply/ops_splice_test.go | 188 +++++++++++++++++++++++++++++++ apply/wireorder_internal_test.go | 32 ++++++ dash/ui/app.js | 2 +- docs/dashboard.md | 3 +- 5 files changed, 295 insertions(+), 16 deletions(-) create mode 100644 apply/wireorder_internal_test.go diff --git a/apply/apply.go b/apply/apply.go index 9bb13a7a..0f6ee85c 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -1982,21 +1982,27 @@ func rebuildCountChanged(body []byte, orig []gjson.Result, slots []slot, out []b toolSlotByID[id] = k } } - // keptWire is the wire messages a kept tool message names, by its tool_call_id. A text block that sits + // keptWireOf is the wire message a kept tool message names, by its tool_call_id. A text block that sits // beside a tool_result in one wire message has the same bytes on every turn ("continue"), so matching // it by bytes alone takes the first turn's slot, which a splice may have removed. It would emit that // turn's wire message, with its orphan tool_result, and leave the survivor's message out (#910). - // A byte match therefore prefers a slot of a wire message a kept tool message already names. - keptWire := map[int]bool{} - for i := range out { - if out[i].Role != bschemas.ChatMessageRoleTool || out[i].ChatToolMessage == nil || out[i].ChatToolMessage.ToolCallID == nil { - continue + // A sibling text is therefore matched to a slot of the wire message that its ADJACENT kept tool message + // names. Nothing else uses this preference: a standalone message (a wholeMessage slot) has the same + // bytes as a sibling text, and giving it the sibling's wire would put a tool_result before its tool_use. + keptWireOf := func(i int) (int, bool) { + if i < 0 || i >= len(out) { + return 0, false } - if k, ok := toolSlotByID[*out[i].ChatToolMessage.ToolCallID]; ok { - if bi, _, ok := splitSlotPath(slots[k].path); ok { - keptWire[bi] = true - } + m := out[i] + if m.Role != bschemas.ChatMessageRoleTool || m.ChatToolMessage == nil || m.ChatToolMessage.ToolCallID == nil { + return 0, false } + k, ok := toolSlotByID[*m.ChatToolMessage.ToolCallID] + if !ok { + return 0, false + } + bi, _, ok := splitSlotPath(slots[k].path) + return bi, ok } var parts [][]byte // emitted guards against emitting one body message TWICE: several normalized @@ -2004,6 +2010,7 @@ func rebuildCountChanged(body []byte, orig []gjson.Result, slots []slot, out []b // tool_result blocks), and a count-changing component may leave them // non-contiguous — which duplicated the raw message, and with it its tool_use_id. emitted := map[int]bool{} + var order []int // the body index of each kept message, in emitted order for i := range out { mb, err := json.Marshal(out[i]) if err != nil { @@ -2020,13 +2027,19 @@ func rebuildCountChanged(body []byte, orig []gjson.Result, slots []slot, out []b } } if matched < 0 { + // The wire messages of the kept tool messages next to this one: a sibling text sits before or + // after the tool_result block of its own wire message. + prevWire, hasPrev := keptWireOf(i - 1) + nextWire, hasNext := keptWireOf(i + 1) for k := range slots { if used[k] || !bytes.Equal(mb, slots[k].pre) { continue } - if bi, _, ok := splitSlotPath(slots[k].path); ok && keptWire[bi] { - matched = k // the survivor of a kept wire message - break + if slots[k].kind == anthropicSiblingText { + if bi, _, ok := splitSlotPath(slots[k].path); ok && ((hasPrev && bi == prevWire) || (hasNext && bi == nextWire)) { + matched = k // the sibling text of the kept tool message beside it + break + } } if matched < 0 { matched = k @@ -2047,6 +2060,12 @@ func rebuildCountChanged(body []byte, orig []gjson.Result, slots []slot, out []b } emitted[bi] = true parts = append(parts, []byte(orig[bi].Raw)) + order = append(order, bi) + } + // Defence in depth: a matching mistake must never reach the provider as an invalid body. + if !wireOrderValid(orig, order) { + slog.Warn("context-guru: rebuilt messages break the wire order; forwarding the original", "kept", order) + return body, false } var buf bytes.Buffer buf.WriteByte('[') @@ -2064,6 +2083,47 @@ func rebuildCountChanged(body []byte, orig []gjson.Result, slots []slot, out []b return res, true } +// wireOrderValid checks the kept body messages of a rebuild, in emitted order. Two rules must hold. +// The kept messages keep their original relative order. A message that holds a tool_result does not +// come before the message that holds its tool_use. If a rule fails, the caller forwards the original. +func wireOrderValid(orig []gjson.Result, order []int) bool { + useAt := map[string]int{} // tool_use id -> position in order of the message that holds it + for pos, bi := range order { + if pos > 0 && bi <= order[pos-1] { + return false + } + for _, blk := range orig[bi].Get("content").Array() { + if blk.Get("type").String() == "tool_use" { + useAt[blk.Get("id").String()] = pos + } + } + } + // A tool_result is checked against the use in the ORIGINAL body, so a use that was removed is not blamed here. + origUse := map[string]int{} + for bi := range orig { + for _, blk := range orig[bi].Get("content").Array() { + if blk.Get("type").String() == "tool_use" { + origUse[blk.Get("id").String()] = bi + } + } + } + for pos, bi := range order { + for _, blk := range orig[bi].Get("content").Array() { + if blk.Get("type").String() != "tool_result" { + continue + } + id := blk.Get("tool_use_id").String() + if u, ok := origUse[id]; ok && u >= bi { + continue // the original body already had this order; not our change + } + if up, ok := useAt[id]; ok && up >= pos { + return false + } + } + } + return true +} + // toolCallIDAt reads the tool_use_id of the tool_result block a tool-text slot points at. // The slot path is "messages..content..content", so the block is its parent. func toolCallIDAt(orig []gjson.Result, s slot) string { diff --git a/apply/ops_splice_test.go b/apply/ops_splice_test.go index 00fad58b..68bdb410 100644 --- a/apply/ops_splice_test.go +++ b/apply/ops_splice_test.go @@ -1,6 +1,8 @@ package apply_test import ( + "math/rand" + "strconv" "strings" "testing" @@ -110,3 +112,189 @@ func TestSpliceKeepsSurvivorsWithIdenticalSiblingText(t *testing.T) { t.Fatalf("ops = %+v dropped = %d: the splice was not reported", res.Trace.Ops, res.Trace.OpsDropped) } } + +// dropExchanges is a count-changing component for the tests below. It removes every tool exchange whose +// tool_use id is in dropIDs (the assistant message, its tool messages, and the sibling user text that follows +// them up to the next assistant message) and puts one summary message where the first removed exchange began. +// The tests do not run in parallel, so the package variable is safe. +type dropExchanges struct{} + +var dropIDs map[string]bool + +func (dropExchanges) Name() string { return "dropex" } +func (dropExchanges) Enabled(*components.Ctx) bool { return true } +func (dropExchanges) Reformat(req *bschemas.BifrostChatRequest, _ *components.Report, _ *components.Ctx) error { + var out []bschemas.ChatMessage + summarized := false + in := req.Input + for i := 0; i < len(in); i++ { + m := in[i] + if m.Role == bschemas.ChatMessageRoleAssistant && m.ChatAssistantMessage != nil && + len(m.ChatAssistantMessage.ToolCalls) > 0 && + m.ChatAssistantMessage.ToolCalls[0].ID != nil && dropIDs[*m.ChatAssistantMessage.ToolCalls[0].ID] { + j := i + 1 + for j < len(in) && in[j].Role != bschemas.ChatMessageRoleAssistant { + j++ + } + if !summarized { + sum := m + sum.ChatAssistantMessage = nil + schema.SetMessageText(&sum, "summary") + out = append(out, sum) + summarized = true + } + i = j - 1 + continue + } + out = append(out, m) + } + req.Input = out + return nil +} + +func init() { + components.Register("dropex", func([]byte) (components.Component, error) { return dropExchanges{}, nil }) +} + +func spliceTU(id string) string { + return `{"role":"assistant","content":[{"type":"tool_use","id":"` + id + `","name":"Bash","input":{"command":"` + id + `"}}]}` +} + +func spliceTR(id, text string) string { + return `{"role":"user","content":[{"type":"tool_result","tool_use_id":"` + id + `","content":"out ` + id + `"},{"type":"text","text":"` + text + `"}]}` +} + +// checkWireOrder asserts the invariant on a forwarded body: every tool_result follows the message that holds +// its tool_use, and the kept messages are exactly the wanted raw messages, in order, each once. +func checkWireOrder(t *testing.T, body []byte, want []string) { + t.Helper() + msgs := gjson.GetBytes(body, "messages").Array() + seenUse := map[string]bool{} + var got []string + for i, m := range msgs { + for _, blk := range m.Get("content").Array() { + if blk.Get("type").String() == "tool_result" && !seenUse[blk.Get("tool_use_id").String()] { + t.Fatalf("message %d holds a tool_result for %q before its tool_use: %s", i, blk.Get("tool_use_id").String(), body) + } + } + for _, blk := range m.Get("content").Array() { + if blk.Get("type").String() == "tool_use" { + seenUse[blk.Get("id").String()] = true + } + } + got = append(got, m.Raw) + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("forwarded messages:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +func summaryMsg() string { return `{"role":"assistant","content":"summary"}` } + +// Round-1 finding of #930, trace A: a standalone user text and a kept exchange's sibling text have the same +// bytes. The standalone message must keep its own wire message. +func TestSpliceStandaloneTextEqualToKeptSiblingText(t *testing.T) { + dropIDs = map[string]bool{"d": true} + u0 := `{"role":"user","content":"CALL_TOOL again"}` + b := []string{u0, spliceTU("b"), spliceTR("b", "CALL_TOOL again"), spliceTU("d"), spliceTR("d", "x")} + body := `{"model":"claude-x","messages":[` + strings.Join(b, ",") + `]}` + res := traceOf(t, "dropex", bschemas.Anthropic, body) + if !res.Changed { + t.Fatal("precondition: the count-changing rewrite must be written back") + } + checkWireOrder(t, res.Body, []string{b[0], b[1], b[2], summaryMsg()}) +} + +// Round-1 finding of #930, trace B. +func TestSpliceKeptPlainUserBeforeExchangeWithSameText(t *testing.T) { + dropIDs = map[string]bool{"d": true} + b := []string{`{"role":"user","content":"continue"}`, spliceTU("a"), spliceTR("a", "continue"), spliceTU("d"), spliceTR("d", "x")} + body := `{"model":"claude-x","messages":[` + strings.Join(b, ",") + `]}` + res := traceOf(t, "dropex", bschemas.Anthropic, body) + if !res.Changed { + t.Fatal("precondition: the count-changing rewrite must be written back") + } + checkWireOrder(t, res.Body, []string{b[0], b[1], b[2], summaryMsg()}) +} + +// Two kept exchanges with the same sibling text, a standalone text of that text before and after each. +func TestSpliceRepeatedSiblingTextAdversarial(t *testing.T) { + dropIDs = map[string]bool{"d": true} + s := "CALL_TOOL again" + u := `{"role":"user","content":"` + s + `"}` + a1 := `{"role":"assistant","content":"ok1"}` + a2 := `{"role":"assistant","content":"ok2"}` + a3 := `{"role":"assistant","content":"ok3"}` + b := []string{u, a1, u, spliceTU("a"), spliceTR("a", s), a2, u, spliceTU("b"), spliceTR("b", s), a3, u, spliceTU("d"), spliceTR("d", s)} + body := `{"model":"claude-x","messages":[` + strings.Join(b, ",") + `]}` + res := traceOf(t, "dropex", bschemas.Anthropic, body) + if !res.Changed { + t.Fatal("precondition: the count-changing rewrite must be written back") + } + checkWireOrder(t, res.Body, append(append([]string{}, b[:11]...), summaryMsg())) +} + +// A seeded property test: random transcripts of standalone texts and tool exchanges with repeated sibling text, +// a random subset of the exchanges dropped. Every result must keep the invariant and the exact kept sequence. +func TestSplicePropertyKeepsWireInvariant(t *testing.T) { + texts := []string{"CALL_TOOL again", "continue", "x"} + rng := rand.New(rand.NewSource(930)) + changed := 0 + const cases = 400 + for n := 0; n < cases; n++ { + var wire []string + var exch [][]string // the wire messages of each exchange, with its id + var ids []string + textOf := func() string { return texts[rng.Intn(len(texts))] } + turns := 2 + rng.Intn(6) + for k := 0; k < turns; k++ { + if rng.Intn(2) == 0 { + wire = append(wire, `{"role":"user","content":"`+textOf()+`"}`) + wire = append(wire, `{"role":"assistant","content":"ok `+strconv.Itoa(k)+`"}`) + continue + } + id := "t" + strconv.Itoa(k) + ids = append(ids, id) + wire = append(wire, spliceTU(id), spliceTR(id, textOf())) + exch = append(exch, []string{spliceTU(id), spliceTR(id, "")}) + wire = append(wire, `{"role":"assistant","content":"ok `+strconv.Itoa(k)+`"}`) + } + if len(ids) == 0 { + continue + } + dropIDs = map[string]bool{} + for _, id := range ids { + if rng.Intn(2) == 0 { + dropIDs[id] = true + } + } + if len(dropIDs) == 0 { + dropIDs[ids[rng.Intn(len(ids))]] = true + } + // The expected kept sequence: drop the wire messages of a dropped exchange, summary at the first drop. + var want []string + sum := false + for i := 0; i < len(wire); i++ { + id := gjson.Get(wire[i], "content.0.id").String() + if id != "" && dropIDs[id] { + if !sum { + want = append(want, summaryMsg()) + sum = true + } + i += 1 // skip the tool_result message; the plain assistant after it is kept + continue + } + want = append(want, wire[i]) + } + body := `{"model":"claude-x","messages":[` + strings.Join(wire, ",") + `]}` + res := traceOf(t, "dropex", bschemas.Anthropic, body) + if !res.Changed { + continue + } + changed++ + checkWireOrder(t, res.Body, want) + } + if changed < cases/2 { + t.Fatalf("only %d of %d generated cases were rewritten; the test is not exercising the rebuild", changed, cases) + } +} diff --git a/apply/wireorder_internal_test.go b/apply/wireorder_internal_test.go new file mode 100644 index 00000000..0e31020c --- /dev/null +++ b/apply/wireorder_internal_test.go @@ -0,0 +1,32 @@ +package apply + +import ( + "testing" + + "github.com/tidwall/gjson" +) + +// wireOrderValid is the last check of the count-change rebuild. It must reject an order that a matching +// mistake could produce, so the caller forwards the original body. +func TestWireOrderValid(t *testing.T) { + orig := gjson.Parse(`[ + {"role":"user","content":"u"}, + {"role":"assistant","content":[{"type":"tool_use","id":"a","name":"Bash","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"a","content":"o"},{"type":"text","text":"u"}]} + ]`).Array() + for _, c := range []struct { + name string + order []int + want bool + }{ + {"original order", []int{0, 1, 2}, true}, + {"a dropped middle", []int{0, 2}, true}, // a lone tool_result is the splice's own business + {"tool_result before its tool_use", []int{2, 1}, false}, + {"relative order of kept messages changed", []int{1, 0, 2}, false}, + {"a message twice", []int{0, 0}, false}, + } { + if got := wireOrderValid(orig, c.order); got != c.want { + t.Errorf("%s: wireOrderValid(%v) = %v, want %v", c.name, c.order, got, c.want) + } + } +} diff --git a/dash/ui/app.js b/dash/ui/app.js index 94015415..469829a8 100644 --- a/dash/ui/app.js +++ b/dash/ui/app.js @@ -1783,7 +1783,7 @@ function renderSavedLedger(o, ka, partial) { const cacheRan = o.keepalive_pings > 0 || !!o.keepalive_net_usd || parts.reorder !== 0 || (mp.count || 0) > 0; const brk = el('ul', { class: 'ov-split', 'data-testid': 'saved-cache-split' }, el('li', {}, el('span', { text: 'Automatic pings' }), el('strong', { text: usd(parts.keepalive) })), - el('li', { 'data-testid': 'saved-manual' }, el('span', { text: 'Manual pings' + (mp.count ? ' (' + num(mp.count) + ')' : '') + ' · estimate' }), + el('li', { 'data-testid': 'saved-manual' }, el('span', { text: 'Manual pings' + (mp.count ? ' (' + num(mp.count) + ')' : '') + (mp.count ? ' · estimate' : '') }), el('strong', { text: usd(parts.manual) })), parts.reorder !== 0 ? el('li', {}, el('span', { text: 'Cache reorder' }), el('strong', { text: usd(parts.reorder) })) : null); // The strategy's name, as the plugin calls it, beside "Cache" the way the preset's name sits beside diff --git a/docs/dashboard.md b/docs/dashboard.md index f33f37b8..ce0d63ed 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -308,8 +308,7 @@ per line when there are several headers). The proxy never forwards it upstream. Its row is a keep-alive row with `manual_ping` set. `GET /api/stats` reports these rows as `manual_pings` (`count`, `ping_usd`, `saved_usd` net of the pings' cost, and `writes_avoided`: the pings that found the cache warm). They are a savings category of their own: they join `total_saved_usd`, and no cache strategy's ledger counts them. -The figure is an estimate, not a measurement: it is the write the ping avoided, taken from the cache lifetime and the prefix size, and no request read it back. The Overview marks it "estimate". The figure is an upper bound. A press needs the proxy to hold the thread for keep-alive, so with -automatic pings on it can overlap the keep-alive credit for the same avoided write. `saved_usd` is +The figure is an estimate, not a measurement. It is the write the ping avoided. It comes from the cache lifetime and the prefix size. No request read it back. The Overview marks it "estimate". The figure is an upper bound. A press needs the proxy to hold the thread for keep-alive. With automatic pings on, it can overlap the keep-alive credit for the same avoided write. `saved_usd` is `avoided - ping` summed over all presses, so a ping that missed shows as a loss; the per-press `saved_usd` in the reply is never below 0, so the two need not match. A press is refused with `409 over budget` when one ping would cost more than the per-ping ceiling, and `?tenant=` is ignored: From d9276e157962d16206f721bead5f11dfe2168f0c Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Thu, 5 Nov 2026 09:08:27 +0200 Subject: [PATCH 48/58] test: Remove an unused slice from the splice property test Both reviewers of #930 noted that exch is written and never read. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- apply/ops_splice_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/apply/ops_splice_test.go b/apply/ops_splice_test.go index 68bdb410..e26bea34 100644 --- a/apply/ops_splice_test.go +++ b/apply/ops_splice_test.go @@ -243,7 +243,6 @@ func TestSplicePropertyKeepsWireInvariant(t *testing.T) { const cases = 400 for n := 0; n < cases; n++ { var wire []string - var exch [][]string // the wire messages of each exchange, with its id var ids []string textOf := func() string { return texts[rng.Intn(len(texts))] } turns := 2 + rng.Intn(6) @@ -256,7 +255,6 @@ func TestSplicePropertyKeepsWireInvariant(t *testing.T) { id := "t" + strconv.Itoa(k) ids = append(ids, id) wire = append(wire, spliceTU(id), spliceTR(id, textOf())) - exch = append(exch, []string{spliceTU(id), spliceTR(id, "")}) wire = append(wire, `{"role":"assistant","content":"ok `+strconv.Itoa(k)+`"}`) } if len(ids) == 0 { From 441604d66712d3a693252cbfa493d713ad5dcba9 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sat, 7 Nov 2026 17:50:12 +0200 Subject: [PATCH 49/58] fix(accountant): Price the mark premium and pin accountant 41d7cc5 Accountant PR 50 adds the markPremium effect: the write premium of a breakpoint the host added. The dashboard did not know the kind. It put the effect in the unpriced list as effect:markPremium:unknown_kind, and a request with any unpriced entry books no saving. The default tail breakpoint adds such a mark. So with the new library every request in the default configuration fell back with "unpriced" and the Overview showed net saved 0. The earlier tests passed because they ran against the older library, which did not emit the effect. Charge the premium as a cost to the operation whose span covers the mark, else to the tail breakpoint, the only component that adds a mark. It is in the delta and has a row, so the sum invariant holds. Pin accountant at 41d7cc5: PRs 40 to 48 and 50. Tail scenario, 7 requests, default config, against the fake upstream: true saving 0.0736 (control 0.1315527 - treatment 0.0579329) before 0.0779 (+5.8%) now 0.0705 (-4.2%, the safe direction) No request falls back. The accountant case suite agrees with the dashboard for all ten markpremium cases. TestProxyPathMatchesTheAccountantCases now counts a mark premium in the expected figure of the component that owns it. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accountant_agree_test.go | 20 ++++++++++++--- dash/accountant_price.go | 13 ++++++++++ dash/accountant_switch_test.go | 47 ++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +-- 5 files changed, 79 insertions(+), 7 deletions(-) diff --git a/dash/accountant_agree_test.go b/dash/accountant_agree_test.go index 4d6120d5..44b83c63 100644 --- a/dash/accountant_agree_test.go +++ b/dash/accountant_agree_test.go @@ -120,8 +120,9 @@ func sharedComp(c cases.Case, comp string) bool { // expectedByComponent is the case's expected figure of each component, the way AccFigure.ByComponent // builds it: the priced rewrite and setting ops (a spend is priced by the host, so it is left out) -// plus the counted breaks whose ChangeID a component owns. Rescue, split and TTL effects are their -// own AccFigure fields, not component figures; a counted break nobody owns is EndedBreakUSD. +// plus the counted breaks whose ChangeID a component owns, plus each mark premium (on the owner of its +// ChangeID, else on the tail breakpoint). Rescue, split and TTL effects are their own AccFigure fields, +// not component figures; a counted break nobody owns is EndedBreakUSD. func expectedByComponent(c cases.Case, s cases.Step, comps map[string]string) map[string]int64 { picos := map[string]int64{} rate := c.StepRate(s) @@ -134,10 +135,21 @@ func expectedByComponent(c cases.Case, s cases.Step, comps map[string]string) ma } } for _, e := range s.Expected.Effects { - if !e.Counted() || e.Kind != accountant.EffectBreak { + if !e.Counted() { continue } - if comp, ok := comps[e.ChangeID]; ok && e.ChangeID != "" { + switch e.Kind { + case accountant.EffectBreak: + if comp, ok := comps[e.ChangeID]; ok && e.ChangeID != "" { + picos[comp] += rate.Picos(e.Saved) - rate.Picos(e.Caused) + } + case accountant.EffectMarkPremium: + // The premium of a breakpoint the host added: a cost on the component whose span covers the + // mark, else on the tail breakpoint (AccountantFigure does the same). + comp, ok := comps[e.ChangeID] + if !ok || e.ChangeID == "" { + comp = "tailbreakpoint" + } picos[comp] += rate.Picos(e.Saved) - rate.Picos(e.Caused) } } diff --git a/dash/accountant_price.go b/dash/accountant_price.go index c17bd093..e263ec30 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -92,6 +92,8 @@ func (e *Event) AccountantFigure(p modelinfo.Price) AccFigure { // split: same shape as rescue: a saving. // ttlPremium: Saved is the 5m write, Caused the dearer 1h write: value < 0 (a cost). // ttlRescue: Saved is the 5m write, Caused the read: a saving. + // markPremium: Saved is the input the client body would have billed, Caused the dearer write the + // host's added mark made the provider bill: value < 0 (a cost). value := tierUSD(p, ce.Saved) - tierUSD(p, ce.Caused) switch ce.Kind { case accountant.EffectBreak: @@ -113,6 +115,17 @@ func (e *Event) AccountantFigure(p modelinfo.Price) AccFigure { f.SplitUSD += value case accountant.EffectTTLPremium, accountant.EffectTTLRescue: f.TTLUSD += value + case accountant.EffectMarkPremium: + // The write premium of a breakpoint the host added: a cost (value < 0), charged to the + // operation whose span covers the mark. With no owner it belongs to the tail breakpoint, + // the only component that adds a mark (accountant design §4.4). Without this case the + // effect was unpriced and every request with the default tail breakpoint booked no saving. + c, ok := owner[ce.ChangeID] + if !ok || ce.ChangeID == "" { + c = "tailbreakpoint" + } + f.ByComponent[c] += value + f.DeltaUSD += value default: f.Unpriced = append(f.Unpriced, "effect:"+string(ce.Kind)+":unknown_kind") continue diff --git a/dash/accountant_switch_test.go b/dash/accountant_switch_test.go index c7e08e7b..1927a7d1 100644 --- a/dash/accountant_switch_test.go +++ b/dash/accountant_switch_test.go @@ -416,3 +416,50 @@ func TestPriceMode_ToolSectionTokensAreNotBoundedByMessageTokens(t *testing.T) { t.Fatalf("message operation: fallback %q, want %q", fb, FallbackImplausibleSize) } } + +// A markPremium effect is the write premium of a breakpoint the host added (accountant#49). It is a cost. +// Before the dashboard knew the kind, every request with the default tail breakpoint fell back with +// "unpriced" (effect:markPremium:unknown_kind) and booked no saving at all. +func TestPriceMode_MarkPremiumIsACostOnTheComponentThatAddedTheMark(t *testing.T) { + premium := accountant.CacheEffect{Kind: accountant.EffectMarkPremium, + Saved: accountant.TierTokens{Input: 2000}, Caused: accountant.TierTokens{CacheWrite5m: 2000}} + // 2,000 tokens move from input ($3/M) to a 5m write ($3.75/M): a cost of 2000 x 0.75e-6. + cost := 2000 * (3.75e-6 - 3e-6) + run := func(changeID string) *Event { + e := switchEvent() + e.Components = append(e.Components, CompRow{Component: "tailbreakpoint", Acted: true, Mutated: true}) + ops := e.accRes.Ops + ops = append(ops, accountant.PricedOp{Operator: "context-guru.tailbreakpoint", ChangeID: "cg:tailbreakpoint:x"}) + res := *e.accRes + res.Ops = ops + p := premium + p.ChangeID = changeID + res.CacheEffects = append(res.CacheEffects, p) + e.SetAccountant(res) + e.PriceMode(accTestPrice, true) + return e + } + for name, changeID := range map[string]string{"owned": "cg:tailbreakpoint:x", "no owner": ""} { + t.Run(name, func(t *testing.T) { + e := run(changeID) + if fb := e.AccountantFallback(); fb != "" { + t.Fatalf("fell back: %q (a markPremium effect must be priced)", fb) + } + var tail, sum float64 + for _, c := range e.Components { + sum += c.SavedUSD + if c.Component == "tailbreakpoint" { + tail = c.SavedUSD + } + } + if !accNear(tail, -cost) { + t.Fatalf("tailbreakpoint saved_usd = %v, want the premium as a cost %v", tail, -cost) + } + // The mask saving (1000 cache-read tokens) minus the premium is the whole delta, and the rows add up to it. + want := 1000*0.3e-6 - cost + if !accNear(e.BaselineCostUSD-e.CostUSD, want) || !accNear(sum, want) { + t.Fatalf("baseline delta %v, component sum %v, want %v", e.BaselineCostUSD-e.CostUSD, sum, want) + } + }) + } +} diff --git a/go.mod b/go.mod index 25a26601..cfd8c842 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.26.4 require github.com/maximhq/bifrost/core v1.7.0 // accountant prices context operations. This module's tests import it and its shared test cases. -require github.com/rossoctl/accountant v0.1.1-0.20261105063645-c1f0f7a052db +require github.com/rossoctl/accountant v0.1.1-0.20261107153853-41d7cc522cc2 require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 diff --git a/go.sum b/go.sum index 26bf71a5..ca82637e 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rossoctl/accountant v0.1.1-0.20261105063645-c1f0f7a052db h1:v/nXeG3UNGlkktPIF7AxhRMEJ2kkMfB8F4RI4qXxz2c= -github.com/rossoctl/accountant v0.1.1-0.20261105063645-c1f0f7a052db/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261107153853-41d7cc522cc2 h1:pNKgg7fjZzIlMZZzRqTn6ijsdNlyhM4J+8ZblABV94Q= +github.com/rossoctl/accountant v0.1.1-0.20261107153853-41d7cc522cc2/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 08942987446b3a125ec8f9d1a100e8dd48e5d9c4 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sat, 7 Nov 2026 17:58:39 +0200 Subject: [PATCH 50/58] docs: Say that cacheinject marks land on the tail breakpoint row Reviewer 2 of #930 noted that the comment on the mark premium was not exact. The opt-in cacheinject also adds marks. Its premium with no covering operation lands on the tail breakpoint row. The total is right and only the row name is approximate. Comment only. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accountant_price.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dash/accountant_price.go b/dash/accountant_price.go index e263ec30..c7c05c04 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -117,8 +117,10 @@ func (e *Event) AccountantFigure(p modelinfo.Price) AccFigure { f.TTLUSD += value case accountant.EffectMarkPremium: // The write premium of a breakpoint the host added: a cost (value < 0), charged to the - // operation whose span covers the mark. With no owner it belongs to the tail breakpoint, - // the only component that adds a mark (accountant design §4.4). Without this case the + // operation whose span covers the mark. With no owner it goes on the tail breakpoint row, + // the component that adds the mark in every default preset (accountant design §4.4). The + // opt-in cacheinject also adds marks: its premium with no covering operation lands on the same + // row, so the total is right and only the row name is approximate. Without this case the // effect was unpriced and every request with the default tail breakpoint booked no saving. c, ok := owner[ce.ChangeID] if !ok || ce.ChangeID == "" { From d9f0396a22cbace4e6229b23f162ebb52594865f Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sun, 8 Nov 2026 07:26:05 +0200 Subject: [PATCH 51/58] perf(proxy): Parse the request body once for the accountant accParse returns the Body and the breakpoints from one pass over the top-level keys, used by begin and end. Messages without cache_control are sliced from the original bytes instead of copied. Hashes, sizes and breakpoints are unchanged; a test holds them to a copy of the old code. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- proxy/accountant_parse_equiv_test.go | 165 +++++++++++++++++++++++++++ proxy/accountant_savings.go | 4 +- proxy/accountant_savings_body.go | 111 +++++++++++++++--- 3 files changed, 259 insertions(+), 21 deletions(-) create mode 100644 proxy/accountant_parse_equiv_test.go diff --git a/proxy/accountant_parse_equiv_test.go b/proxy/accountant_parse_equiv_test.go new file mode 100644 index 00000000..f245ccf0 --- /dev/null +++ b/proxy/accountant_parse_equiv_test.go @@ -0,0 +1,165 @@ +package proxy + +import ( + "fmt" + "strings" + "testing" + + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + "github.com/rossoctl/accountant" +) + +// TestAccParseMatchesReference holds accParse to the implementation it replaced (kept below as oldAccBodies and +// oldAccBreakpoints): the same Body (Start, every Pos hash and Bytes) and the same breakpoints. +func TestAccParseMatchesReference(t *testing.T) { + big := strings.Repeat("x", 2000) + cases := map[string]string{ + "empty": ``, + "invalid": `{"messages":[{"role":"user"`, + "not object": `[1,2,3]`, + "no messages": `{"system":"s","tools":[{"name":"a"}]}`, + "messages object": `{"messages":{"a":1}}`, + "string system": `{"system":"be nice","messages":[{"role":"user","content":"hi"}]}`, + "array system": `{"system":[{"type":"text","text":"a","cache_control":{"type":"ephemeral","ttl":"1h"}},{"type":"text","text":"b"}],"messages":[{"role":"user","content":"hi"}]}`, + "tools": `{"tools":[{"name":"a","cache_control":{"type":"ephemeral"}},{"name":"b"}],"system":"s","messages":[{"role":"user","content":"hi"}]}`, + "msg mark": `{"messages":[{"role":"user","content":"hi","cache_control":{"type":"ephemeral"}},{"role":"assistant","content":"yo"}]}`, + "block mark": `{"messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral","ttl":"1h"}},{"type":"text","text":"b","cache_control":{"type":"ephemeral"}}]}]}`, + "nested tool_res": `{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"1","content":[{"type":"text","text":"r","cache_control":{"type":"ephemeral"}}]}]}]}`, + "mark in string": `{"messages":[{"role":"user","content":"cache_control in text"}]}`, + "messages first": `{"messages":[{"role":"user","content":"` + big + `"}],"system":"s","tools":[{"name":"t"}]}`, + "duplicate keys": `{"messages":[{"role":"user","content":"a"}],"messages":[{"role":"user","content":"b"}],"system":"x","system":"y"}`, + "whitespace": " \n{ \"messages\" : [ { \"role\" : \"user\" } ] , \"system\" : [ \"s\" ] }", + "escaped key text": `{"messages":[{"role":"user","content":"a\"messages\":[]"}],"nested":{"messages":[1]}}`, + "nested only": `{"x":{"messages":[{"role":"user"}]}}`, + } + cases["bench"] = string(benchBody()) + for name, in := range cases { + t.Run(name, func(t *testing.T) { + raw := []byte(in) + wb, wbp := oldAccBodies(raw), oldAccBreakpoints(raw) + gb, gbp := accParse(raw) + if gb.Start != wb.Start || fmt.Sprint(gb.Pos) != fmt.Sprint(wb.Pos) { + t.Fatalf("body\n got %+v\nwant %+v", gb, wb) + } + if fmt.Sprint(gbp) != fmt.Sprint(wbp) || (gbp == nil) != (wbp == nil) { + t.Fatalf("breakpoints got %+v want %+v", gbp, wbp) + } + if g := accBodies(raw); fmt.Sprint(g.Pos) != fmt.Sprint(wb.Pos) { + t.Fatalf("accBodies wrapper differs") + } + if g := accBreakpoints(raw); fmt.Sprint(g) != fmt.Sprint(wbp) { + t.Fatalf("accBreakpoints wrapper differs") + } + }) + } +} + +// oldAccBodies is the reference copy. +func oldAccBodies(raw []byte) accountant.Body { + if !gjson.GetBytes(raw, "messages").IsArray() { + return accountant.NewBody(nil, nil, nil) + } + var tools, system, msgs [][]byte + var sizes []int // the wire size of each element, in hash order + add := func(dst *[][]byte, v gjson.Result) { + *dst = append(*dst, oldWithoutCacheMarks(v.Raw)) + sizes = append(sizes, len(v.Raw)) + } + gjson.GetBytes(raw, "tools").ForEach(func(_, v gjson.Result) bool { add(&tools, v); return true }) + if s := gjson.GetBytes(raw, "system"); s.IsArray() { + s.ForEach(func(_, v gjson.Result) bool { add(&system, v); return true }) + } else if s.Exists() { + add(&system, s) + } + gjson.GetBytes(raw, "messages").ForEach(func(_, v gjson.Result) bool { add(&msgs, v); return true }) + built := accountant.NewBody(tools, system, msgs) + for i := range built.Pos { + built.Pos[i].Bytes = int32(sizes[i]) + } + // NewBody also keeps a table of cumulative bytes, built from the mark-free input, which the library prefers to + // summing Pos[i].Bytes. A Body without that table sums the positions, so every byte count the library reads + // (the token ratio, the span of a splice, sawOriginal) is the wire size, as the operations' sizes are. A body + // decoded from a snapshot has no table either, so a restart changes nothing. The table is unexported: build the + // value again from the exported fields, which drops it (review of #893, finding 2). + return accountant.Body{Start: built.Start, Pos: built.Pos} +} + +// withoutCacheMarks is one element's JSON without any cache_control: the element's own, the ones on its content +// blocks, and the ones nested deeper, such as a mark inside a tool_result block's own content array (issue #917). +// An element with none is returned as it is, byte for byte, so the common case costs no allocation. The result is +// only hashed. It is never sent anywhere. +func oldWithoutCacheMarks(raw string) []byte { + if !strings.Contains(raw, "cache_control") { + return []byte(raw) + } + out := raw + // Each delete removes one mark and the walk starts again, so the paths stay right after the edit. A body has + // at most four marks on the wire, so the loop is short. + for n := 0; n < 64; n++ { + path := findCacheMark(gjson.Parse(out), "") + if path == "" { + break + } + next, err := sjson.Delete(out, path) + if err != nil { + break + } + out = next + } + return []byte(out) +} + +// accBreakpoints lists the cache_control marks, as positions in the same space as accBodies, one per +// position. A mark inside a message's content blocks belongs to that message, so an element mark +// and a block mark (or two block marks) can fall on one position: it is listed once, with the +// longer TTL (1h over 5m), because the provider keeps the entry for the longest TTL asked for at +// it. ttl defaults to "5m". +func oldAccBreakpoints(raw []byte) []accountant.Breakpoint { + if !gjson.GetBytes(raw, "messages").IsArray() { + return nil + } + var out []accountant.Breakpoint + at := map[accountant.Position]int{} // position -> index in out + add := func(p accountant.Position, ttl string) { + if i, ok := at[p]; ok { + if ttl == "1h" { + out[i].TTL = "1h" + } + return + } + at[p] = len(out) + out = append(out, accountant.Breakpoint{Pos: p, TTL: ttl}) + } + ttl := func(cc gjson.Result) string { + if cc.Get("ttl").String() == "1h" { + return "1h" + } + return "5m" + } + scan := func(sec accountant.Section, arr gjson.Result) { + i := 0 + arr.ForEach(func(_, el gjson.Result) bool { + if cc := el.Get("cache_control"); cc.Exists() { + add(accountant.Position{Section: sec, Index: i}, ttl(cc)) + } + if content := el.Get("content"); content.IsArray() { + content.ForEach(func(_, blk gjson.Result) bool { + if cc := blk.Get("cache_control"); cc.Exists() { + add(accountant.Position{Section: sec, Index: i}, ttl(cc)) + } + return true + }) + } + i++ + return true + }) + } + scan(accountant.SectionTools, gjson.GetBytes(raw, "tools")) + if s := gjson.GetBytes(raw, "system"); s.IsArray() { + scan(accountant.SectionSystem, s) + } + scan(accountant.SectionMessages, gjson.GetBytes(raw, "messages")) + return out +} diff --git a/proxy/accountant_savings.go b/proxy/accountant_savings.go index 6786d53e..58e191ad 100644 --- a/proxy/accountant_savings.go +++ b/proxy/accountant_savings.go @@ -44,7 +44,7 @@ func newSavingsTracker() *savingsTracker { // begin opens the request record with the client body, before any component ran. The Trackers bound // their own memory (accountant design §6.1, Memory), so there is nothing to sweep here. func (s *savingsTracker) begin(meta accountant.RequestMeta, client []byte) *accountant.Request { - body, bps := accBodies(client), accBreakpoints(client) + body, bps := accParse(client) req := s.trackers.BeginRequest(meta, body, bps) s.clients.Store(req, clientHash{raw: client, body: body, bps: bps}) return req @@ -63,7 +63,7 @@ func (s *savingsTracker) end(req *accountant.Request, th accountant.Thread, forw if v, ok := s.clients.LoadAndDelete(req); ok && bytes.Equal(v.(clientHash).raw, forwarded) { fwd, bps = v.(clientHash).body, v.(clientHash).bps } else { - fwd, bps = accBodies(forwarded), accBreakpoints(forwarded) + fwd, bps = accParse(forwarded) } facts.SharedWarm = s.siblings.sharedWarm(req.Meta.Session, th.ID, req.Meta.At, facts.TTL, fwd, bps) req.EndRequest(th, fwd, bps, facts) diff --git a/proxy/accountant_savings_body.go b/proxy/accountant_savings_body.go index dc87dd2b..3e4a8c5f 100644 --- a/proxy/accountant_savings_body.go +++ b/proxy/accountant_savings_body.go @@ -22,22 +22,63 @@ import ( // although the provider read it (accountant#21). The size is the wire size, with the marks, because the // library checks the operations' bytes against the bytes that really went upstream. func accBodies(raw []byte) accountant.Body { - if !gjson.GetBytes(raw, "messages").IsArray() { - return accountant.NewBody(nil, nil, nil) + body, _ := accParse(raw) + return body +} + +// accSections are the three top-level values accParse reads, found in one pass over the body. Like gjson.Get, the +// first key of a name wins. +type accSections struct { + tools, system, messages gjson.Result +} + +func accScan(raw []byte) accSections { + var sec accSections + var seen [3]bool + root := gjson.ParseBytes(raw) + if !root.IsObject() { + return sec + } + root.ForEach(func(k, v gjson.Result) bool { + var i int + var dst *gjson.Result + switch k.String() { + case "tools": + i, dst = 0, &sec.tools + case "system": + i, dst = 1, &sec.system + case "messages": + i, dst = 2, &sec.messages + default: + return true + } + if !seen[i] { + seen[i], *dst = true, v + } + return !(seen[0] && seen[1] && seen[2]) + }) + return sec +} + +// accParse is accBodies and accBreakpoints on one parse of the body. +func accParse(raw []byte) (accountant.Body, []accountant.Breakpoint) { + sec := accScan(raw) + if !sec.messages.IsArray() { + return accountant.NewBody(nil, nil, nil), nil } var tools, system, msgs [][]byte var sizes []int // the wire size of each element, in hash order add := func(dst *[][]byte, v gjson.Result) { - *dst = append(*dst, withoutCacheMarks(v.Raw)) + *dst = append(*dst, withoutCacheMarks(raw, v)) sizes = append(sizes, len(v.Raw)) } - gjson.GetBytes(raw, "tools").ForEach(func(_, v gjson.Result) bool { add(&tools, v); return true }) - if s := gjson.GetBytes(raw, "system"); s.IsArray() { - s.ForEach(func(_, v gjson.Result) bool { add(&system, v); return true }) - } else if s.Exists() { - add(&system, s) + sec.tools.ForEach(func(_, v gjson.Result) bool { add(&tools, v); return true }) + if sec.system.IsArray() { + sec.system.ForEach(func(_, v gjson.Result) bool { add(&system, v); return true }) + } else if sec.system.Exists() { + add(&system, sec.system) } - gjson.GetBytes(raw, "messages").ForEach(func(_, v gjson.Result) bool { add(&msgs, v); return true }) + sec.messages.ForEach(func(_, v gjson.Result) bool { add(&msgs, v); return true }) built := accountant.NewBody(tools, system, msgs) for i := range built.Pos { built.Pos[i].Bytes = int32(sizes[i]) @@ -47,15 +88,20 @@ func accBodies(raw []byte) accountant.Body { // (the token ratio, the span of a splice, sawOriginal) is the wire size, as the operations' sizes are. A body // decoded from a snapshot has no table either, so a restart changes nothing. The table is unexported: build the // value again from the exported fields, which drops it (review of #893, finding 2). - return accountant.Body{Start: built.Start, Pos: built.Pos} + return accountant.Body{Start: built.Start, Pos: built.Pos}, accBreakpointsOf(sec) } // withoutCacheMarks is one element's JSON without any cache_control: the element's own, the ones on its content // blocks, and the ones nested deeper, such as a mark inside a tool_result block's own content array (issue #917). // An element with none is returned as it is, byte for byte, so the common case costs no allocation. The result is // only hashed. It is never sent anywhere. -func withoutCacheMarks(raw string) []byte { +func withoutCacheMarks(body []byte, v gjson.Result) []byte { + raw := v.Raw if !strings.Contains(raw, "cache_control") { + // Slice the original bytes when Index points at this very element, else copy. The result is only read. + if i := v.Index; i > 0 && i+len(raw) <= len(body) && string(body[i:i+len(raw)]) == raw { + return body[i : i+len(raw) : i+len(raw)] + } return []byte(raw) } out := raw @@ -121,7 +167,12 @@ func findCacheMark(v gjson.Result, prefix string) string { // longer TTL (1h over 5m), because the provider keeps the entry for the longest TTL asked for at // it. ttl defaults to "5m". func accBreakpoints(raw []byte) []accountant.Breakpoint { - if !gjson.GetBytes(raw, "messages").IsArray() { + _, bps := accParse(raw) + return bps +} + +func accBreakpointsOf(sec accSections) []accountant.Breakpoint { + if !sec.messages.IsArray() { return nil } var out []accountant.Breakpoint @@ -142,15 +193,37 @@ func accBreakpoints(raw []byte) []accountant.Breakpoint { } return "5m" } + // mark finds the first cache_control and the first content of an object in one pass, as Get does for each key. + mark := func(el gjson.Result) (cc, content gjson.Result) { + if !el.IsObject() { + return + } + var gotCC, gotContent bool + el.ForEach(func(k, v gjson.Result) bool { + switch k.String() { + case "cache_control": + if !gotCC { + gotCC, cc = true, v + } + case "content": + if !gotContent { + gotContent, content = true, v + } + } + return !(gotCC && gotContent) + }) + return + } scan := func(sec accountant.Section, arr gjson.Result) { i := 0 arr.ForEach(func(_, el gjson.Result) bool { - if cc := el.Get("cache_control"); cc.Exists() { + cc, content := mark(el) + if cc.Exists() { add(accountant.Position{Section: sec, Index: i}, ttl(cc)) } - if content := el.Get("content"); content.IsArray() { + if content.IsArray() { content.ForEach(func(_, blk gjson.Result) bool { - if cc := blk.Get("cache_control"); cc.Exists() { + if cc, _ := mark(blk); cc.Exists() { add(accountant.Position{Section: sec, Index: i}, ttl(cc)) } return true @@ -160,10 +233,10 @@ func accBreakpoints(raw []byte) []accountant.Breakpoint { return true }) } - scan(accountant.SectionTools, gjson.GetBytes(raw, "tools")) - if s := gjson.GetBytes(raw, "system"); s.IsArray() { - scan(accountant.SectionSystem, s) + scan(accountant.SectionTools, sec.tools) + if sec.system.IsArray() { + scan(accountant.SectionSystem, sec.system) } - scan(accountant.SectionMessages, gjson.GetBytes(raw, "messages")) + scan(accountant.SectionMessages, sec.messages) return out } From 52c68de07cb45a819a90ba8aa4096ab3e9493965 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sun, 8 Nov 2026 07:43:25 +0200 Subject: [PATCH 52/58] build: Pin the accountant at the NewBody media pre-filter (accountant#51) NewBody skips the JSON decode for positions that cannot hold media. On the 600 KB bench body the accountant's begin/end costs 2.8 ms and 87 allocations, from 4.8 ms and 4,905. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index cfd8c842..b6cfb87e 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.26.4 require github.com/maximhq/bifrost/core v1.7.0 // accountant prices context operations. This module's tests import it and its shared test cases. -require github.com/rossoctl/accountant v0.1.1-0.20261107153853-41d7cc522cc2 +require github.com/rossoctl/accountant v0.1.1-0.20261108054032-8e160764bb71 require ( github.com/alexaandru/go-sitter-forest/c v1.9.4 diff --git a/go.sum b/go.sum index ca82637e..77380d58 100644 --- a/go.sum +++ b/go.sum @@ -102,6 +102,8 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rossoctl/accountant v0.1.1-0.20261107153853-41d7cc522cc2 h1:pNKgg7fjZzIlMZZzRqTn6ijsdNlyhM4J+8ZblABV94Q= github.com/rossoctl/accountant v0.1.1-0.20261107153853-41d7cc522cc2/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= +github.com/rossoctl/accountant v0.1.1-0.20261108054032-8e160764bb71 h1:fKOviX7sAMIbtgzyKiWPZstlUr3HU2ZuqAl3F57epnE= +github.com/rossoctl/accountant v0.1.1-0.20261108054032-8e160764bb71/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From d192f18f289809f1d72fea968fe695bbd76c3862 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sun, 8 Nov 2026 10:27:00 +0200 Subject: [PATCH 53/58] docs(accountant): Add the causes found on main to the cg root-cause report Run of the restored TestCasesAgreeWithCurrentLogic against main (f43d9fa5) and accountant 8e16076: 114 cases, 58 differ. All nine earlier causes still hold. Adds causes 10 to 14, marked NEW: the mark premium, the per-entry TTL tier, the keep-alive rescue guard, the missing size and range checks, and six cases not yet traced. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- docs/accountant/cg-differences-root-causes.md | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/docs/accountant/cg-differences-root-causes.md b/docs/accountant/cg-differences-root-causes.md index d9d404e1..eda42f8f 100644 --- a/docs/accountant/cg-differences-root-causes.md +++ b/docs/accountant/cg-differences-root-causes.md @@ -27,6 +27,12 @@ from the same run (63 cases, 27 differ). | 6 | Credits booked outside the per-component figure | 4 | 27,450 | Adapter, not evidence | | 7 | No check for an unreported edit | 1 | 1,500 | Missing check | | 8 | The unique-versus-replay split has no session or thread | 3 | 153,525 | cg defect or open question | +| 9 | No sibling-thread input to the keep-alive rescue | 3 | see Cause 9 | Missing input | +| 10 | **NEW** No charge for a cache mark the host adds | 13 | see Cause 10 | cg defect | +| 11 | **NEW** The TTL tier follows the current request, not the stored entry | 3 | 13,500 | cg defect | +| 12 | **NEW** The keep-alive rescue needs the new turn to write less than the ping read | 2 | 41,400 | cg defect | +| 13 | **NEW** No size or range check on a reported operation | 4 | see Cause 13 | Missing check | +| 14 | **NEW** Cases not yet traced | 6 | see Cause 14 | Unknown | ## Cause 1: no cache-break charge @@ -192,3 +198,138 @@ They are listed in `dash/testdata/accountant/known-differences.json` as `notCove something the model does not. Whether cg should learn about sibling threads is open. The proxy path does not report `SharedWarm` either, so `TestProxyPathMatchesTheAccountantCases` passes on these cases only because it plays the case's own facts. + +## New causes found on `main` + +**Added 2026-11-08.** These causes come from a run of `TestCasesAgreeWithCurrentLogic` against context-guru +`main` (`f43d9fa5`) and accountant `8e16076` (114 cases, 58 differ). The harness is the one from +commit `4125331f~1`. `main` reverted the accountant integration in #903, so it has no such test now. +All 9 causes above still hold on `main`. None is resolved. The 28 cases below are not in +`known-differences.json`, so the old harness reports them as plain failures. Gaps are the first +differing line of each case, in µ$. + +## Cause 10 (NEW): no charge for a cache mark the host adds + +**What the spec does.** If the host adds a cache breakpoint, the provider writes the tokens before +it at the write rate. The client body would have billed them as input. The spec charges the host +tokens × (write − read tier) as a cost. It books it with no ChangeID, because the host reports no +operation for the mark (`markPremium`). + +**What cg does.** cg books only what a component removes. A mark that adds a write premium has no +component, so cg books 0. + +**Example, `markpremium/added-tail/hot/5m`.** The client has one breakpoint at `messages[0]`. The host +adds one at the last message. The provider writes 6,000 tokens at the 5m rate. The client body would +bill them as input. The mark costs 6,000 × ($3.75 − $3.00) = 4,500 µ$. The spec books −4,500. cg +books 0. + +**Cases (13, all `markpremium/*`).** + +| Case | Gap (µ$) | +|---|---:| +| `added-tail/first-request/5m` | 4,500 | +| `added-tail/hot/5m` | 4,500 | +| `added-tail/hot/1h` | 18,000 | +| `adding-op-before-head/first-request/1h` | 5,250 | +| `headttl-after-client-mark/first-request/1h` | 9,000 | +| `host-1h-mark-before-client-last/first-request/1h` | 6,750 | +| `host-1h-mark-on-evicted-entry/hot/1h` | 2,250 | +| `inside-previous-read/hot/5m` | 4,500 | +| `mixed-ttl/1h-inside-5m-last/5m` | 4,500 | +| `mixed-ttl/5m-inside-1h-last/5m` | 6,750 | +| `with-headttl-and-adding-op/first-request/1h` | 11,250 | +| `with-headttl/first-request/1h` | 8,250 | +| `with-mark-text/hot/5m` | 4,515 | + +**Relation to other causes.** This is related to Cause 1, which also books only removed tokens. It is a +separate mechanism: Cause 1 is a missing break charge on an edit, and this is a missing charge for a +mark. Several of these cases use the 1h head TTL, so the cg side overlaps with Cause 6. The adapter +does not feed the head-TTL credit. + +**Unverified.** It is not known whether the live proxy adds marks of this kind. If it does not, these +cases are theoretical for cg. + +## Cause 11 (NEW): the TTL tier follows the current request, not the stored entry + +**What the spec does.** An entry keeps the TTL it was written with. If a 1h entry is read by a request +that asks for 5m, the entry is alive, and the replayed tokens are reads. The case text says the older +library used the request's TTL, said "cold", and booked a 5m write. That was 12.5 times too much. + +**What cg does.** `AttributeCache` and `keepaliveCreditFloorMs` take the TTL the returning request +asked for (`CacheTTL`). cg has no per-entry TTL. + +**Cases (3).** In each, the spec books 12,000 µ$ and cg books 7,500 µ$. The gap is 4,500 µ$. + +- `ttl/1h-entry-asked-at-5m/hot-after-10-minutes` +- `ttl/5m-entry-asked-at-1h/cold-after-30-minutes` +- `ttl/mixed-1h-head-5m-tail/read-after-10-minutes` + +**Unverified.** The cg side was read from the figures and the case text. The code path that picks the +TTL was not traced. Whether a real client changes its TTL tier inside one session is not known. + +## Cause 12 (NEW): the keep-alive rescue needs the new turn to write less than the ping read + +**What cg does.** `keepaliveSavedUSD` returns 0 when `CacheWrite >= CacheRead`. + +**What the spec does.** The rescue is the smaller of what the ping read and what the request read +(design §4.4, condition 4). It does not matter how much the new turn wrote. + +**Example, `keepalive/ping-rescue-large-turn/5m`.** Turn 0 caches 6,000 tokens. A ping at minute 4 +reads all 6,000. The request at minute 8 reads 6,000 and writes 20,000 new tokens. Without the ping +the entry would have expired at minute 5. The rescue is 6,000 × ($3.75 − $0.30) = 20,700 µ$. cg books 0, +because 20,000 ≥ 6,000. + +**Cases (2).** `keepalive/ping-rescue-large-turn/5m` and `keepalive/ping-rescue-write-equals-read/5m`. +The gap is 20,700 µ$ in each. + +**Relation to Cause 5.** The two cases in Cause 5 still agree. This is a different branch of the same +function. The write-versus-read guard is the cause here, not the credit floor. + +## Cause 13 (NEW): no size or range check on a reported operation + +**What the spec does.** The spec bounds every report (check 13). A report with a size above the bound +is not priced and the check `size_range` fails. Other bad input is handled the same way, such as a +negative cost or negative output tokens. A failed check does not stop the pricing of the honest +operations on the same request. + +**What cg does.** cg has no such checks (see Cause 7 for the related missing check). Two things follow: + +1. cg books 0 where the spec prices a negative net. These cases hold an insertion by one operator and + a cut by another. +2. cg prices a huge input as if it were true. + +**Cases (4).** + +| Case | Spec | cg | Gap (µ$) | +|---|---:|---:|---:| +| `check/honest-chain-beside-malformed-rereport/priced` | −93,750 | 0 | 93,750 | +| `check/malformed-rereport-keeps-insertion/priced` | −93,750 | 0 | 93,750 | +| `check/size-after-inserting-operator/priced` | −18,750 | 0 | 18,750 | +| `check/size-above-client-body/keeps-active-change` | 1,500 | 600,000,001,500 | 600,000,000,000 | + +**The last case is the worst result of the run.** A report of 2e12 tokens makes cg book +600,000,001,500 µ$ for one change. The spec does not price that report. Even with a small fix, an +unbounded `SizeBefore` must not become a credit. + +**Unverified.** It is not known whether the live pipeline can send a size this large. The adapter +passes the case's value as given. + +## Cause 14 (NEW): cases not yet traced + +These six cases differ and are not in any cause above. The reason in each line comes from the case +text and the figures. The cg code was not read. + +| Case | Spec | cg | Gap (µ$) | Probable reason | +|---|---:|---:|---:|---| +| `media/big-image-host-tokens-stay-exact/5m` | 3,000 | 15,000 | 12,000 | cg prices the image at its own estimate. The spec uses the host's token counts as given. | +| `media/big-image-hot-request-op-over-media/5m` | 0 | 15,000 | 15,000 | Same. | +| `media/big-image-small-conversation/bill-below-media-price/5m` | 1,313 | 6,562 | 5,249 | Same. The spec cuts the media to the bill. | +| `extent/no-breakpoint-long-body/hot/5m` | 900 | 1,125 | 225 | Rate by position, like Cause 2. | +| `rewrite/no-breakpoint/cut-inside-cache/5m` | −49,500 | 9,000 | 58,500 | The sign error of Cause 1, with no breakpoint. | +| `split/tail-unchanged-hash-differs-system/no-effect` | 0 | 10,350 | 10,350 | cg's cachesplit credits a split that moved no content. | + +## Updated priority + +Cause 13 comes first. It is the only one where a wrong input produces an unbounded figure. Then follow +the steps in the Priority section above. Add Cause 10 to the decisions for Cause 1, because both book +only removed tokens. Cause 12 is a small change in one function. From 5e47f96e2ff2501160f100e7405a86268b34af1a Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sun, 8 Nov 2026 10:38:41 +0200 Subject: [PATCH 54/58] docs(prometheus): Add the metrics reporting spec Spec for pushing inference and accountant savings metrics from every proxy to one central Prometheus: requirements, metric list, labels, cardinality budget, dashboards, analyses and test plan. Co-Authored-By: Claude Sonnet 5.5 --- docs/prometheus/accountant-hook.svg | 86 +++ docs/prometheus/discussion.md | 133 ++++ docs/prometheus/existing-data-flow.svg | 49 ++ .../prometheus/otel-prometheus-components.svg | 149 +++++ docs/prometheus/spec.md | 603 ++++++++++++++++++ 5 files changed, 1020 insertions(+) create mode 100644 docs/prometheus/accountant-hook.svg create mode 100644 docs/prometheus/discussion.md create mode 100644 docs/prometheus/existing-data-flow.svg create mode 100644 docs/prometheus/otel-prometheus-components.svg create mode 100644 docs/prometheus/spec.md diff --git a/docs/prometheus/accountant-hook.svg b/docs/prometheus/accountant-hook.svg new file mode 100644 index 00000000..620ccbc3 --- /dev/null +++ b/docs/prometheus/accountant-hook.svg @@ -0,0 +1,86 @@ + + + + + + + Accountant hook: from one request to metrics and the ledger + + + + Proxy (cortex or context-guru) + + + + Request handler + sends request to LLM, + gets response + usage + + + + accountant (standard library only) + + Compute(Usage) -> Result + pure, no side effects + + Result.Record(meta) -> Record + pure; adds IDs, model, totals + + interface Sink { Observe(Record) } + one method; the accountant only + declares it, never calls it + + + + 1. Usage + + Result + + 2. Record + + + + Multi(sinks...) + 3. proxy calls Observe + + + implements + + + + accountant/metrics (separate module, defined once) + metric names, units and labels + tokens by tier + money (tokens x Rates) + saved and added counters + failed-check counter + options: user label on/off, prefix + input: Meter + Rates from the proxy + uses the OTel API only + + + + Ledger sink (proxy's own) + stores the full Record with request, + thread, session and user IDs + + + + Proxy-only sink or metric (optional) + own prefix: cortex_ or contextguru_ + + + + + + same Record goes to every sink, so all numbers agree + + + 4. counters (low-cardinality labels) + + + metrics: OTel -> collector -> Prometheus + + + push (OTLP) + answers per-session, per-thread, per-request sums + diff --git a/docs/prometheus/discussion.md b/docs/prometheus/discussion.md new file mode 100644 index 00000000..8689d419 --- /dev/null +++ b/docs/prometheus/discussion.md @@ -0,0 +1,133 @@ +# Reporting inference and savings metrics with Prometheus + +This note summarizes a design discussion. It is not a final decision. Open questions are at the end. + +## 1. Goal + +1. Collect regular inference metrics (requests, latency, errors, tokens, cost). +2. Collect savings metrics reported by the `accountant` library. +3. Do this for many local proxy instances (Cortex proxy or context-guru proxy) that report to one central Prometheus. +4. Define the metrics once, so two sets do not need to be maintained. +5. Support roll-up from request, to thread, to session, to user. Report both tokens and money. + +## 2. Prometheus basics + +- Prometheus normally **pulls**. The app serves plain text at `/metrics`, and Prometheus fetches it on a schedule. +- Counters are cumulative totals. Use `rate()` or `increase()` in a query to get values per time interval. +- A **label** is a key and value on a metric, such as `model="claude-sonnet"`. Each different set of labels is a separate series. +- Each new label value makes a new series. Labels with many or unlimited values (user ID, session ID, request ID, prompt text) make Prometheus slow and costly. Use labels with few values: `model`, `status`, `endpoint`, `host`. +- Good metric types: Counter for totals that only grow, Histogram for latency and sizes, Gauge for values that go up and down. + +## 3. Collecting from many local instances + +Pull works only if the central Prometheus can reach each instance. Laptops behind NAT or a firewall usually cannot be reached. Options: + +| Option | How | Fit | +|---|---|---| +| Central scrape | Prometheus scrapes each instance | Servers with stable addresses | +| Local agent + remote write | A small agent on each machine scrapes `localhost` and sends out over HTTPS | Existing Prometheus-library code | +| OpenTelemetry (OTel) push | The app sends metrics to a central collector | Chosen option in this note | +| Pushgateway | Apps push to a gateway | Do not use. For short batch jobs only | + +### Chosen setup: OTel push + +- The proxy records metrics through the OTel library and **pushes** them to a central OTel Collector over HTTPS (outgoing connections only). +- The collector adds a machine label and a `proxy` label, then **pushes** to Prometheus with remote write. Prometheus must run with `--web.enable-remote-write-receiver`. Nothing calls `/metrics` on this path. +- Alternatively, the collector can expose its own `/metrics` and Prometheus can scrape it. +- The collector, Prometheus, Grafana and Alertmanager can run on one server for a small team. Expose only the collector to the local machines. +- Offline machines may lose data, because the library keeps no long backlog and Prometheus rejects very old samples. +- Adding a new metric changes only the proxy code. The collector, Prometheus and Grafana need no setup change. Check the final metric name in Prometheus, because it may be renamed (for example `_total` is added to counters). + +![Components: local machines, collector, Prometheus, Grafana](otel-prometheus-components.svg) + +## 4. Who owns the metric logic + +The proxy owns metrics reporting. The accountant returns data only. + +1. The accountant module imports only the standard library, and `boundary_test.go` enforces that. It must not import OTel or Prometheus. +2. `Compute` is deterministic and has no side effects. Keep it that way. +3. Two proxies share the accountant. The metric definitions must exist once, not twice. +4. The metrics may differ between proxies, so the shared part must not fix the full metric list. + +### Layers + +``` +accountant -> Result (tokens saved per change, per tier, checks); no OTel +accountant/metrics -> separate Go module; defines common metrics once; uses the OTel API only +proxy -> passes Meter and Rates, picks options, adds its own metrics +``` + +### Share and keep separate + +| Share (accountant or the metrics module) | Keep in each proxy | +|---|---| +| Roll-ups of `Result` (totals, per operation kind, failed checks) | Which extra metrics exist | +| Tier and operation-kind constants | Prefix for proxy-only metrics (`cortex_`, `contextguru_`) | +| Common metric names, units and labels | Rates used to convert tokens to money | +| | OTel setup, endpoint, token, interval | + +### Keep the data comparable + +1. Use the same names and labels for the metrics both proxies have. +2. The collector adds `proxy="cortex"` or `proxy="context-guru"` to everything. +3. Write the convention in one short file in the accountant `docs/`. + +## 5. The accountant hook + +The accountant declares a small interface. The proxy calls it after each finished request. The accountant never calls it. + +```go +type Sink interface{ Observe(Meta, Usage, Result) } +``` + +- `Compute` stays pure. The proxy calls `sink.Observe(...)` itself, after the response returns. +- The proxy can pass the data to several sinks with a `Multi(sinks...)` helper: shared metrics, the proxy's ledger, and optional proxy-only metrics. +- Because the sink gets the same data, all numbers agree. + +### Record, or Result plus Meta? + +A separate `Record` type is not required. Passing `Meta`, `Usage` and `Result` works and has one type fewer. The risk is that each sink sums the per-change numbers itself, and the copies drift. Avoid this with pure helper methods on `Result`: + +```go +res.Totals() // saved and added, per tier +res.ByKind() // the same, per rewrite / spend / setting +res.Failed() // names of failed checks +``` + +Recommendation: use `Observe(Meta, Usage, Result)` plus the helpers. Add a `Record` type later only if the ledger needs a frozen on-disk format. Confirm in `docs/design.md` whether `Result` already carries the meta and usage. If it does, the call can be `Observe(Result)`. + +![Accountant hook: from one request to metrics and the ledger](accountant-hook.svg) + +The diagram shows a `Record` for clarity. Read it as the data passed to `Observe`. + +## 6. Tokens and money + +1. The accountant knows tokens only. The host turns tokens into money with its own rates. +2. The shared metrics module takes `Rates` and records both tokens and money, by tier. +3. Compute money when the request happens, with the rate of that moment, and store it. Do not recompute old money with new rates. +4. Tokens by tier: `input`, `cache_write_5m`, `cache_write_1h`, `cache_read`. +5. A change can break the cache and cost more than it saves. Savings can be negative. Use two counters, `saved` and `added`. Net savings is `saved - added` in a query. +6. Label by operation kind (`rewrite`, `spend`, `setting`), not by change ID. Change IDs are unbounded. +7. Count failed checks from `Result.Checks`. + +## 7. Roll-up: request, thread, session, user + +Prometheus labels cannot hold request, thread or session IDs. Split the job: + +| Level | Where | How | +|---|---|---| +| Fleet, proxy, model, operation kind, tier | Prometheus | Labels with few values | +| User | Prometheus if there are few users (tens). Otherwise the ledger | Label `user` | +| Session, thread, request | Ledger (event store) | Store each record in full with all IDs | + +- All values are sums (tokens and money), so they roll up at every level. +- Prometheus answers "how much did we save per model per hour". The ledger answers "how much did this session save". +- For a jump from a chart to one request, use OTel traces and put the request and session IDs on the trace, not on the metric. + +## 8. Open questions + +1. Where does the shared metrics module live: as a separate module in the accountant repo, or in its own repo? +2. How many users are expected? This decides whether `user` is a metric label. +3. Do both proxies already store full per-request records? If not, the ledger sink is the larger piece of work. +4. Does `Result` already carry the meta and usage? +5. Is the first version tokens and money, or tokens only? diff --git a/docs/prometheus/existing-data-flow.svg b/docs/prometheus/existing-data-flow.svg new file mode 100644 index 00000000..6544b45e --- /dev/null +++ b/docs/prometheus/existing-data-flow.svg @@ -0,0 +1,49 @@ + + + + + + + What exists today: one proxy, two readers + + One machine (no data leaves it) + + + Request + + Proxy + + + + SQL store + SQLite, one row per request + + In-memory counters + reset at restart + + + + + Dashboard UI + reads /api/* + + + + /metrics + cg_tenant_* and cg_* + + + + + Local Prometheus + scrapes every 15 s + + pull + + + Grafana + 4 dashboards, 3 alert rules + + + The UI and /metrics do not read each other. Both read the store or process memory. + diff --git a/docs/prometheus/otel-prometheus-components.svg b/docs/prometheus/otel-prometheus-components.svg new file mode 100644 index 00000000..de896834 --- /dev/null +++ b/docs/prometheus/otel-prometheus-components.svg @@ -0,0 +1,149 @@ + + + + + + + + Inference metrics with OpenTelemetry (OTel) and Prometheus + + + + + Local machine 1 + + User + + Harness + + Proxy + + + + Proxy + dashboard + + + SQLite + small local DB + + writes + + reads + + views + + + + + + Local machine 2 + + User + + Harness + + Proxy + + + + Proxy + dashboard + + + SQLite + small local DB + + writes + + reads + + views + + + + + + Local machine N + + User + + Harness + + Proxy + + + + Proxy + dashboard + + + SQLite + small local DB + + writes + + reads + + views + + + + + LLM provider + hosted or self-hosted + + 1. Each proxy sends inference requests (all machines do this) + + + + + + 2. Proxy pushes metrics + (OTLP over HTTPS, + every ~30 s, + outgoing only) + + + + + Central server (these four can run on one machine) + + + OTel Collector + checks the token + adds a machine label + batches the data + + + Prometheus + stores numbers with + timestamps + answers queries + + + 3. Collector pushes + (remote write; no /metrics scrape) + + + Grafana + charts and dashboards + + 4. Ask for data + + + Alertmanager + sends alerts (optional) + + alert rules fire + + + Analysis user + + 5. Views + dashboards + + + Solid arrow = data flow. Dashed arrow = optional. Arrows from the machines point outward, so firewalls and NAT do not block them. + Purple arrow = the user opens the proxy dashboard. It reads the small local SQLite DB that the proxy writes. This data stays on the machine. + diff --git a/docs/prometheus/spec.md b/docs/prometheus/spec.md new file mode 100644 index 00000000..ee8a169e --- /dev/null +++ b/docs/prometheus/spec.md @@ -0,0 +1,603 @@ +# Metrics reporting: spec + +Status: draft. Issue: #886. Input: [discussion.md](discussion.md). + +This spec turns the discussion into requirements, a metric list, dashboards and analyses. Items marked **Open** need a decision before the work starts. + +## 1. Goal + +1. Report inference metrics and `accountant` savings metrics from every proxy to one central Prometheus. +2. Define the common metrics once. The Cortex proxy and the context-guru proxy share them. +3. Show money and tokens at every roll-up level that Prometheus can hold: fleet, proxy, machine, model, operation kind and tier. +4. Show whether context-guru earns its cost: gross saving, added cost, net saving. +5. Support many local proxies that report to one central metrics server. Managers use it to estimate the savings of the whole fleet. +6. Let each user see the details of their own traffic on their own proxy. +7. Support at least two proxies: the context-guru proxy (cg) and the Cortex proxy. + +Not goals: +- Per-session, per-thread and per-request figures. The dashboard store and the ledger hold these. +- Replacing the dashboard (`dash/`). Prometheus gives the fleet view. The dashboard gives the session view. +- Prompt text or any customer content in a metric or label. + +## 2. What exists today + +Three parts exist. They share one data flow: + +![What exists today: one proxy, two readers](existing-data-flow.svg) + +The dashboard UI and `/metrics` both read the store or the proxy's memory. Neither reads the other. + +### 2.1 The dashboard UI (`dash/ui`) + +A single-page app embedded in the binary. It is **off by default**. `--dashboard` turns it on. It needs no Prometheus, no CDN and no network. + +What it covers: +- **Overview.** Dollars saved out of what the traffic would have cost, with percent saved and tokens removed. Cards for cache, preset and tool filter. Five diagnostic plots and a live request table. +- **Savings.** Cache strategy (keep-alive), tool inventory, cleanup advice, usage over time. +- **Traffic.** Sessions and requests. A request view shows a diff of what was rewritten on the wire. +- **Admin.** Config, cache strategies, tenants, setup, settings, archive, feedback (some only in a hosted service). +- **Advanced.** Per-component economics, the KV-cache view, strategy campaigns. + +Limits: +1. **One proxy only.** Each proxy has its own store. No view spans machines. +2. **Needs the store.** It shows nothing without `--dashboard`. +3. **Access is by network location.** Aggregates are open to anyone who can reach the port. Content and prompt text need loopback or a trusted CIDR. There is no fleet-wide sign-in outside the hosted service. +4. **No alerts.** It shows numbers. It does not page anyone. +5. **No export format for other tools.** Other tools read `/api/*` or `/metrics`. + +### 2.2 The SQL store (`dash/store.go`, `dash/schema.go`) + +SQLite in WAL mode (pure Go). One row per request in `requests`. Other tables hold per-component figures (`request_components`), request content (`request_content`), tool declarations and uses, session state, request lifecycle, extraction calls, tenant spend and the archive index. + +What it covers: +- **Every request, in full detail.** Tokens by tier, cost, baseline cost, cache fields, cache miss cause, thread and session IDs, keep-alive fields, settings. +- **Per-component savings.** `saved_usd`, the split `saved_first_usd` and `saved_replay_usd`, and `priced_by`. The `accountant` library produces these figures. +- **State that survives a restart.** Session state, checkpoints, and the data the cache tracker needs. +- **Cold storage.** Old sessions move to an archive. An index and the monthly tenant spend are kept. + +Limits: +1. **Local to one proxy.** There is no sync to a central place. +2. **Retention is bounded** by age (`--dashboard-retention`) and by size (`--dashboard-max-bytes`). A burst can evict old rows. +3. **Derived data can be dropped.** When the schema version changes, the metrics tables restart. Only the archive index and tenant spend carry over. +4. **Not built for fleet queries.** Cross-machine roll-up needs a second system. +5. **May hold customer text.** The content tables are a deliberate exception to "metadata first". They are gated by storage level and by access rules. +6. **Old rows keep old figures.** Rows written before the `accountant` switch keep the legacy figure (`priced_by` empty). A window across the switch mixes two definitions. + +### 2.3 The `/metrics` endpoint (`proxy/promexport.go`) + +Prometheus text, served by the proxy. It is hand-written and uses no client library. A local Prometheus scrapes it every 15 seconds. `deploy/grafana/` has the scrape config, 4 dashboards (`context-guru`, `host`, `slo`, `logs`) and 3 alert rules (`cg-service-down`, `cg-refusal-rate`, `cg-metrics-stale`). `install.sh grafana` installs it. + +Two families: + +| Family | Source | Scope | Behavior | +|---|---|---|---| +| `cg_*` | In-memory aggregator, the same data as `/stats` | All tenants together | Starts at 0 when the process starts. Real counters. Use `rate()`. | +| `cg_tenant_*` | The SQL store, cached about 60 s | One tenant | Month to date. Most are gauges, including four that end in `_total`. | + +What it covers: +- Requests, refusals, latency, added latency, time to first byte for streams. +- Tokens before and after compaction, billed tokens by tier. +- Cost, baseline cost, saved dollars, net saved dollars, the compaction model's own cost. +- Per-component runs, events, tokens removed and time spent. +- Expand and stash counters, cache-frozen headroom, extraction value. +- Exporter health: `cg_metrics_age_seconds`, render time, dropped capture events. +- Per-tenant cost, tokens, sessions and archive size. + +Limits: +1. **Pull only.** A central Prometheus must reach the proxy. A laptop behind NAT or a firewall cannot be reached. +2. **Two families do not agree.** `cg_*` is per process and resets at restart. `cg_tenant_*` is per tenant and month to date. A panel that mixes them is wrong. The HELP text says so. +3. **Month-to-date gauges.** `rate()` and `increase()` read a fall as a counter reset. No shipped panel uses them on these series. +4. **Savings are single figures.** `cg_saved_usd` and `cg_net_saved_usd` are gauges. There is no `saved` and `added` pair. There is no split by operation kind, by tier, or by cache effect (break, rescue, split, TTL). +5. **No keep-alive or cache-miss series by cause.** The data is in the store. It is not exported. +6. **Label set is small.** Labels in use: `tenant`, `tier` (billed tokens only), `component` (component series only), `reason`, `gate` and `event`. There is no `model` or `provider` label, so a cost or a saving cannot be split by model. +7. **Service-wide view.** It carries every tenant's spend. It is open on loopback only, or with `METRICS_TOKEN`. The deploy keeps Prometheus and Grafana on loopback. +8. **Not shared with the Cortex proxy.** The names live in this proxy's code. A second proxy would copy them. +9. **Grafana is a second view.** It reads only `/metrics`. It cannot show a session or a request. + +### 2.4 What the push work adds + +| Gap | Part that has it | What this spec adds | +|---|---|---| +| No fleet view | UI, store, `/metrics` | A push path to one central Prometheus (section 4). | +| Savings are one figure | `/metrics` | `saved` and `added` counters by kind, tier and component (5.2). | +| Not shared with Cortex | `/metrics` | One metrics module for both proxies (R27). | +| No cache-miss or keep-alive series | `/metrics` | New counters (5.3, 5.4). | + +What stays as it is: the UI and the store. The pull `/metrics` and the local Prometheus and Grafana are retired (R22). Prometheus cannot replace the store or the UI, because it cannot hold session, thread or request IDs as labels. Section 4 shows where each part sits. + +## 3. Requirements + +### 3.1 Needs: fleet, user and proxies + +Three needs drive these requirements. + +| Need | Who | Where it is served | +|---|---|---| +| Estimate the savings of the whole fleet | Manager | Central Prometheus and Grafana | +| See the details of my own traffic | User | Own proxy dashboard and local DB | +| Run on more than one proxy | Both | Shared metrics, one `proxy` label | + +**Many proxies, one central server (managers)** + +- R1. Many local proxies report to one central metrics server. A proxy needs no inbound connection, so a laptop behind NAT or a firewall can report. +- R2. A manager can estimate the savings of the fleet from the central server alone. The manager needs no access to any local proxy and sees no customer content. +- R3. The manager view shows gross saving, added cost and net saving. It can be split by `proxy`, `machine`, `model` and `preset`. It shows the share of the figure that is an estimate (R36, section 5.2). +- R4. The central server shows which machine each number comes from. If a machine stops sending data, the manager sees it within 15 minutes. +- R5. The design supports at least **50 machines** with the cardinality budget in section 5.8. **Open:** confirm the target (tens, hundreds or more). + +**Details for a user on the own proxy** + +- R6. A user can see the details of their own traffic on their own proxy: sessions, requests, tokens by tier, cost, saving per component and cache misses. This is the proxy dashboard and its local DB. +- R7. The details stay on the machine. Session, thread and request IDs, and any stored content, are never sent to the central server (R29, R31). +- R8. The proxy dashboard and its local DB work with no collector and no network. Turning the push on or off does not change what the user sees there. + +**More than one proxy** + +- R9. The design supports at least two proxies: the context-guru proxy (`cg`) and the Cortex proxy. +- R10. Both proxies report the common metrics (section 5) with the same names, units and labels. The `proxy` label tells them apart (R40). +- R11. Adding a third proxy changes only that proxy's code and a new `proxy` value. The collector, Prometheus and Grafana need no new setup. +- R12. Each proxy keeps its own extra metrics under its own prefix (R28). A dashboard panel on a common metric works for both proxies without a filter. +- R13. The manager view (R2, R3) shows the two proxies side by side, so managers can compare them. +- R14. The Cortex proxy gives its user the same details as R6. **Open:** confirm what local view and store the Cortex proxy has. If it has none, R6 applies to the cg proxy first and the Cortex proxy later. + +### 3.2 Transport (design rules) + +- R15. The proxy pushes metrics to an OpenTelemetry Collector over HTTPS. It makes outgoing connections only. +- R16. The push is **asynchronous**. The proxy records a metric in memory and returns at once. A separate background task sends the data to the collector in batches. The push never runs on the path of a request. It adds no time to a request or to the answer, and it never waits for the collector. +- R17. The collector adds the labels `machine` and `proxy`. It sends the data to Prometheus with remote write. +- R19. The product ships a default collector URL, so the push works from the first run. The URL is a setting. The user or the operator can change it in the config file, with a flag, or with an environment variable. The access token is a setting too (R44). A setting turns the push off. If the URL is empty, the proxy makes no connection and writes one log line at start. The PR chooses the names of the settings. Data leaves every machine from the first run, so the PR names this at the top: who it affects, what is sent, and how to turn it off (`RELEASING.md`). +- R20. If the collector is down or slow, the proxy keeps working. A failed push never delays or changes a request, and it never blocks a request. Fail open. +- R22. The pull path is retired. The `/metrics` endpoint and the local Prometheus and Grafana setup are removed. The product is new and has no legacy users, and the central server replaces them. The developer (David Boaz) decided this. The PR names the change at the top: who it affects, the old and new behavior, and that the user's own view is the proxy dashboard (R8). The dashboard and its local DB do not use `/metrics`, so they are not affected. + +### 3.3 Consistency across proxies + +- R23. The `accountant` library stays free of metrics code. It imports the standard library only, and it never sends or stores a metric. `boundary_test.go` keeps enforcing this. +- R24. Pricing a request stays pure. It has no side effects. Reporting a metric happens after the pricing, from the result. +- R25. The savings metrics are the same on every proxy. Each metric has the same name, unit and meaning, and every proxy computes it from the same `accountant` result. +- R26. A proxy needs no copy of the metric code. Adding the savings metrics to a proxy takes a small, fixed amount of code in that proxy. The totals, the split by kind, the failed checks and the metric definitions exist once. +- R27. The common metrics are defined once, in one place that both proxies import. A fix or a new common metric is made once and reaches every proxy. +- R28. Each proxy can add its own metrics under its own prefix (`cortex_`, `contextguru_`). The common set does not fix the full list. + +### 3.4 Data rules + +- R29. Metrics use counts, sizes, token numbers, prices, model names and status codes only. They use no stored customer data. +- R30. Allowed labels have few values: `proxy`, `machine`, `model`, `provider`, `status_class`, `stream`, `stage`, `direction`, `source`, `kind`, `tier`, `effect`, `cause`, `component`, `result`, `reason`, `check`, `outcome`, `version`, `preset`. +- R31. Forbidden labels: request ID, session ID, thread ID, change ID, user ID (unless R32), prompt text, tool name, file path. +- R32. `user` is a label only if the user count is about 50 or fewer. **Open.** Otherwise per-user figures come from the ledger. +- R33. Tokens are recorded as tokens. Money is recorded when the request happens, at the rate of that moment. Old money is never recomputed with new rates. +- R34. Tokens by tier use the values `input` (uncached input), `cache_read`, `cache_write_5m`, `cache_write_1h` and `output`. The two write tiers stay apart because they have different prices. +- R35. `saved` and `added` are two counters. A change can cost more than it saves. Net is `saved - added` in a query. The cost of the compaction model's own calls is part of `added`, booked under the component that made the call (R45). +- R36. A request the `accountant` cannot price adds nothing to `saved` or `added`. It adds 1 to a counter of unpriced requests, by reason. + +- R45. The cost of an LLM call that a component makes itself, such as compaction, is booked as `added` (cost and tokens) under that component's `component` label. There is no separate counter, so net is always `saved - added`. **Open:** confirm that the `accountant` sees these calls (open question 4). If it does not, the proxy must pass them to the `accountant` as a `spend` change. + +### 3.5 Convention + +- R39. One short file in the `accountant` `docs/` lists the common names, units and labels. +- R40. Both proxies use the same names and labels for the metrics they share. + +### 3.6 Documentation + +- R41. The PR adds a row to `docs/data-dependence.md`. The row says: metadata only, works at every storage level. + +### 3.7 Privacy and access + +- R43. `machine` is a pseudonymous install ID: a random value made at first start. It is not the host name, because a host name often holds a person's name. The operator keeps the map from install ID to person or team outside the metrics system. +- R44. The collector accepts a push only over HTTPS and only with a valid access token. Each install has its own token, so one token can be revoked. Read access to the central Prometheus and Grafana is limited to named managers and operators. **Open:** who issues the tokens, and who is on the access list. + +## 4. Proposed architecture + +There is one path: push. The pull path (section 2) is retired (R22). The picture shows the push path: each proxy sends metrics to a central collector, which sends them to Prometheus and Grafana. The push is asynchronous: it runs in the background, off the path of a request, so it adds no time to the proxy (R16). + +![Components: local machines, collector, Prometheus, Grafana](otel-prometheus-components.svg) + +| Component | Status | Role | +|---|---|---| +| proxy | exists | Forwards requests and calls the `accountant`. | +| `accountant` | exists | Returns `Result`. Imports no OTel (R23). | +| `Sink`, `Result` helpers | new | One place that turns `Result` into numbers (R25, R26, section 4.1). | +| `accountant/metrics` | new | Defines the common metrics once (R27). | +| OTel SDK in the proxy | new | Aggregates in memory and pushes in the background (R16). | +| dash store and `dash/ui` | exists | Per-session and per-request views. Do not use Prometheus. | +| OTel Collector | new | Receives pushes. Adds `machine` and `proxy`. | +| central Prometheus | new | Stores the pushed series. | +| central Grafana, Alertmanager | new | Fleet dashboards and alerts. | + +### 4.1 Design notes + +These notes say how the requirements can be met. They are proposals. The PR may change them if R23 to R28 still hold. + +| Requirement | Proposed way | +|---|---| +| R23, R24 | The `accountant` keeps its pure `Compute`. The proxy reports after the response returns. | +| R25, R26 | A small hook in the `accountant` (`Sink`, called as `Observe(Meta, Usage, Result)`) and helper methods on `Result` (`Totals()`, `ByKind()`, `Failed()`). A `Multi(...)` helper passes the same data to several consumers: the shared metrics, the local ledger and any proxy-only metrics. | +| R27 | A separate Go module `accountant/metrics`. It uses the OpenTelemetry API only. The proxy gives it a `Meter` and the price rates. | +| R28 | The proxy adds its own metrics with its own prefix. | + +Open choices: +- If `Result` already carries the meta and the usage, the hook can be `Observe(Result)`. +- The shared module can live in the `accountant` repo or in its own repo. + +## 5. Proposed metrics + +OTel metric names use dots and units. The Prometheus names below are what you see after the collector converts them. Counters get `_total`. A unit suffix such as `_usd` stays. Check the final names in Prometheus. + +Prefix `cg_` is for the common set. It matches the existing convention. Section 5.6 maps the retired pull series to the new ones. + +### 5.1 Inference (common) +Labels in this section: + +| Label | Values | Meaning | +|---|---|---| +| `model` | The model name the request was sent for, such as `claude-sonnet-5-5`. About 10 values per machine. | Which model served the request. If a component changed the model, this is the model after the change. | +| `provider` | The upstream provider, such as `anthropic`, `openai`, `bedrock`. A few values. | Who billed the request. | +| `status_class` | `2xx`, `3xx`, `4xx`, `5xx` | The HTTP status of the answer, in classes. A single code is not a label. | +| `stream` | `true`, `false` | Whether the answer was streamed. | +| `source` | `upstream`, `proxy`, `client` | Who caused an error. `upstream`: the provider failed. `proxy`: context-guru failed. `client`: the client hung up or sent a bad request. | +| `tier` | `input`, `cache_read`, `cache_write_5m`, `cache_write_1h`, `output` (R34) | The price class of the tokens. Each class has its own rate. | +| `stage` | Request bytes: `client`, `forwarded`. Response bytes: `upstream`, `delivered`. | Where in the proxy the body was measured. The two values of one counter show what the proxy changed. | + + +| Metric | Type | Unit | Labels | Meaning | +|---|---|---|---|---| +| `cg_inference_requests_total` | counter | 1 | `model`, `provider`, `status_class`, `stream` | Requests that reached the proxy. | +| `cg_inference_errors_total` | counter | 1 | `model`, `provider`, `status_class`, `source` | Requests that failed. `source` is `upstream`, `proxy` or `client`. | +| `cg_inference_request_bytes_total` | counter | By | `model`, `provider`, `stage` | Size of the request body. `stage` is `client` (as the client sent it to the proxy) or `forwarded` (as the proxy sent it upstream, after the components ran). Bytes removed is `client - forwarded`. Headers are not counted. | +| `cg_inference_response_bytes_total` | counter | By | `model`, `provider`, `stage` | Size of the response body. `stage` is `upstream` (as the provider sent it to the proxy) or `delivered` (as the proxy sent it to the client, after any rewrite). For a stream, the sum of the chunks. Headers are not counted. | +| `cg_inference_duration_seconds` | histogram | s | `model`, `provider`, `stream` | Total time, from request to last byte. | +| `cg_inference_ttfb_seconds` | histogram | s | `model`, `provider` | Time to first byte for streamed answers. | +| `cg_inference_added_seconds` | histogram | s | `model` | Time the proxy added on top of the upstream time. | +| `cg_inference_tokens_total` | counter | 1 | `model`, `provider`, `tier` | Provider-billed tokens. `tier` is from R34: read input (`cache_read`), write input (`cache_write_5m`, `cache_write_1h`), uncached `input`, and `output`. | +| `cg_inference_cost_usd_total` | counter | USD | `model`, `provider`, `tier` | Provider cost at request time. | +| `cg_inference_estimated_cost_usd_total` | counter | USD | `model`, `provider` | Part of the cost priced from a fallback rate. Sub-set of the line above. | + +The cache read ratio is not exported. Query it: `cache_read / (cache_read + cache_write_* + input)` on `cg_inference_tokens_total`. + +**Names (11).** Every counter is `cg__[_]_total`, with `usd`, `tokens` or `bytes` as the unit when the counter has one. Histograms end in `_seconds`. The collector must not add a second unit suffix (for example `_USD`). A test compares the names Prometheus shows with this table. + +### 5.2 Savings (from the `accountant`) +Labels in this section: + +| Label | Values | Meaning | +|---|---|---| +| `kind` | `rewrite`, `spend`, `setting` | The type of change, from the `accountant`'s `Operation.Kind`. `rewrite`: an edit to the prompt body, such as `mask` or `dedup`. `spend`: the component's own LLM call, such as compaction (R45). `setting`: a changed request parameter that the body does not show but that changes the cost or the cache, such as the model or the cache TTL. | +| `tier` | Same values as 5.1 (R34) | The price class of the tokens saved or added. | +| `component` | The operator name without the host prefix, such as `mask`, `dedup`, `headttl`. About 30 values. | Which operator made the change. One component can report more than one `kind`. | +| `effect` | `break`, `rescue`, `split`, `ttl_premium`, `ttl_rescue` | The cache effect of a change. `break`: a change moved the prefix and the cache missed. `rescue`: a change or a ping kept a cache that would have expired. `split`: a change split one cached prefix into two. `ttl_premium`: extra cost of a longer cache TTL. `ttl_rescue`: saving from a longer TTL. | +| `cause` | `model_changed`, `ended_change`, `none` | Why the effect happened. `model_changed`: the model changed. `ended_change`: a change that was active before stopped. `none`: no single cause. | +| `direction` | `saved`, `added` | Whether the cache effect lowered or raised the bill. | +| `model` | As in 5.1 | The model of the baseline cost. | +| `result` | `priced`, `unpriced`, `skipped` | What the `accountant` did with the request. | +| `reason` | For `unpriced`: `penalty_unknown`, `unknown_kind`, `operator_not_allowed` and others from the library. For `fallback`: the gate that declined. | Why the request was not priced, or why no saving was booked. | +| `check` | `sum`, `order`, `unique_change`, `spans`, `attribution`, `conservation`, `shared_warm` | The library check that failed. | + + +| Metric | Type | Unit | Labels | Meaning | +|---|---|---|---|---| +| `cg_savings_saved_tokens_total` | counter | 1 | `kind`, `tier`, `component` | Tokens the provider did not bill, by tier. | +| `cg_savings_added_tokens_total` | counter | 1 | `kind`, `tier`, `component` | Tokens the provider billed because of a change. | +| `cg_savings_saved_usd_total` | counter | USD | `kind`, `tier`, `component` | `tokens_saved` priced at the tier rate at request time. | +| `cg_savings_added_usd_total` | counter | USD | `kind`, `tier`, `component` | `tokens_added` priced the same way. | +| `cg_savings_cache_effect_usd_total` | counter | USD | `effect`, `cause`, `direction` | Cache effects: `effect` is `break`, `rescue`, `split`, `ttl_premium`, `ttl_rescue`. `direction` is `saved` or `added`. `cause` is `model_changed`, `ended_change` or `none`. | +| `cg_savings_baseline_usd_total` | counter | USD | `model` | What the traffic would have cost with no change. | +| `cg_savings_requests_total` | counter | 1 | `result` | `result` is `priced`, `unpriced`, `skipped`. | +| `cg_savings_unpriced_total` | counter | 1 | `reason` | Why a request was not priced. Values come from the library: `penalty_unknown`, `unknown_kind`, `operator_not_allowed` and others. | +| `cg_savings_estimated_total` | counter | 1 | none | Requests whose figure is marked estimated. | +| `cg_savings_check_failures_total` | counter | 1 | `check` | Failed library checks: `sum`, `order`, `unique_change`, `spans`, `attribution`, `conservation`, `shared_warm`. | +| `cg_savings_fallback_total` | counter | 1 | `reason` | Requests that booked no saving because the dashboard gate declined. | + +**Saved tokens and added tokens.** Both counters compare the request as it was sent with the request as the client wrote it. They are two sides of one change, and neither one is net. +- `cg_savings_saved_tokens_total`: tokens the provider billed less, because of a change. Example: `dedup` removes 10,000 tokens from the prompt. Saved: 10,000 at tier `input`, or at tier `cache_read` if those tokens would have been read from the cache. +- `cg_savings_added_tokens_total`: tokens the provider billed more, because of a change. Examples: a rewrite changes the prefix, the cache breaks, and 50,000 tokens are billed as `cache_write_5m` where they would have been `cache_read`. A keep-alive ping bills its own tokens. A compaction call bills the compaction model's tokens (R45). +- One change can add to both. A rewrite can save 5,000 input tokens and also cause a break that adds 50,000 write tokens. Do not net them inside the change. Each counter takes its own part, so the reports can show a component that saves tokens but loses money. +- Both counters are never negative. They only rise. +- Tokens at different tiers have different prices, so `saved_tokens - added_tokens` is not a money figure. Use the `usd` counters for money. Each `usd` counter is its tokens priced at the tier rate at request time (R33). +- The tokens counters answer "how much did the traffic shrink or grow?". The `usd` counters answer "did it pay?". + +**No double counting (5).** `cg_savings_saved_usd_total` and `cg_savings_added_usd_total` are the only counters that add up to the saving. They already include the cache effects. `cg_savings_cache_effect_usd_total` is a breakdown of the same money by effect. A query must never add it to `saved` or `added`. A test checks that, for each priced request, the sum of the cache effects is not larger than `saved + added`. **Open:** confirm with the `accountant` that cache effects are part of the component figures. If they are not, give them `kind="cache"` and drop the separate counter. + +The `component` count is bounded by the shipped list. **Open:** confirm the count is small enough to keep as a label. + +### 5.3 Keep-alive +Labels in this section: + +| Label | Values | Meaning | +|---|---|---| +| `model` | As in 5.1 | The model whose cache the ping kept warm. | +| `outcome` | `ok`, `failed`, `timeout`, `skipped` | `ok`: the ping succeeded. `failed`: the provider returned an error. `timeout`: no answer in time. `skipped`: the proxy chose not to ping, for example because a sibling thread kept the cache warm. | + + +| Metric | Type | Unit | Labels | Meaning | +|---|---|---|---|---| +| `cg_keepalive_pings_total` | counter | 1 | `model`, `outcome` | `outcome` is `ok`, `failed`, `timeout`, `skipped`. | +| `cg_keepalive_cost_usd_total` | counter | USD | `model` | What pings cost. | +| `cg_keepalive_rescue_usd_total` | counter | USD | `model` | Saving from rescues (`rescue` effect). | +| `cg_keepalive_shared_warm_tokens_total` | counter | 1 | `model` | Tokens not credited because a sibling thread kept them warm (#853). | + +Net keep-alive value is `rescue_usd - cost_usd`. + +### 5.4 Cache health + +| Metric | Type | Unit | Labels | Meaning | +|---|---|---|---|---| +| `cg_cache_misses_total` | counter | 1 | `model`, `cause` | Misses by cause from `cache_miss_cause`: `ttl_expiry`, `prefix_change`, `model_changed`, `compaction_after_rewrite`, `bypass_flip`, `cold_start`, `other`. | +| `cg_cache_miss_cost_usd_total` | counter | USD | `model`, `cause` | Extra cost of those misses. | + +### 5.5 Proxy health + +| Metric | Type | Unit | Labels | Meaning | +|---|---|---|---|---| +| `cg_proxy_info` | gauge | 1 | `version`, `preset` | Always 1. Shows build and config. | +| `cg_proxy_component_duration_seconds` | histogram | s | `component` | Time in each component. | +| `cg_proxy_component_failures_total` | counter | 1 | `component`, `reason` | A component error or panic that reverted. | +| `cg_proxy_fail_open_total` | counter | 1 | `reason` | Requests forwarded unchanged. | +| `cg_proxy_export_failures_total` | counter | 1 | `reason` | Failed pushes to the collector. The next push carries the totals, so a failure loses no count. | + +### 5.6 Retired pull series + +The pull series go away with `/metrics` (R22). This table shows what replaces each one, so that dashboards and alerts can be moved. + +| Retired pull series | Replaced by | +|---|---| +| `cg_requests_total` | `cg_inference_requests_total` | +| `cg_saved_usd` (gauge, month to date) | `cg_savings_saved_usd_total` (counter) | +| `cg_net_saved_usd` (gauge) | Query: `saved - added`. | +| `cg_cache_hit_ratio` | Query on the token counters. | +| `cg_tenant_*` (month to date) | Queries on the counters, over any time range. | +| `cg_metrics_age_seconds`, `cg_metrics_render_seconds` | None. They described the pull exporter. | + +### 5.7 Labels the collector adds + +| Label | Source | Values | +|---|---|---| +| `proxy` | collector config | `cortex`, `context-guru` | +| `machine` | resource attribute `service.instance.id`, the install ID (R43) | One per install. Bounded by the fleet size. | +| `preset` | resource attribute | A few values. A change of preset starts a new series, so a split by `preset` follows the change (8). | +| `service_version` | resource attribute | A few values. | + +`machine` has as many values as there are machines. This is acceptable for tens of machines. Do not use it for thousands. + +### 5.8 Cardinality budget + +Estimate series per machine. A histogram has 12 buckets, plus a sum and a count: 14 series per label set (11 bounds in section 12, plus `+Inf`). + +| Metric group | Label sets | Series | +|---|---|---| +| Savings counters (4) by `kind` x `tier` x `component` | 3 x 4 x 30 = 360 each, worst case. About 1 in 4 is used. | about 360 | +| Cache effects | 5 x 3 x 2 = 30 | 30 | +| Inference counters (2) by `model` x `provider` x `tier` | about 10 x 1 x 5 = 50 each | about 100 | +| Request and response bytes | 10 x 2 stages x 2 = 40 | 40 | +| `duration_seconds` | `model` 10 x `stream` 2 = 20 label sets x 14 | 280 | +| `ttfb_seconds` | 10 x 14 | 140 | +| `added_seconds` | 10 x 14 | 140 | +| `component_duration_seconds` | 30 x 14 | 420 | +| Keep-alive and cache health | about 10 models x 4 outcomes + 7 causes x 2 | about 150 | +| Proxy health and failures | about 100 | about 100 | +| **Total** | | **about 1,700** (the 40 for bytes is inside the rounding) | + +The total sits close to the target, so the target is a real limit and not a safe margin. If the count is too high, drop `model` from `added_seconds` and `ttfb_seconds`, or use fewer buckets for the component histogram. + +Target: **under 2,000 active series per machine.** For 50 machines this is 100,000 series. Check this limit in a test (R42 below). + +- R42. A test counts the label sets a typical session creates. It fails if the count exceeds the budget. + +## 6. Dashboards + +All dashboards use the central Prometheus. They are Grafana JSON files in `deploy/grafana/dashboards/`. Each panel states its query. Use `$__rate_interval`. + +Filters on every dashboard: `proxy`, `machine`, `model`, time range. + +### 6.1 Fleet overview + +Question: is the fleet healthy, and do we save money? + +| Panel | Type | Query idea | +|---|---|---| +| Machines reporting | stat | `count(count by (machine) (cg_proxy_info))`. Use `cg_proxy_info`: it is always 1, so an idle machine still shows. A request counter has no series until the first request. | +| Requests per minute | time series | `sum(rate(cg_inference_requests_total[...]))` by `proxy` | +| Error rate | stat | errors / requests | +| Spend today | stat | `increase(cg_inference_cost_usd_total[1d])` | +| Gross saved today | stat | `increase(cg_savings_saved_usd_total[1d])` | +| Added cost today | stat | `increase(cg_savings_added_usd_total[1d])` | +| **Net saved today** | stat | saved - added | +| Saving rate | stat | net / baseline | +| Net saved per day | bar | `increase(...[1d])` for 30 days | + +### 6.2 Savings + +Question: where does the saving come from, and where does it leak? + +| Panel | Type | Query idea | +|---|---|---| +| Saved vs added | time series | both counters, stacked | +| Net by kind | stacked bars | `kind` = rewrite, spend, setting | +| Net by component | bar gauge, sorted | top 15 components | +| Components with negative net | table | `component` where saved - added < 0 | +| Saving by tier | pie or stacked | which tier the saved tokens belong to | +| Cache effects | stacked bars | `break`, `rescue`, `split`, `ttl_premium`, `ttl_rescue` | +| Break cost by cause | bar | `effect="break"` by `cause` | +| Estimated share | stat | `estimated_total / priced_total` | +| Unpriced requests | time series | by `reason` | + +### 6.3 Cache and keep-alive + +Question: does the cache pay, and do pings pay? + +| Panel | Type | Query idea | +|---|---|---| +| Cache read ratio | time series | by `model` | +| Misses by cause | stacked bars | `cg_cache_misses_total` by `cause` | +| Miss cost by cause | bar | `cg_cache_miss_cost_usd_total` | +| Ping outcomes | stacked bars | by `outcome` | +| Ping cost vs rescue | time series | cost, rescue and net | +| Pings that paid | stat | `rescue_usd / cost_usd` | +| Shared-warm tokens | time series | credit not given (#853) | + +### 6.4 Inference + +Question: how does the traffic behave? + +| Panel | Type | Query idea | +|---|---|---| +| Requests by model | time series | `rate` by `model` | +| Latency p50, p95, p99 | time series | `histogram_quantile` on `duration_seconds` | +| Added latency p95 | time series | `added_seconds` | +| TTFB p95 for streams | time series | `ttfb_seconds` | +| Errors by status class and source | stacked bars | | +| Tokens by tier | stacked bars | `cg_inference_tokens_total` | +| Cost by model | stacked bars | | +| Cost per 1,000 requests | time series | derived | + +### 6.5 Proxy health + +Question: is context-guru itself working? + +| Panel | Type | Query idea | +|---|---|---| +| Proxy versions | table | `cg_proxy_info` | +| Component failures | time series | by `component` | +| Fail-open count | time series | by `reason` | +| Component time p95 | time series | `component_duration_seconds` | +| Export failures | time series | by `reason` | +| Machines silent | table | `last_over_time(cg_proxy_info[1d]) unless cg_proxy_info`. Prometheus writes no stale marker for pushed data, so use a query that compares with the last day. | +| Check failures | time series | `cg_savings_check_failures_total` by `check` | + +### 6.6 Machine comparison + +Question: which machines or configs save the most? + +| Panel | Type | Query idea | +|---|---|---| +| Net saved by machine | bar gauge | sorted | +| Saving rate by machine | table | net / baseline | +| Saving rate by preset | table | by the `preset` label | +| Saving rate by proxy | bar | `cortex` vs `context-guru` | + +## 7. Aggregations + +Recording rules make dashboard queries fast. Put them in one rules file. Name them `level:metric:operation`. + +| Rule | Expression | +|---|---| +| `fleet:cg_savings_usd_net:increase1d` | `sum(increase(cg_savings_saved_usd_total[1d])) - sum(increase(cg_savings_added_usd_total[1d]))` | +| `proxy:cg_savings_usd_net:rate5m` | same, `sum by (proxy)` | +| `machine:cg_savings_usd_net:rate5m` | same, `sum by (machine)` | +| `model:cg_savings_usd_net:rate5m` | same, `sum by (model)` if `model` is kept on savings. **Open.** | +| `component:cg_savings_usd_net:rate5m` | `sum by (component, kind)` | +| `fleet:cg_saving_rate:ratio1d` | net / baseline | +| `fleet:cg_cache_read_ratio:ratio5m` | cache_read / all input tokens | +| `fleet:cg_inference_error_ratio:ratio5m` | errors / requests | +| `fleet:cg_keepalive_payoff:ratio1d` | rescue_usd / cost_usd | + +Roll-up levels and where each one lives: + +| Level | Where | How | +|---|---|---| +| Fleet, proxy, machine, model, kind, tier, component | Prometheus | Labels and recording rules. | +| User | Prometheus if R32 allows. Otherwise the ledger. | Label `user`. | +| Session, thread, request | Ledger (event store) | All IDs stored with the record. | + +All values are sums (tokens and money), so they add up at every level. Ratios are always computed from sums, never averaged. + +## 8. Analysis + +Questions the data must answer. Each has a query or a drill-down. + +| # | Question | Method | +|---|---|---| +| A1 | Does context-guru save money net of its cost? | `saved - added` over a week. `added` includes the compaction model's own calls (R45). Must be above 0. | +| A2 | Which components lose money? | Net by component below 0 for 7 days. Candidate to turn off. This is #670. | +| A3 | How much do cache breaks cost compared with the savings? | Break `added` cost / `saved` cost. | +| A4 | Is keep-alive worth it? | Payoff ratio above 1 by model and by idle time. | +| A5 | Which preset saves most? | Saving rate grouped by `preset`. Needs enough machines per preset. | +| A6 | Do the two proxies differ? | Compare `proxy` values on the same metric names (R40). | +| A7 | How much of the figure is an estimate? | `estimated_total / priced_total`. If above 20 percent, treat the headline as a range. | +| A8 | Why do requests miss the cache? | Misses by cause, and cost by cause. | +| A9 | Does a model change break the cache often? | `cause="model_changed"` count and cost. | +| A10 | Is the saving real, or does our baseline overstate it? | Compare `baseline - cost` with the provider's own cache saving. Review monthly. | +| A11 | Does the proxy dashboard agree with the central metrics? | Compare the dashboard totals of one machine with the central totals for that `machine` over one week. Differences must be explained (retention, estimates, a push that was off). | +| A12 | What did one session save? | Drill down: ledger, by session ID. Not a Prometheus query. | + +### Alerts + +Alerts go to Alertmanager. Thresholds are proposals. + +| Alert | Condition | Severity | +|---|---|---| +| Net saving negative | Fleet net below 0 for 24 hours | warning | +| Component losing money | A component net below -$5 over 7 days | info | +| Many unpriced requests | Unpriced above 10 percent for 1 hour | warning | +| Check failures | Any `sum` or `conservation` failure | warning | +| Export failures | `export_failures_total` rising for 1 hour | warning | +| Machine silent | `cg_proxy_info` missing for 15 minutes | info | +| Error rate | Above 5 percent for 10 minutes | critical | +| Cache read ratio low | Below 0.5 for 1 hour on a model with traffic | info | + +## 9. Delivery plan + +1. **Hook.** Add `Sink`, `Result.Totals()`, `ByKind()`, `Failed()` to the `accountant`. Test them. No OTel. +2. **Shared module.** Create `accountant/metrics` with the savings set (section 5.2). Test with an in-memory OTel reader. +3. **Proxy wiring.** Call the sink after each response. Add the inference set (5.1) and the health set (5.5). Add the settings for the collector URL, the token and the interval, and the switch that turns the push off (R19). +4. **Keep-alive and cache sets** (5.3, 5.4). +5. **Collector and Prometheus config.** Add the collector config, the remote-write receiver flag, and the recording rules file. +6. **Dashboards and alerts** (sections 6 and 8). +7. **Retire the pull path** (R22). Remove `/metrics` (`proxy/promexport.go`, `dash/metrics_export.go`) and the local Prometheus and Grafana install. Move or delete the dashboards and alert rules in `deploy/grafana/`. Do this after the central dashboards (item 6) work. +8. **Docs.** Convention file in `accountant/docs/`, row in `docs/data-dependence.md` (the "Metrics (`/stats`, `/metrics`)" row changes), a section in `docs/hosted.md` (the Grafana section changes). + +Version one: items 1 to 3, and the savings dashboard (6.2). Tokens and money both. **Open:** confirm that tokens-only is not the first step. + +## 10. Test plan + +| Test | What it checks | +|---|---| +| Same numbers everywhere | The same priced result gives the same totals in the shared metrics, in the local ledger and in any proxy-only metric (R25, R26). | +| Totals add up | The split by kind adds up to the total. The total equals the sum of the parts (R26). | +| No duplicate code | The cg proxy and the Cortex proxy report the savings metrics through the same shared code. A check fails if either proxy defines a common metric itself (R26, R27). | +| Money at request time | A rate change does not change counters already recorded. | +| Saved and added | A request with a break adds to `added`. Net can be negative. | +| Unpriced request | Adds nothing to `saved` or `added`. Adds 1 to `unpriced_total`. | +| Label rule | No forbidden label (R31) appears on any metric. | +| Cardinality | Series per machine stay under the budget (R42). | +| Many machines | Several test proxies push at once. The central server shows each `machine` and the fleet sum equals the sum of the parts (R1, R4). | +| No inbound connection | A proxy behind a blocked inbound port still reports (R1). | +| Manager view has no content | The central series carry no session, thread, request or content value (R2, R7). | +| Local view without the push | With the push off, or the collector down, the proxy dashboard shows the same figures (R8). | +| Two proxies | A cg proxy and a Cortex proxy report the same common metric. One panel shows both, split by `proxy` (R10, R13). | +| Third proxy | A test proxy with a new `proxy` value appears with no change to the collector or Prometheus (R11). | +| Fail open | A collector that hangs or refuses does not delay a request or change a response. | +| Async push | With a collector that never answers, the time of a request and its answer is the same as with the push turned off, within test noise (R16). | +| Record is cheap | Recording one metric takes a fixed small time and makes no network call. A benchmark in the PR reports the time (R16). | +| Export failure | A failed push is counted. The next successful push carries the totals, so a counter does not lose counts. | +| Machine ID | The `machine` label is the install ID. It holds no host name (R43). | +| Pull path gone | After the retirement, `/metrics` is not served. The proxy dashboard shows the same figures as before (R22, R8). | +| Default URL | With the default config, the proxy pushes to the shipped URL with no other setting (R19). | +| No URL | With an empty URL, the proxy makes no outgoing connection, starts normally and logs one line (R19). | +| Turned off | With the push turned off, the proxy makes no outgoing connection for metrics (R19). | +| Agreement | For one session, the sink totals equal the dashboard `request_components` totals. | +| End to end | Proxy to a local collector to Prometheus. The expected names and labels appear. | + +## 11. Open questions + +1. Where does `accountant/metrics` live: in the `accountant` repo, or in its own repo? (R27, section 4.1) +2. How many users? Is `user` a label? (R32) +3. Do both proxies store full per-request records? If not, the ledger is the larger work item. +4. Does `Result` carry meta and usage already? It decides the shape of the hook (section 4.1). Does the `accountant` see the compaction model's own calls? If not, R45 needs the proxy to report them as a `spend` change. +5. Version one: tokens and money, or tokens only? +6. Who issues the access tokens, and who is on the read access list? (R44) +7. Is the `component` label small enough to keep? Is `model` kept on savings metrics? (5.2, section 7) +8. What happens to the dashboards and alert rules in `deploy/grafana/`? Port them to the central Grafana, or delete them. The host and logs dashboards use the node exporter and Loki, not `/metrics`. Do they stay? (R22, 5.6) +9. Which retention does the central Prometheus use? 30 days is enough for the dashboards above. Long-term trends need more. +10. How many machines must the fleet view support? (R5) +11. What local view and store does the Cortex proxy have? (R14) +12. Who are the managers, and may they see a split by user or team? A `team` label would need a rule like R32. + +## 12. Technical notes + +- Prometheus needs `--web.enable-remote-write-receiver`. +- The OTel to Prometheus conversion adds `_total` to counters and may add the unit as a suffix. It changes dots to underscores. Check the final names. +- Histogram buckets: use explicit bounds for latency. Suggested seconds: 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120. +- A machine that is offline sends its totals when it is back. The totals are correct, but the time of the increase is the time of the push, not the time of the events. Time detail stays in the dashboard store. Prometheus rejects out-of-order samples unless `out_of_order_time_window` is set. +- The OTel SDK aggregates in memory and keeps no queue of points, so a failed push loses no count. +- Counters restart at 0 when a process restarts. Use `rate()` and `increase()`, which handle this. +- Do not use the Pushgateway. +- Related existing code: `proxy/promexport.go` (pull exporter, to retire), `dash/metrics_export.go` (to retire), `deploy/grafana/` (to move or delete), `proxy/accountant_*.go` (the accountant wiring), `dash/accountant_price.go` (pricing). From 695a8a1bdd92f44377e5aa3c694425bc7b32b92f Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sun, 8 Nov 2026 11:21:20 +0200 Subject: [PATCH 55/58] build: Drop the stale accountant lines from go.sum go mod tidy -diff failed after the pin moved to 8e160764: the go.sum lines of 41d7cc52 were still there. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index deb72afb..9a95aa96 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rossoctl/accountant v0.1.1-0.20261107153853-41d7cc522cc2 h1:pNKgg7fjZzIlMZZzRqTn6ijsdNlyhM4J+8ZblABV94Q= -github.com/rossoctl/accountant v0.1.1-0.20261107153853-41d7cc522cc2/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/rossoctl/accountant v0.1.1-0.20261108054032-8e160764bb71 h1:fKOviX7sAMIbtgzyKiWPZstlUr3HU2ZuqAl3F57epnE= github.com/rossoctl/accountant v0.1.1-0.20261108054032-8e160764bb71/go.mod h1:AA9aM/QvO3AksXUGjOJiunMSxiZ1PqRsCxqKvq4IsrI= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= From 3f07381910c479f1c0ccbb953063bc88e311e57a Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sun, 8 Nov 2026 11:24:41 +0200 Subject: [PATCH 56/58] Revert "docs(prometheus): Add the metrics reporting spec" The spec is not part of the accountant integration. It stays on feat/prometheus-metrics. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- docs/prometheus/accountant-hook.svg | 86 --- docs/prometheus/discussion.md | 133 ---- docs/prometheus/existing-data-flow.svg | 49 -- .../prometheus/otel-prometheus-components.svg | 149 ----- docs/prometheus/spec.md | 603 ------------------ 5 files changed, 1020 deletions(-) delete mode 100644 docs/prometheus/accountant-hook.svg delete mode 100644 docs/prometheus/discussion.md delete mode 100644 docs/prometheus/existing-data-flow.svg delete mode 100644 docs/prometheus/otel-prometheus-components.svg delete mode 100644 docs/prometheus/spec.md diff --git a/docs/prometheus/accountant-hook.svg b/docs/prometheus/accountant-hook.svg deleted file mode 100644 index 620ccbc3..00000000 --- a/docs/prometheus/accountant-hook.svg +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - - Accountant hook: from one request to metrics and the ledger - - - - Proxy (cortex or context-guru) - - - - Request handler - sends request to LLM, - gets response + usage - - - - accountant (standard library only) - - Compute(Usage) -> Result - pure, no side effects - - Result.Record(meta) -> Record - pure; adds IDs, model, totals - - interface Sink { Observe(Record) } - one method; the accountant only - declares it, never calls it - - - - 1. Usage - - Result - - 2. Record - - - - Multi(sinks...) - 3. proxy calls Observe - - - implements - - - - accountant/metrics (separate module, defined once) - metric names, units and labels - tokens by tier + money (tokens x Rates) - saved and added counters - failed-check counter - options: user label on/off, prefix - input: Meter + Rates from the proxy - uses the OTel API only - - - - Ledger sink (proxy's own) - stores the full Record with request, - thread, session and user IDs - - - - Proxy-only sink or metric (optional) - own prefix: cortex_ or contextguru_ - - - - - - same Record goes to every sink, so all numbers agree - - - 4. counters (low-cardinality labels) - - - metrics: OTel -> collector -> Prometheus - - - push (OTLP) - answers per-session, per-thread, per-request sums - diff --git a/docs/prometheus/discussion.md b/docs/prometheus/discussion.md deleted file mode 100644 index 8689d419..00000000 --- a/docs/prometheus/discussion.md +++ /dev/null @@ -1,133 +0,0 @@ -# Reporting inference and savings metrics with Prometheus - -This note summarizes a design discussion. It is not a final decision. Open questions are at the end. - -## 1. Goal - -1. Collect regular inference metrics (requests, latency, errors, tokens, cost). -2. Collect savings metrics reported by the `accountant` library. -3. Do this for many local proxy instances (Cortex proxy or context-guru proxy) that report to one central Prometheus. -4. Define the metrics once, so two sets do not need to be maintained. -5. Support roll-up from request, to thread, to session, to user. Report both tokens and money. - -## 2. Prometheus basics - -- Prometheus normally **pulls**. The app serves plain text at `/metrics`, and Prometheus fetches it on a schedule. -- Counters are cumulative totals. Use `rate()` or `increase()` in a query to get values per time interval. -- A **label** is a key and value on a metric, such as `model="claude-sonnet"`. Each different set of labels is a separate series. -- Each new label value makes a new series. Labels with many or unlimited values (user ID, session ID, request ID, prompt text) make Prometheus slow and costly. Use labels with few values: `model`, `status`, `endpoint`, `host`. -- Good metric types: Counter for totals that only grow, Histogram for latency and sizes, Gauge for values that go up and down. - -## 3. Collecting from many local instances - -Pull works only if the central Prometheus can reach each instance. Laptops behind NAT or a firewall usually cannot be reached. Options: - -| Option | How | Fit | -|---|---|---| -| Central scrape | Prometheus scrapes each instance | Servers with stable addresses | -| Local agent + remote write | A small agent on each machine scrapes `localhost` and sends out over HTTPS | Existing Prometheus-library code | -| OpenTelemetry (OTel) push | The app sends metrics to a central collector | Chosen option in this note | -| Pushgateway | Apps push to a gateway | Do not use. For short batch jobs only | - -### Chosen setup: OTel push - -- The proxy records metrics through the OTel library and **pushes** them to a central OTel Collector over HTTPS (outgoing connections only). -- The collector adds a machine label and a `proxy` label, then **pushes** to Prometheus with remote write. Prometheus must run with `--web.enable-remote-write-receiver`. Nothing calls `/metrics` on this path. -- Alternatively, the collector can expose its own `/metrics` and Prometheus can scrape it. -- The collector, Prometheus, Grafana and Alertmanager can run on one server for a small team. Expose only the collector to the local machines. -- Offline machines may lose data, because the library keeps no long backlog and Prometheus rejects very old samples. -- Adding a new metric changes only the proxy code. The collector, Prometheus and Grafana need no setup change. Check the final metric name in Prometheus, because it may be renamed (for example `_total` is added to counters). - -![Components: local machines, collector, Prometheus, Grafana](otel-prometheus-components.svg) - -## 4. Who owns the metric logic - -The proxy owns metrics reporting. The accountant returns data only. - -1. The accountant module imports only the standard library, and `boundary_test.go` enforces that. It must not import OTel or Prometheus. -2. `Compute` is deterministic and has no side effects. Keep it that way. -3. Two proxies share the accountant. The metric definitions must exist once, not twice. -4. The metrics may differ between proxies, so the shared part must not fix the full metric list. - -### Layers - -``` -accountant -> Result (tokens saved per change, per tier, checks); no OTel -accountant/metrics -> separate Go module; defines common metrics once; uses the OTel API only -proxy -> passes Meter and Rates, picks options, adds its own metrics -``` - -### Share and keep separate - -| Share (accountant or the metrics module) | Keep in each proxy | -|---|---| -| Roll-ups of `Result` (totals, per operation kind, failed checks) | Which extra metrics exist | -| Tier and operation-kind constants | Prefix for proxy-only metrics (`cortex_`, `contextguru_`) | -| Common metric names, units and labels | Rates used to convert tokens to money | -| | OTel setup, endpoint, token, interval | - -### Keep the data comparable - -1. Use the same names and labels for the metrics both proxies have. -2. The collector adds `proxy="cortex"` or `proxy="context-guru"` to everything. -3. Write the convention in one short file in the accountant `docs/`. - -## 5. The accountant hook - -The accountant declares a small interface. The proxy calls it after each finished request. The accountant never calls it. - -```go -type Sink interface{ Observe(Meta, Usage, Result) } -``` - -- `Compute` stays pure. The proxy calls `sink.Observe(...)` itself, after the response returns. -- The proxy can pass the data to several sinks with a `Multi(sinks...)` helper: shared metrics, the proxy's ledger, and optional proxy-only metrics. -- Because the sink gets the same data, all numbers agree. - -### Record, or Result plus Meta? - -A separate `Record` type is not required. Passing `Meta`, `Usage` and `Result` works and has one type fewer. The risk is that each sink sums the per-change numbers itself, and the copies drift. Avoid this with pure helper methods on `Result`: - -```go -res.Totals() // saved and added, per tier -res.ByKind() // the same, per rewrite / spend / setting -res.Failed() // names of failed checks -``` - -Recommendation: use `Observe(Meta, Usage, Result)` plus the helpers. Add a `Record` type later only if the ledger needs a frozen on-disk format. Confirm in `docs/design.md` whether `Result` already carries the meta and usage. If it does, the call can be `Observe(Result)`. - -![Accountant hook: from one request to metrics and the ledger](accountant-hook.svg) - -The diagram shows a `Record` for clarity. Read it as the data passed to `Observe`. - -## 6. Tokens and money - -1. The accountant knows tokens only. The host turns tokens into money with its own rates. -2. The shared metrics module takes `Rates` and records both tokens and money, by tier. -3. Compute money when the request happens, with the rate of that moment, and store it. Do not recompute old money with new rates. -4. Tokens by tier: `input`, `cache_write_5m`, `cache_write_1h`, `cache_read`. -5. A change can break the cache and cost more than it saves. Savings can be negative. Use two counters, `saved` and `added`. Net savings is `saved - added` in a query. -6. Label by operation kind (`rewrite`, `spend`, `setting`), not by change ID. Change IDs are unbounded. -7. Count failed checks from `Result.Checks`. - -## 7. Roll-up: request, thread, session, user - -Prometheus labels cannot hold request, thread or session IDs. Split the job: - -| Level | Where | How | -|---|---|---| -| Fleet, proxy, model, operation kind, tier | Prometheus | Labels with few values | -| User | Prometheus if there are few users (tens). Otherwise the ledger | Label `user` | -| Session, thread, request | Ledger (event store) | Store each record in full with all IDs | - -- All values are sums (tokens and money), so they roll up at every level. -- Prometheus answers "how much did we save per model per hour". The ledger answers "how much did this session save". -- For a jump from a chart to one request, use OTel traces and put the request and session IDs on the trace, not on the metric. - -## 8. Open questions - -1. Where does the shared metrics module live: as a separate module in the accountant repo, or in its own repo? -2. How many users are expected? This decides whether `user` is a metric label. -3. Do both proxies already store full per-request records? If not, the ledger sink is the larger piece of work. -4. Does `Result` already carry the meta and usage? -5. Is the first version tokens and money, or tokens only? diff --git a/docs/prometheus/existing-data-flow.svg b/docs/prometheus/existing-data-flow.svg deleted file mode 100644 index 6544b45e..00000000 --- a/docs/prometheus/existing-data-flow.svg +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - What exists today: one proxy, two readers - - One machine (no data leaves it) - - - Request - - Proxy - - - - SQL store - SQLite, one row per request - - In-memory counters - reset at restart - - - - - Dashboard UI - reads /api/* - - - - /metrics - cg_tenant_* and cg_* - - - - - Local Prometheus - scrapes every 15 s - - pull - - - Grafana - 4 dashboards, 3 alert rules - - - The UI and /metrics do not read each other. Both read the store or process memory. - diff --git a/docs/prometheus/otel-prometheus-components.svg b/docs/prometheus/otel-prometheus-components.svg deleted file mode 100644 index de896834..00000000 --- a/docs/prometheus/otel-prometheus-components.svg +++ /dev/null @@ -1,149 +0,0 @@ - - - - - - - - Inference metrics with OpenTelemetry (OTel) and Prometheus - - - - - Local machine 1 - - User - - Harness - - Proxy - - - - Proxy - dashboard - - - SQLite - small local DB - - writes - - reads - - views - - - - - - Local machine 2 - - User - - Harness - - Proxy - - - - Proxy - dashboard - - - SQLite - small local DB - - writes - - reads - - views - - - - - - Local machine N - - User - - Harness - - Proxy - - - - Proxy - dashboard - - - SQLite - small local DB - - writes - - reads - - views - - - - - LLM provider - hosted or self-hosted - - 1. Each proxy sends inference requests (all machines do this) - - - - - - 2. Proxy pushes metrics - (OTLP over HTTPS, - every ~30 s, - outgoing only) - - - - - Central server (these four can run on one machine) - - - OTel Collector - checks the token - adds a machine label - batches the data - - - Prometheus - stores numbers with - timestamps - answers queries - - - 3. Collector pushes - (remote write; no /metrics scrape) - - - Grafana - charts and dashboards - - 4. Ask for data - - - Alertmanager - sends alerts (optional) - - alert rules fire - - - Analysis user - - 5. Views - dashboards - - - Solid arrow = data flow. Dashed arrow = optional. Arrows from the machines point outward, so firewalls and NAT do not block them. - Purple arrow = the user opens the proxy dashboard. It reads the small local SQLite DB that the proxy writes. This data stays on the machine. - diff --git a/docs/prometheus/spec.md b/docs/prometheus/spec.md deleted file mode 100644 index ee8a169e..00000000 --- a/docs/prometheus/spec.md +++ /dev/null @@ -1,603 +0,0 @@ -# Metrics reporting: spec - -Status: draft. Issue: #886. Input: [discussion.md](discussion.md). - -This spec turns the discussion into requirements, a metric list, dashboards and analyses. Items marked **Open** need a decision before the work starts. - -## 1. Goal - -1. Report inference metrics and `accountant` savings metrics from every proxy to one central Prometheus. -2. Define the common metrics once. The Cortex proxy and the context-guru proxy share them. -3. Show money and tokens at every roll-up level that Prometheus can hold: fleet, proxy, machine, model, operation kind and tier. -4. Show whether context-guru earns its cost: gross saving, added cost, net saving. -5. Support many local proxies that report to one central metrics server. Managers use it to estimate the savings of the whole fleet. -6. Let each user see the details of their own traffic on their own proxy. -7. Support at least two proxies: the context-guru proxy (cg) and the Cortex proxy. - -Not goals: -- Per-session, per-thread and per-request figures. The dashboard store and the ledger hold these. -- Replacing the dashboard (`dash/`). Prometheus gives the fleet view. The dashboard gives the session view. -- Prompt text or any customer content in a metric or label. - -## 2. What exists today - -Three parts exist. They share one data flow: - -![What exists today: one proxy, two readers](existing-data-flow.svg) - -The dashboard UI and `/metrics` both read the store or the proxy's memory. Neither reads the other. - -### 2.1 The dashboard UI (`dash/ui`) - -A single-page app embedded in the binary. It is **off by default**. `--dashboard` turns it on. It needs no Prometheus, no CDN and no network. - -What it covers: -- **Overview.** Dollars saved out of what the traffic would have cost, with percent saved and tokens removed. Cards for cache, preset and tool filter. Five diagnostic plots and a live request table. -- **Savings.** Cache strategy (keep-alive), tool inventory, cleanup advice, usage over time. -- **Traffic.** Sessions and requests. A request view shows a diff of what was rewritten on the wire. -- **Admin.** Config, cache strategies, tenants, setup, settings, archive, feedback (some only in a hosted service). -- **Advanced.** Per-component economics, the KV-cache view, strategy campaigns. - -Limits: -1. **One proxy only.** Each proxy has its own store. No view spans machines. -2. **Needs the store.** It shows nothing without `--dashboard`. -3. **Access is by network location.** Aggregates are open to anyone who can reach the port. Content and prompt text need loopback or a trusted CIDR. There is no fleet-wide sign-in outside the hosted service. -4. **No alerts.** It shows numbers. It does not page anyone. -5. **No export format for other tools.** Other tools read `/api/*` or `/metrics`. - -### 2.2 The SQL store (`dash/store.go`, `dash/schema.go`) - -SQLite in WAL mode (pure Go). One row per request in `requests`. Other tables hold per-component figures (`request_components`), request content (`request_content`), tool declarations and uses, session state, request lifecycle, extraction calls, tenant spend and the archive index. - -What it covers: -- **Every request, in full detail.** Tokens by tier, cost, baseline cost, cache fields, cache miss cause, thread and session IDs, keep-alive fields, settings. -- **Per-component savings.** `saved_usd`, the split `saved_first_usd` and `saved_replay_usd`, and `priced_by`. The `accountant` library produces these figures. -- **State that survives a restart.** Session state, checkpoints, and the data the cache tracker needs. -- **Cold storage.** Old sessions move to an archive. An index and the monthly tenant spend are kept. - -Limits: -1. **Local to one proxy.** There is no sync to a central place. -2. **Retention is bounded** by age (`--dashboard-retention`) and by size (`--dashboard-max-bytes`). A burst can evict old rows. -3. **Derived data can be dropped.** When the schema version changes, the metrics tables restart. Only the archive index and tenant spend carry over. -4. **Not built for fleet queries.** Cross-machine roll-up needs a second system. -5. **May hold customer text.** The content tables are a deliberate exception to "metadata first". They are gated by storage level and by access rules. -6. **Old rows keep old figures.** Rows written before the `accountant` switch keep the legacy figure (`priced_by` empty). A window across the switch mixes two definitions. - -### 2.3 The `/metrics` endpoint (`proxy/promexport.go`) - -Prometheus text, served by the proxy. It is hand-written and uses no client library. A local Prometheus scrapes it every 15 seconds. `deploy/grafana/` has the scrape config, 4 dashboards (`context-guru`, `host`, `slo`, `logs`) and 3 alert rules (`cg-service-down`, `cg-refusal-rate`, `cg-metrics-stale`). `install.sh grafana` installs it. - -Two families: - -| Family | Source | Scope | Behavior | -|---|---|---|---| -| `cg_*` | In-memory aggregator, the same data as `/stats` | All tenants together | Starts at 0 when the process starts. Real counters. Use `rate()`. | -| `cg_tenant_*` | The SQL store, cached about 60 s | One tenant | Month to date. Most are gauges, including four that end in `_total`. | - -What it covers: -- Requests, refusals, latency, added latency, time to first byte for streams. -- Tokens before and after compaction, billed tokens by tier. -- Cost, baseline cost, saved dollars, net saved dollars, the compaction model's own cost. -- Per-component runs, events, tokens removed and time spent. -- Expand and stash counters, cache-frozen headroom, extraction value. -- Exporter health: `cg_metrics_age_seconds`, render time, dropped capture events. -- Per-tenant cost, tokens, sessions and archive size. - -Limits: -1. **Pull only.** A central Prometheus must reach the proxy. A laptop behind NAT or a firewall cannot be reached. -2. **Two families do not agree.** `cg_*` is per process and resets at restart. `cg_tenant_*` is per tenant and month to date. A panel that mixes them is wrong. The HELP text says so. -3. **Month-to-date gauges.** `rate()` and `increase()` read a fall as a counter reset. No shipped panel uses them on these series. -4. **Savings are single figures.** `cg_saved_usd` and `cg_net_saved_usd` are gauges. There is no `saved` and `added` pair. There is no split by operation kind, by tier, or by cache effect (break, rescue, split, TTL). -5. **No keep-alive or cache-miss series by cause.** The data is in the store. It is not exported. -6. **Label set is small.** Labels in use: `tenant`, `tier` (billed tokens only), `component` (component series only), `reason`, `gate` and `event`. There is no `model` or `provider` label, so a cost or a saving cannot be split by model. -7. **Service-wide view.** It carries every tenant's spend. It is open on loopback only, or with `METRICS_TOKEN`. The deploy keeps Prometheus and Grafana on loopback. -8. **Not shared with the Cortex proxy.** The names live in this proxy's code. A second proxy would copy them. -9. **Grafana is a second view.** It reads only `/metrics`. It cannot show a session or a request. - -### 2.4 What the push work adds - -| Gap | Part that has it | What this spec adds | -|---|---|---| -| No fleet view | UI, store, `/metrics` | A push path to one central Prometheus (section 4). | -| Savings are one figure | `/metrics` | `saved` and `added` counters by kind, tier and component (5.2). | -| Not shared with Cortex | `/metrics` | One metrics module for both proxies (R27). | -| No cache-miss or keep-alive series | `/metrics` | New counters (5.3, 5.4). | - -What stays as it is: the UI and the store. The pull `/metrics` and the local Prometheus and Grafana are retired (R22). Prometheus cannot replace the store or the UI, because it cannot hold session, thread or request IDs as labels. Section 4 shows where each part sits. - -## 3. Requirements - -### 3.1 Needs: fleet, user and proxies - -Three needs drive these requirements. - -| Need | Who | Where it is served | -|---|---|---| -| Estimate the savings of the whole fleet | Manager | Central Prometheus and Grafana | -| See the details of my own traffic | User | Own proxy dashboard and local DB | -| Run on more than one proxy | Both | Shared metrics, one `proxy` label | - -**Many proxies, one central server (managers)** - -- R1. Many local proxies report to one central metrics server. A proxy needs no inbound connection, so a laptop behind NAT or a firewall can report. -- R2. A manager can estimate the savings of the fleet from the central server alone. The manager needs no access to any local proxy and sees no customer content. -- R3. The manager view shows gross saving, added cost and net saving. It can be split by `proxy`, `machine`, `model` and `preset`. It shows the share of the figure that is an estimate (R36, section 5.2). -- R4. The central server shows which machine each number comes from. If a machine stops sending data, the manager sees it within 15 minutes. -- R5. The design supports at least **50 machines** with the cardinality budget in section 5.8. **Open:** confirm the target (tens, hundreds or more). - -**Details for a user on the own proxy** - -- R6. A user can see the details of their own traffic on their own proxy: sessions, requests, tokens by tier, cost, saving per component and cache misses. This is the proxy dashboard and its local DB. -- R7. The details stay on the machine. Session, thread and request IDs, and any stored content, are never sent to the central server (R29, R31). -- R8. The proxy dashboard and its local DB work with no collector and no network. Turning the push on or off does not change what the user sees there. - -**More than one proxy** - -- R9. The design supports at least two proxies: the context-guru proxy (`cg`) and the Cortex proxy. -- R10. Both proxies report the common metrics (section 5) with the same names, units and labels. The `proxy` label tells them apart (R40). -- R11. Adding a third proxy changes only that proxy's code and a new `proxy` value. The collector, Prometheus and Grafana need no new setup. -- R12. Each proxy keeps its own extra metrics under its own prefix (R28). A dashboard panel on a common metric works for both proxies without a filter. -- R13. The manager view (R2, R3) shows the two proxies side by side, so managers can compare them. -- R14. The Cortex proxy gives its user the same details as R6. **Open:** confirm what local view and store the Cortex proxy has. If it has none, R6 applies to the cg proxy first and the Cortex proxy later. - -### 3.2 Transport (design rules) - -- R15. The proxy pushes metrics to an OpenTelemetry Collector over HTTPS. It makes outgoing connections only. -- R16. The push is **asynchronous**. The proxy records a metric in memory and returns at once. A separate background task sends the data to the collector in batches. The push never runs on the path of a request. It adds no time to a request or to the answer, and it never waits for the collector. -- R17. The collector adds the labels `machine` and `proxy`. It sends the data to Prometheus with remote write. -- R19. The product ships a default collector URL, so the push works from the first run. The URL is a setting. The user or the operator can change it in the config file, with a flag, or with an environment variable. The access token is a setting too (R44). A setting turns the push off. If the URL is empty, the proxy makes no connection and writes one log line at start. The PR chooses the names of the settings. Data leaves every machine from the first run, so the PR names this at the top: who it affects, what is sent, and how to turn it off (`RELEASING.md`). -- R20. If the collector is down or slow, the proxy keeps working. A failed push never delays or changes a request, and it never blocks a request. Fail open. -- R22. The pull path is retired. The `/metrics` endpoint and the local Prometheus and Grafana setup are removed. The product is new and has no legacy users, and the central server replaces them. The developer (David Boaz) decided this. The PR names the change at the top: who it affects, the old and new behavior, and that the user's own view is the proxy dashboard (R8). The dashboard and its local DB do not use `/metrics`, so they are not affected. - -### 3.3 Consistency across proxies - -- R23. The `accountant` library stays free of metrics code. It imports the standard library only, and it never sends or stores a metric. `boundary_test.go` keeps enforcing this. -- R24. Pricing a request stays pure. It has no side effects. Reporting a metric happens after the pricing, from the result. -- R25. The savings metrics are the same on every proxy. Each metric has the same name, unit and meaning, and every proxy computes it from the same `accountant` result. -- R26. A proxy needs no copy of the metric code. Adding the savings metrics to a proxy takes a small, fixed amount of code in that proxy. The totals, the split by kind, the failed checks and the metric definitions exist once. -- R27. The common metrics are defined once, in one place that both proxies import. A fix or a new common metric is made once and reaches every proxy. -- R28. Each proxy can add its own metrics under its own prefix (`cortex_`, `contextguru_`). The common set does not fix the full list. - -### 3.4 Data rules - -- R29. Metrics use counts, sizes, token numbers, prices, model names and status codes only. They use no stored customer data. -- R30. Allowed labels have few values: `proxy`, `machine`, `model`, `provider`, `status_class`, `stream`, `stage`, `direction`, `source`, `kind`, `tier`, `effect`, `cause`, `component`, `result`, `reason`, `check`, `outcome`, `version`, `preset`. -- R31. Forbidden labels: request ID, session ID, thread ID, change ID, user ID (unless R32), prompt text, tool name, file path. -- R32. `user` is a label only if the user count is about 50 or fewer. **Open.** Otherwise per-user figures come from the ledger. -- R33. Tokens are recorded as tokens. Money is recorded when the request happens, at the rate of that moment. Old money is never recomputed with new rates. -- R34. Tokens by tier use the values `input` (uncached input), `cache_read`, `cache_write_5m`, `cache_write_1h` and `output`. The two write tiers stay apart because they have different prices. -- R35. `saved` and `added` are two counters. A change can cost more than it saves. Net is `saved - added` in a query. The cost of the compaction model's own calls is part of `added`, booked under the component that made the call (R45). -- R36. A request the `accountant` cannot price adds nothing to `saved` or `added`. It adds 1 to a counter of unpriced requests, by reason. - -- R45. The cost of an LLM call that a component makes itself, such as compaction, is booked as `added` (cost and tokens) under that component's `component` label. There is no separate counter, so net is always `saved - added`. **Open:** confirm that the `accountant` sees these calls (open question 4). If it does not, the proxy must pass them to the `accountant` as a `spend` change. - -### 3.5 Convention - -- R39. One short file in the `accountant` `docs/` lists the common names, units and labels. -- R40. Both proxies use the same names and labels for the metrics they share. - -### 3.6 Documentation - -- R41. The PR adds a row to `docs/data-dependence.md`. The row says: metadata only, works at every storage level. - -### 3.7 Privacy and access - -- R43. `machine` is a pseudonymous install ID: a random value made at first start. It is not the host name, because a host name often holds a person's name. The operator keeps the map from install ID to person or team outside the metrics system. -- R44. The collector accepts a push only over HTTPS and only with a valid access token. Each install has its own token, so one token can be revoked. Read access to the central Prometheus and Grafana is limited to named managers and operators. **Open:** who issues the tokens, and who is on the access list. - -## 4. Proposed architecture - -There is one path: push. The pull path (section 2) is retired (R22). The picture shows the push path: each proxy sends metrics to a central collector, which sends them to Prometheus and Grafana. The push is asynchronous: it runs in the background, off the path of a request, so it adds no time to the proxy (R16). - -![Components: local machines, collector, Prometheus, Grafana](otel-prometheus-components.svg) - -| Component | Status | Role | -|---|---|---| -| proxy | exists | Forwards requests and calls the `accountant`. | -| `accountant` | exists | Returns `Result`. Imports no OTel (R23). | -| `Sink`, `Result` helpers | new | One place that turns `Result` into numbers (R25, R26, section 4.1). | -| `accountant/metrics` | new | Defines the common metrics once (R27). | -| OTel SDK in the proxy | new | Aggregates in memory and pushes in the background (R16). | -| dash store and `dash/ui` | exists | Per-session and per-request views. Do not use Prometheus. | -| OTel Collector | new | Receives pushes. Adds `machine` and `proxy`. | -| central Prometheus | new | Stores the pushed series. | -| central Grafana, Alertmanager | new | Fleet dashboards and alerts. | - -### 4.1 Design notes - -These notes say how the requirements can be met. They are proposals. The PR may change them if R23 to R28 still hold. - -| Requirement | Proposed way | -|---|---| -| R23, R24 | The `accountant` keeps its pure `Compute`. The proxy reports after the response returns. | -| R25, R26 | A small hook in the `accountant` (`Sink`, called as `Observe(Meta, Usage, Result)`) and helper methods on `Result` (`Totals()`, `ByKind()`, `Failed()`). A `Multi(...)` helper passes the same data to several consumers: the shared metrics, the local ledger and any proxy-only metrics. | -| R27 | A separate Go module `accountant/metrics`. It uses the OpenTelemetry API only. The proxy gives it a `Meter` and the price rates. | -| R28 | The proxy adds its own metrics with its own prefix. | - -Open choices: -- If `Result` already carries the meta and the usage, the hook can be `Observe(Result)`. -- The shared module can live in the `accountant` repo or in its own repo. - -## 5. Proposed metrics - -OTel metric names use dots and units. The Prometheus names below are what you see after the collector converts them. Counters get `_total`. A unit suffix such as `_usd` stays. Check the final names in Prometheus. - -Prefix `cg_` is for the common set. It matches the existing convention. Section 5.6 maps the retired pull series to the new ones. - -### 5.1 Inference (common) -Labels in this section: - -| Label | Values | Meaning | -|---|---|---| -| `model` | The model name the request was sent for, such as `claude-sonnet-5-5`. About 10 values per machine. | Which model served the request. If a component changed the model, this is the model after the change. | -| `provider` | The upstream provider, such as `anthropic`, `openai`, `bedrock`. A few values. | Who billed the request. | -| `status_class` | `2xx`, `3xx`, `4xx`, `5xx` | The HTTP status of the answer, in classes. A single code is not a label. | -| `stream` | `true`, `false` | Whether the answer was streamed. | -| `source` | `upstream`, `proxy`, `client` | Who caused an error. `upstream`: the provider failed. `proxy`: context-guru failed. `client`: the client hung up or sent a bad request. | -| `tier` | `input`, `cache_read`, `cache_write_5m`, `cache_write_1h`, `output` (R34) | The price class of the tokens. Each class has its own rate. | -| `stage` | Request bytes: `client`, `forwarded`. Response bytes: `upstream`, `delivered`. | Where in the proxy the body was measured. The two values of one counter show what the proxy changed. | - - -| Metric | Type | Unit | Labels | Meaning | -|---|---|---|---|---| -| `cg_inference_requests_total` | counter | 1 | `model`, `provider`, `status_class`, `stream` | Requests that reached the proxy. | -| `cg_inference_errors_total` | counter | 1 | `model`, `provider`, `status_class`, `source` | Requests that failed. `source` is `upstream`, `proxy` or `client`. | -| `cg_inference_request_bytes_total` | counter | By | `model`, `provider`, `stage` | Size of the request body. `stage` is `client` (as the client sent it to the proxy) or `forwarded` (as the proxy sent it upstream, after the components ran). Bytes removed is `client - forwarded`. Headers are not counted. | -| `cg_inference_response_bytes_total` | counter | By | `model`, `provider`, `stage` | Size of the response body. `stage` is `upstream` (as the provider sent it to the proxy) or `delivered` (as the proxy sent it to the client, after any rewrite). For a stream, the sum of the chunks. Headers are not counted. | -| `cg_inference_duration_seconds` | histogram | s | `model`, `provider`, `stream` | Total time, from request to last byte. | -| `cg_inference_ttfb_seconds` | histogram | s | `model`, `provider` | Time to first byte for streamed answers. | -| `cg_inference_added_seconds` | histogram | s | `model` | Time the proxy added on top of the upstream time. | -| `cg_inference_tokens_total` | counter | 1 | `model`, `provider`, `tier` | Provider-billed tokens. `tier` is from R34: read input (`cache_read`), write input (`cache_write_5m`, `cache_write_1h`), uncached `input`, and `output`. | -| `cg_inference_cost_usd_total` | counter | USD | `model`, `provider`, `tier` | Provider cost at request time. | -| `cg_inference_estimated_cost_usd_total` | counter | USD | `model`, `provider` | Part of the cost priced from a fallback rate. Sub-set of the line above. | - -The cache read ratio is not exported. Query it: `cache_read / (cache_read + cache_write_* + input)` on `cg_inference_tokens_total`. - -**Names (11).** Every counter is `cg__[_]_total`, with `usd`, `tokens` or `bytes` as the unit when the counter has one. Histograms end in `_seconds`. The collector must not add a second unit suffix (for example `_USD`). A test compares the names Prometheus shows with this table. - -### 5.2 Savings (from the `accountant`) -Labels in this section: - -| Label | Values | Meaning | -|---|---|---| -| `kind` | `rewrite`, `spend`, `setting` | The type of change, from the `accountant`'s `Operation.Kind`. `rewrite`: an edit to the prompt body, such as `mask` or `dedup`. `spend`: the component's own LLM call, such as compaction (R45). `setting`: a changed request parameter that the body does not show but that changes the cost or the cache, such as the model or the cache TTL. | -| `tier` | Same values as 5.1 (R34) | The price class of the tokens saved or added. | -| `component` | The operator name without the host prefix, such as `mask`, `dedup`, `headttl`. About 30 values. | Which operator made the change. One component can report more than one `kind`. | -| `effect` | `break`, `rescue`, `split`, `ttl_premium`, `ttl_rescue` | The cache effect of a change. `break`: a change moved the prefix and the cache missed. `rescue`: a change or a ping kept a cache that would have expired. `split`: a change split one cached prefix into two. `ttl_premium`: extra cost of a longer cache TTL. `ttl_rescue`: saving from a longer TTL. | -| `cause` | `model_changed`, `ended_change`, `none` | Why the effect happened. `model_changed`: the model changed. `ended_change`: a change that was active before stopped. `none`: no single cause. | -| `direction` | `saved`, `added` | Whether the cache effect lowered or raised the bill. | -| `model` | As in 5.1 | The model of the baseline cost. | -| `result` | `priced`, `unpriced`, `skipped` | What the `accountant` did with the request. | -| `reason` | For `unpriced`: `penalty_unknown`, `unknown_kind`, `operator_not_allowed` and others from the library. For `fallback`: the gate that declined. | Why the request was not priced, or why no saving was booked. | -| `check` | `sum`, `order`, `unique_change`, `spans`, `attribution`, `conservation`, `shared_warm` | The library check that failed. | - - -| Metric | Type | Unit | Labels | Meaning | -|---|---|---|---|---| -| `cg_savings_saved_tokens_total` | counter | 1 | `kind`, `tier`, `component` | Tokens the provider did not bill, by tier. | -| `cg_savings_added_tokens_total` | counter | 1 | `kind`, `tier`, `component` | Tokens the provider billed because of a change. | -| `cg_savings_saved_usd_total` | counter | USD | `kind`, `tier`, `component` | `tokens_saved` priced at the tier rate at request time. | -| `cg_savings_added_usd_total` | counter | USD | `kind`, `tier`, `component` | `tokens_added` priced the same way. | -| `cg_savings_cache_effect_usd_total` | counter | USD | `effect`, `cause`, `direction` | Cache effects: `effect` is `break`, `rescue`, `split`, `ttl_premium`, `ttl_rescue`. `direction` is `saved` or `added`. `cause` is `model_changed`, `ended_change` or `none`. | -| `cg_savings_baseline_usd_total` | counter | USD | `model` | What the traffic would have cost with no change. | -| `cg_savings_requests_total` | counter | 1 | `result` | `result` is `priced`, `unpriced`, `skipped`. | -| `cg_savings_unpriced_total` | counter | 1 | `reason` | Why a request was not priced. Values come from the library: `penalty_unknown`, `unknown_kind`, `operator_not_allowed` and others. | -| `cg_savings_estimated_total` | counter | 1 | none | Requests whose figure is marked estimated. | -| `cg_savings_check_failures_total` | counter | 1 | `check` | Failed library checks: `sum`, `order`, `unique_change`, `spans`, `attribution`, `conservation`, `shared_warm`. | -| `cg_savings_fallback_total` | counter | 1 | `reason` | Requests that booked no saving because the dashboard gate declined. | - -**Saved tokens and added tokens.** Both counters compare the request as it was sent with the request as the client wrote it. They are two sides of one change, and neither one is net. -- `cg_savings_saved_tokens_total`: tokens the provider billed less, because of a change. Example: `dedup` removes 10,000 tokens from the prompt. Saved: 10,000 at tier `input`, or at tier `cache_read` if those tokens would have been read from the cache. -- `cg_savings_added_tokens_total`: tokens the provider billed more, because of a change. Examples: a rewrite changes the prefix, the cache breaks, and 50,000 tokens are billed as `cache_write_5m` where they would have been `cache_read`. A keep-alive ping bills its own tokens. A compaction call bills the compaction model's tokens (R45). -- One change can add to both. A rewrite can save 5,000 input tokens and also cause a break that adds 50,000 write tokens. Do not net them inside the change. Each counter takes its own part, so the reports can show a component that saves tokens but loses money. -- Both counters are never negative. They only rise. -- Tokens at different tiers have different prices, so `saved_tokens - added_tokens` is not a money figure. Use the `usd` counters for money. Each `usd` counter is its tokens priced at the tier rate at request time (R33). -- The tokens counters answer "how much did the traffic shrink or grow?". The `usd` counters answer "did it pay?". - -**No double counting (5).** `cg_savings_saved_usd_total` and `cg_savings_added_usd_total` are the only counters that add up to the saving. They already include the cache effects. `cg_savings_cache_effect_usd_total` is a breakdown of the same money by effect. A query must never add it to `saved` or `added`. A test checks that, for each priced request, the sum of the cache effects is not larger than `saved + added`. **Open:** confirm with the `accountant` that cache effects are part of the component figures. If they are not, give them `kind="cache"` and drop the separate counter. - -The `component` count is bounded by the shipped list. **Open:** confirm the count is small enough to keep as a label. - -### 5.3 Keep-alive -Labels in this section: - -| Label | Values | Meaning | -|---|---|---| -| `model` | As in 5.1 | The model whose cache the ping kept warm. | -| `outcome` | `ok`, `failed`, `timeout`, `skipped` | `ok`: the ping succeeded. `failed`: the provider returned an error. `timeout`: no answer in time. `skipped`: the proxy chose not to ping, for example because a sibling thread kept the cache warm. | - - -| Metric | Type | Unit | Labels | Meaning | -|---|---|---|---|---| -| `cg_keepalive_pings_total` | counter | 1 | `model`, `outcome` | `outcome` is `ok`, `failed`, `timeout`, `skipped`. | -| `cg_keepalive_cost_usd_total` | counter | USD | `model` | What pings cost. | -| `cg_keepalive_rescue_usd_total` | counter | USD | `model` | Saving from rescues (`rescue` effect). | -| `cg_keepalive_shared_warm_tokens_total` | counter | 1 | `model` | Tokens not credited because a sibling thread kept them warm (#853). | - -Net keep-alive value is `rescue_usd - cost_usd`. - -### 5.4 Cache health - -| Metric | Type | Unit | Labels | Meaning | -|---|---|---|---|---| -| `cg_cache_misses_total` | counter | 1 | `model`, `cause` | Misses by cause from `cache_miss_cause`: `ttl_expiry`, `prefix_change`, `model_changed`, `compaction_after_rewrite`, `bypass_flip`, `cold_start`, `other`. | -| `cg_cache_miss_cost_usd_total` | counter | USD | `model`, `cause` | Extra cost of those misses. | - -### 5.5 Proxy health - -| Metric | Type | Unit | Labels | Meaning | -|---|---|---|---|---| -| `cg_proxy_info` | gauge | 1 | `version`, `preset` | Always 1. Shows build and config. | -| `cg_proxy_component_duration_seconds` | histogram | s | `component` | Time in each component. | -| `cg_proxy_component_failures_total` | counter | 1 | `component`, `reason` | A component error or panic that reverted. | -| `cg_proxy_fail_open_total` | counter | 1 | `reason` | Requests forwarded unchanged. | -| `cg_proxy_export_failures_total` | counter | 1 | `reason` | Failed pushes to the collector. The next push carries the totals, so a failure loses no count. | - -### 5.6 Retired pull series - -The pull series go away with `/metrics` (R22). This table shows what replaces each one, so that dashboards and alerts can be moved. - -| Retired pull series | Replaced by | -|---|---| -| `cg_requests_total` | `cg_inference_requests_total` | -| `cg_saved_usd` (gauge, month to date) | `cg_savings_saved_usd_total` (counter) | -| `cg_net_saved_usd` (gauge) | Query: `saved - added`. | -| `cg_cache_hit_ratio` | Query on the token counters. | -| `cg_tenant_*` (month to date) | Queries on the counters, over any time range. | -| `cg_metrics_age_seconds`, `cg_metrics_render_seconds` | None. They described the pull exporter. | - -### 5.7 Labels the collector adds - -| Label | Source | Values | -|---|---|---| -| `proxy` | collector config | `cortex`, `context-guru` | -| `machine` | resource attribute `service.instance.id`, the install ID (R43) | One per install. Bounded by the fleet size. | -| `preset` | resource attribute | A few values. A change of preset starts a new series, so a split by `preset` follows the change (8). | -| `service_version` | resource attribute | A few values. | - -`machine` has as many values as there are machines. This is acceptable for tens of machines. Do not use it for thousands. - -### 5.8 Cardinality budget - -Estimate series per machine. A histogram has 12 buckets, plus a sum and a count: 14 series per label set (11 bounds in section 12, plus `+Inf`). - -| Metric group | Label sets | Series | -|---|---|---| -| Savings counters (4) by `kind` x `tier` x `component` | 3 x 4 x 30 = 360 each, worst case. About 1 in 4 is used. | about 360 | -| Cache effects | 5 x 3 x 2 = 30 | 30 | -| Inference counters (2) by `model` x `provider` x `tier` | about 10 x 1 x 5 = 50 each | about 100 | -| Request and response bytes | 10 x 2 stages x 2 = 40 | 40 | -| `duration_seconds` | `model` 10 x `stream` 2 = 20 label sets x 14 | 280 | -| `ttfb_seconds` | 10 x 14 | 140 | -| `added_seconds` | 10 x 14 | 140 | -| `component_duration_seconds` | 30 x 14 | 420 | -| Keep-alive and cache health | about 10 models x 4 outcomes + 7 causes x 2 | about 150 | -| Proxy health and failures | about 100 | about 100 | -| **Total** | | **about 1,700** (the 40 for bytes is inside the rounding) | - -The total sits close to the target, so the target is a real limit and not a safe margin. If the count is too high, drop `model` from `added_seconds` and `ttfb_seconds`, or use fewer buckets for the component histogram. - -Target: **under 2,000 active series per machine.** For 50 machines this is 100,000 series. Check this limit in a test (R42 below). - -- R42. A test counts the label sets a typical session creates. It fails if the count exceeds the budget. - -## 6. Dashboards - -All dashboards use the central Prometheus. They are Grafana JSON files in `deploy/grafana/dashboards/`. Each panel states its query. Use `$__rate_interval`. - -Filters on every dashboard: `proxy`, `machine`, `model`, time range. - -### 6.1 Fleet overview - -Question: is the fleet healthy, and do we save money? - -| Panel | Type | Query idea | -|---|---|---| -| Machines reporting | stat | `count(count by (machine) (cg_proxy_info))`. Use `cg_proxy_info`: it is always 1, so an idle machine still shows. A request counter has no series until the first request. | -| Requests per minute | time series | `sum(rate(cg_inference_requests_total[...]))` by `proxy` | -| Error rate | stat | errors / requests | -| Spend today | stat | `increase(cg_inference_cost_usd_total[1d])` | -| Gross saved today | stat | `increase(cg_savings_saved_usd_total[1d])` | -| Added cost today | stat | `increase(cg_savings_added_usd_total[1d])` | -| **Net saved today** | stat | saved - added | -| Saving rate | stat | net / baseline | -| Net saved per day | bar | `increase(...[1d])` for 30 days | - -### 6.2 Savings - -Question: where does the saving come from, and where does it leak? - -| Panel | Type | Query idea | -|---|---|---| -| Saved vs added | time series | both counters, stacked | -| Net by kind | stacked bars | `kind` = rewrite, spend, setting | -| Net by component | bar gauge, sorted | top 15 components | -| Components with negative net | table | `component` where saved - added < 0 | -| Saving by tier | pie or stacked | which tier the saved tokens belong to | -| Cache effects | stacked bars | `break`, `rescue`, `split`, `ttl_premium`, `ttl_rescue` | -| Break cost by cause | bar | `effect="break"` by `cause` | -| Estimated share | stat | `estimated_total / priced_total` | -| Unpriced requests | time series | by `reason` | - -### 6.3 Cache and keep-alive - -Question: does the cache pay, and do pings pay? - -| Panel | Type | Query idea | -|---|---|---| -| Cache read ratio | time series | by `model` | -| Misses by cause | stacked bars | `cg_cache_misses_total` by `cause` | -| Miss cost by cause | bar | `cg_cache_miss_cost_usd_total` | -| Ping outcomes | stacked bars | by `outcome` | -| Ping cost vs rescue | time series | cost, rescue and net | -| Pings that paid | stat | `rescue_usd / cost_usd` | -| Shared-warm tokens | time series | credit not given (#853) | - -### 6.4 Inference - -Question: how does the traffic behave? - -| Panel | Type | Query idea | -|---|---|---| -| Requests by model | time series | `rate` by `model` | -| Latency p50, p95, p99 | time series | `histogram_quantile` on `duration_seconds` | -| Added latency p95 | time series | `added_seconds` | -| TTFB p95 for streams | time series | `ttfb_seconds` | -| Errors by status class and source | stacked bars | | -| Tokens by tier | stacked bars | `cg_inference_tokens_total` | -| Cost by model | stacked bars | | -| Cost per 1,000 requests | time series | derived | - -### 6.5 Proxy health - -Question: is context-guru itself working? - -| Panel | Type | Query idea | -|---|---|---| -| Proxy versions | table | `cg_proxy_info` | -| Component failures | time series | by `component` | -| Fail-open count | time series | by `reason` | -| Component time p95 | time series | `component_duration_seconds` | -| Export failures | time series | by `reason` | -| Machines silent | table | `last_over_time(cg_proxy_info[1d]) unless cg_proxy_info`. Prometheus writes no stale marker for pushed data, so use a query that compares with the last day. | -| Check failures | time series | `cg_savings_check_failures_total` by `check` | - -### 6.6 Machine comparison - -Question: which machines or configs save the most? - -| Panel | Type | Query idea | -|---|---|---| -| Net saved by machine | bar gauge | sorted | -| Saving rate by machine | table | net / baseline | -| Saving rate by preset | table | by the `preset` label | -| Saving rate by proxy | bar | `cortex` vs `context-guru` | - -## 7. Aggregations - -Recording rules make dashboard queries fast. Put them in one rules file. Name them `level:metric:operation`. - -| Rule | Expression | -|---|---| -| `fleet:cg_savings_usd_net:increase1d` | `sum(increase(cg_savings_saved_usd_total[1d])) - sum(increase(cg_savings_added_usd_total[1d]))` | -| `proxy:cg_savings_usd_net:rate5m` | same, `sum by (proxy)` | -| `machine:cg_savings_usd_net:rate5m` | same, `sum by (machine)` | -| `model:cg_savings_usd_net:rate5m` | same, `sum by (model)` if `model` is kept on savings. **Open.** | -| `component:cg_savings_usd_net:rate5m` | `sum by (component, kind)` | -| `fleet:cg_saving_rate:ratio1d` | net / baseline | -| `fleet:cg_cache_read_ratio:ratio5m` | cache_read / all input tokens | -| `fleet:cg_inference_error_ratio:ratio5m` | errors / requests | -| `fleet:cg_keepalive_payoff:ratio1d` | rescue_usd / cost_usd | - -Roll-up levels and where each one lives: - -| Level | Where | How | -|---|---|---| -| Fleet, proxy, machine, model, kind, tier, component | Prometheus | Labels and recording rules. | -| User | Prometheus if R32 allows. Otherwise the ledger. | Label `user`. | -| Session, thread, request | Ledger (event store) | All IDs stored with the record. | - -All values are sums (tokens and money), so they add up at every level. Ratios are always computed from sums, never averaged. - -## 8. Analysis - -Questions the data must answer. Each has a query or a drill-down. - -| # | Question | Method | -|---|---|---| -| A1 | Does context-guru save money net of its cost? | `saved - added` over a week. `added` includes the compaction model's own calls (R45). Must be above 0. | -| A2 | Which components lose money? | Net by component below 0 for 7 days. Candidate to turn off. This is #670. | -| A3 | How much do cache breaks cost compared with the savings? | Break `added` cost / `saved` cost. | -| A4 | Is keep-alive worth it? | Payoff ratio above 1 by model and by idle time. | -| A5 | Which preset saves most? | Saving rate grouped by `preset`. Needs enough machines per preset. | -| A6 | Do the two proxies differ? | Compare `proxy` values on the same metric names (R40). | -| A7 | How much of the figure is an estimate? | `estimated_total / priced_total`. If above 20 percent, treat the headline as a range. | -| A8 | Why do requests miss the cache? | Misses by cause, and cost by cause. | -| A9 | Does a model change break the cache often? | `cause="model_changed"` count and cost. | -| A10 | Is the saving real, or does our baseline overstate it? | Compare `baseline - cost` with the provider's own cache saving. Review monthly. | -| A11 | Does the proxy dashboard agree with the central metrics? | Compare the dashboard totals of one machine with the central totals for that `machine` over one week. Differences must be explained (retention, estimates, a push that was off). | -| A12 | What did one session save? | Drill down: ledger, by session ID. Not a Prometheus query. | - -### Alerts - -Alerts go to Alertmanager. Thresholds are proposals. - -| Alert | Condition | Severity | -|---|---|---| -| Net saving negative | Fleet net below 0 for 24 hours | warning | -| Component losing money | A component net below -$5 over 7 days | info | -| Many unpriced requests | Unpriced above 10 percent for 1 hour | warning | -| Check failures | Any `sum` or `conservation` failure | warning | -| Export failures | `export_failures_total` rising for 1 hour | warning | -| Machine silent | `cg_proxy_info` missing for 15 minutes | info | -| Error rate | Above 5 percent for 10 minutes | critical | -| Cache read ratio low | Below 0.5 for 1 hour on a model with traffic | info | - -## 9. Delivery plan - -1. **Hook.** Add `Sink`, `Result.Totals()`, `ByKind()`, `Failed()` to the `accountant`. Test them. No OTel. -2. **Shared module.** Create `accountant/metrics` with the savings set (section 5.2). Test with an in-memory OTel reader. -3. **Proxy wiring.** Call the sink after each response. Add the inference set (5.1) and the health set (5.5). Add the settings for the collector URL, the token and the interval, and the switch that turns the push off (R19). -4. **Keep-alive and cache sets** (5.3, 5.4). -5. **Collector and Prometheus config.** Add the collector config, the remote-write receiver flag, and the recording rules file. -6. **Dashboards and alerts** (sections 6 and 8). -7. **Retire the pull path** (R22). Remove `/metrics` (`proxy/promexport.go`, `dash/metrics_export.go`) and the local Prometheus and Grafana install. Move or delete the dashboards and alert rules in `deploy/grafana/`. Do this after the central dashboards (item 6) work. -8. **Docs.** Convention file in `accountant/docs/`, row in `docs/data-dependence.md` (the "Metrics (`/stats`, `/metrics`)" row changes), a section in `docs/hosted.md` (the Grafana section changes). - -Version one: items 1 to 3, and the savings dashboard (6.2). Tokens and money both. **Open:** confirm that tokens-only is not the first step. - -## 10. Test plan - -| Test | What it checks | -|---|---| -| Same numbers everywhere | The same priced result gives the same totals in the shared metrics, in the local ledger and in any proxy-only metric (R25, R26). | -| Totals add up | The split by kind adds up to the total. The total equals the sum of the parts (R26). | -| No duplicate code | The cg proxy and the Cortex proxy report the savings metrics through the same shared code. A check fails if either proxy defines a common metric itself (R26, R27). | -| Money at request time | A rate change does not change counters already recorded. | -| Saved and added | A request with a break adds to `added`. Net can be negative. | -| Unpriced request | Adds nothing to `saved` or `added`. Adds 1 to `unpriced_total`. | -| Label rule | No forbidden label (R31) appears on any metric. | -| Cardinality | Series per machine stay under the budget (R42). | -| Many machines | Several test proxies push at once. The central server shows each `machine` and the fleet sum equals the sum of the parts (R1, R4). | -| No inbound connection | A proxy behind a blocked inbound port still reports (R1). | -| Manager view has no content | The central series carry no session, thread, request or content value (R2, R7). | -| Local view without the push | With the push off, or the collector down, the proxy dashboard shows the same figures (R8). | -| Two proxies | A cg proxy and a Cortex proxy report the same common metric. One panel shows both, split by `proxy` (R10, R13). | -| Third proxy | A test proxy with a new `proxy` value appears with no change to the collector or Prometheus (R11). | -| Fail open | A collector that hangs or refuses does not delay a request or change a response. | -| Async push | With a collector that never answers, the time of a request and its answer is the same as with the push turned off, within test noise (R16). | -| Record is cheap | Recording one metric takes a fixed small time and makes no network call. A benchmark in the PR reports the time (R16). | -| Export failure | A failed push is counted. The next successful push carries the totals, so a counter does not lose counts. | -| Machine ID | The `machine` label is the install ID. It holds no host name (R43). | -| Pull path gone | After the retirement, `/metrics` is not served. The proxy dashboard shows the same figures as before (R22, R8). | -| Default URL | With the default config, the proxy pushes to the shipped URL with no other setting (R19). | -| No URL | With an empty URL, the proxy makes no outgoing connection, starts normally and logs one line (R19). | -| Turned off | With the push turned off, the proxy makes no outgoing connection for metrics (R19). | -| Agreement | For one session, the sink totals equal the dashboard `request_components` totals. | -| End to end | Proxy to a local collector to Prometheus. The expected names and labels appear. | - -## 11. Open questions - -1. Where does `accountant/metrics` live: in the `accountant` repo, or in its own repo? (R27, section 4.1) -2. How many users? Is `user` a label? (R32) -3. Do both proxies store full per-request records? If not, the ledger is the larger work item. -4. Does `Result` carry meta and usage already? It decides the shape of the hook (section 4.1). Does the `accountant` see the compaction model's own calls? If not, R45 needs the proxy to report them as a `spend` change. -5. Version one: tokens and money, or tokens only? -6. Who issues the access tokens, and who is on the read access list? (R44) -7. Is the `component` label small enough to keep? Is `model` kept on savings metrics? (5.2, section 7) -8. What happens to the dashboards and alert rules in `deploy/grafana/`? Port them to the central Grafana, or delete them. The host and logs dashboards use the node exporter and Loki, not `/metrics`. Do they stay? (R22, 5.6) -9. Which retention does the central Prometheus use? 30 days is enough for the dashboards above. Long-term trends need more. -10. How many machines must the fleet view support? (R5) -11. What local view and store does the Cortex proxy have? (R14) -12. Who are the managers, and may they see a split by user or team? A `team` label would need a rule like R32. - -## 12. Technical notes - -- Prometheus needs `--web.enable-remote-write-receiver`. -- The OTel to Prometheus conversion adds `_total` to counters and may add the unit as a suffix. It changes dots to underscores. Check the final names. -- Histogram buckets: use explicit bounds for latency. Suggested seconds: 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120. -- A machine that is offline sends its totals when it is back. The totals are correct, but the time of the increase is the time of the push, not the time of the events. Time detail stays in the dashboard store. Prometheus rejects out-of-order samples unless `out_of_order_time_window` is set. -- The OTel SDK aggregates in memory and keeps no queue of points, so a failed push loses no count. -- Counters restart at 0 when a process restarts. Use `rate()` and `increase()`, which handle this. -- Do not use the Pushgateway. -- Related existing code: `proxy/promexport.go` (pull exporter, to retire), `dash/metrics_export.go` (to retire), `deploy/grafana/` (to move or delete), `proxy/accountant_*.go` (the accountant wiring), `dash/accountant_price.go` (pricing). From 6df3dfd4624cf2e7054124b83dc5feadd36117d6 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sun, 8 Nov 2026 12:11:49 +0200 Subject: [PATCH 57/58] fix(accountant): Report the cacheinject marks so the accountant prices them (#938) cacheinject adds cache_control marks to messages the client did not mark. Claude Code's assistant messages carry tool_use, which bifrost cannot round-trip, so the mark is written at its path on the original bytes (metadata-only write, #32). That path never set `written`, so opsFromRun dropped the change that carries the mark. The forwarded body grew by 37 bytes per mark with ops=0, the sum check failed, and the event fell back to ops_dropped and booked no saving. Set `written` on that path, as the other two edit paths do. Tests: - apply: the reported operations explain every byte the marks added, for 2, 3 and 4 turns, and on an assistant tool_use message. - proxy: four end-to-end cases (alone, without the tail breakpoint, with cachesplit, with a text rewrite) record no failed check and no fallback. Without the change the new tests fail with the signature from the issue: check=sum ops=0 bodies=-74 bytes, fell back: ops_dropped. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- apply/apply.go | 3 + apply/cacheinject_ops_test.go | 120 +++++++++++++++++++++++++++++ proxy/accountant_fallbacks_test.go | 55 +++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 apply/cacheinject_ops_test.go diff --git a/apply/apply.go b/apply/apply.go index 0f6ee85c..25679c7d 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -977,6 +977,9 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o if nb, ok := applyMetaWrites(cur, len(chat.Input[i].Content.ContentBlocks), w); ok { edits[bi] = nb changed = true + // The mark is on the wire, so the change that carries it is real. Without this the + // operation is dropped and the accountant's sum check fails on the added bytes (#938). + written[i] = true continue } } diff --git a/apply/cacheinject_ops_test.go b/apply/cacheinject_ops_test.go new file mode 100644 index 00000000..ec324ec3 --- /dev/null +++ b/apply/cacheinject_ops_test.go @@ -0,0 +1,120 @@ +package apply_test + +import ( + "context" + "strings" + "testing" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/tidwall/gjson" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/store" +) + +// cacheinjectBody is a Claude Code shaped request with n tool turns. The client sends two marks of its +// own, one on the system block and one on the newest message, as Claude Code does. +func cacheinjectBody(n int) []byte { + var b strings.Builder + b.WriteString(`{"model":"claude-sonnet-4-5","system":[{"type":"text","text":"system prompt","cache_control":{"type":"ephemeral"}}],"messages":[`) + b.WriteString(`{"role":"user","content":[{"type":"text","text":"go"}]}`) + for i := 0; i < n; i++ { + id := string(rune('a' + i)) + b.WriteString(`,{"role":"assistant","content":[{"type":"tool_use","id":"` + id + `","name":"Read","input":{}}]}`) + b.WriteString(`,{"role":"user","content":[{"type":"tool_result","tool_use_id":"` + id + `","content":"` + + strings.Repeat("output line number "+id+"\\n", 30) + `"}`) + if i == n-1 { + b.WriteString(`,{"type":"text","text":"next","cache_control":{"type":"ephemeral"}}`) + } + b.WriteString(`]}`) + } + b.WriteString(`]}`) + return []byte(b.String()) +} + +// #938. cacheinject adds cache_control marks to messages the client did not mark. The marks lengthen the +// forwarded body, and the accountant's sum check compares the bytes the reported operations explain with the +// bytes that went upstream. A mark nobody reports leaves ops=0 for bodies=+37 per mark, the check fails and the +// event books no saving. +func TestCacheinjectMarksAreExplainedByReportedOperations(t *testing.T) { + for _, turns := range []int{2, 3, 4} { + body := cacheinjectBody(turns) + p, err := pipe(t, "pipeline: [cacheinject]\n").Build(nil) + if err != nil { + t.Fatal(err) + } + res := apply.BodyOpts(context.Background(), p, store.NewMemory(store.Options{}), apply.Opts{ + Provider: bschemas.Anthropic, Body: body, + }) + if !res.Changed { + t.Fatalf("%d turns: cacheinject changed nothing; the test body has no mark to add", turns) + } + grew := len(res.Body) - len(body) + if grew <= 0 { + t.Fatalf("%d turns: forwarded body grew by %d bytes, want marks added", turns, grew) + } + if res.Trace.OpsDropped != 0 { + t.Fatalf("%d turns: %d operations dropped", turns, res.Trace.OpsDropped) + } + explained := 0 + for _, o := range res.Trace.Ops { + if o.DryRun { + continue + } + explained += o.SizeAfter.Bytes - o.SizeBefore.Bytes + } + if explained != grew { + t.Errorf("%d turns: operations explain %d bytes, the forwarded body grew by %d (ops %d)", + turns, explained, grew, len(res.Trace.Ops)) + } + // Each reported operation must sit on a message that carries a new mark. + for _, o := range res.Trace.Ops { + if gjson.GetBytes(res.Body, "messages."+itoa(o.At.Index)+".content.#(cache_control)").Exists() == false && + !strings.Contains(gjson.GetBytes(res.Body, "messages."+itoa(o.At.Index)).Raw, "cache_control") { + t.Errorf("%d turns: op %+v is on a message with no mark", turns, o) + } + } + } +} + +func itoa(i int) string { + if i == 0 { + return "0" + } + var d []byte + for ; i > 0; i /= 10 { + d = append([]byte{byte('0' + i%10)}, d...) + } + return string(d) +} + +// #938 on the message #32 names as the real target: an assistant message with a tool_use block, which +// bifrost cannot round-trip. The mark is written at its path on the original bytes and must be reported, +// and the provider fields of the block must still come through unchanged. +func TestCacheinjectMarkOnAToolUseMessageIsReported(t *testing.T) { + body := []byte(`{"model":"claude-x","messages":[ + {"role":"user","content":"run the tool"}, + {"role":"assistant","content":[{"type":"text","text":"on it"},{"type":"tool_use","id":"toolu_abc","name":"Bash","input":{"command":"ls -la"}}]} + ]}`) + p, err := pipe(t, "pipeline: [cacheinject]\n").Build(nil) + if err != nil { + t.Fatal(err) + } + res := apply.BodyOpts(context.Background(), p, store.NewMemory(store.Options{}), apply.Opts{ + Provider: bschemas.Anthropic, Body: body, + }) + if !res.Changed || !gjson.GetBytes(res.Body, "messages.1.content.1.cache_control").Exists() { + t.Fatalf("precondition: the mark must reach the wire: %s", res.Body) + } + if res.Trace.OpsDropped != 0 || len(res.Trace.Ops) != 1 { + t.Fatalf("ops = %+v dropped = %d, want the one mark reported", res.Trace.Ops, res.Trace.OpsDropped) + } + op := res.Trace.Ops[0] + if op.At.Index != 1 || op.SizeAfter.Bytes-op.SizeBefore.Bytes != len(res.Body)-len(body) { + t.Fatalf("op %+v, body grew by %d bytes", op, len(res.Body)-len(body)) + } + if blk := gjson.GetBytes(res.Body, "messages.1.content.1"); blk.Get("id").String() != "toolu_abc" || + blk.Get("input.command").String() != "ls -la" { + t.Fatalf("tool_use provider fields changed: %s", blk.Raw) + } +} diff --git a/proxy/accountant_fallbacks_test.go b/proxy/accountant_fallbacks_test.go index 301aaaaa..8e2cdc41 100644 --- a/proxy/accountant_fallbacks_test.go +++ b/proxy/accountant_fallbacks_test.go @@ -80,6 +80,61 @@ func TestTailBreakpointReportsItsBytes(t *testing.T) { } } +// cacheinjectConversation is a Claude Code shaped body with n tool turns. The client marks the system +// block and its newest message and nothing else, so cacheinject has marks of its own to add. +func cacheinjectConversation(n int) []byte { + msgs := []any{map[string]any{"role": "user", "content": []any{map[string]any{"type": "text", "text": "fix it"}}}} + for k := 0; k < n; k++ { + id := "tu_" + string(rune('a'+k)) + msgs = append(msgs, + map[string]any{"role": "assistant", "content": []any{map[string]any{"type": "tool_use", "id": id, "name": "bash", "input": map[string]any{"command": "ls"}}}}, + map[string]any{"role": "user", "content": []any{map[string]any{"type": "tool_result", "tool_use_id": id, "content": bigToolOutput()}}}) + } + last := msgs[len(msgs)-1].(map[string]any)["content"].([]any) + last[0].(map[string]any)["cache_control"] = map[string]any{"type": "ephemeral"} + b, _ := json.Marshal(map[string]any{"model": "aws/claude-sonnet-5", "max_tokens": 64, "messages": msgs, + "system": []any{map[string]any{"type": "text", "text": "s1", "cache_control": map[string]any{"type": "ephemeral"}}}}) + return b +} + +// cacheinject adds cache_control marks to messages the client did not mark. The marks lengthen the forwarded +// body. The metadata-only write path put them on the wire without recording that the message was written, so +// the operation that carries the mark was dropped: ops=0 for bodies=+37 bytes, the sum check failed on every +// request, and the event booked no saving (#938). +func TestCacheinjectMarksAreReportedAndPriced(t *testing.T) { + for name, yaml := range map[string]string{ + "cacheinject alone": "pipeline: [cacheinject]\n", + "without the tail breakpoint": "pipeline: [cacheinject]\ncache:\n tail_breakpoint: false\n", + "with cachesplit": "pipeline: [cachesplit, cacheinject]\n", + // A text rewrite and a mark can land on one message: the two operations share its wire bytes. + "with a text rewrite": "pipeline: [mask, cacheinject]\ncomponents:\n mask:\n keep_recent: 1\n min_tokens: 50\n", + } { + t.Run(name, func(t *testing.T) { + up := billedUpstream(t) + defer up.Close() + h, rec, tap := accHandlerYAML(t, up.URL, yaml) + for n := 2; n <= 4; n++ { + accPostBody(t, h, "sess-ci", cacheinjectConversation(n)) + } + waitForRows(t, rec, 3) + evs := tap.all() + for i, e := range evs { + if res, ok := e.AccountantResult(); ok && i > 0 { + for _, c := range res.Checks { + t.Errorf("request %d: check failed: %s %s", i+1, c.Check, c.Detail) + } + } + if i == 0 { + continue // the first request of a thread is held out as an estimate + } + if fb := e.AccountantFallback(); fb != "" { + t.Errorf("request %d fell back: %s", i+1, fb) + } + } + }) + } +} + // A small conversation that drops a large tool list: the removed declarations are many times the // size of the messages. The plausibility bound reads message tokens, so it dropped the whole result // with implausible_size (#907). From f5cf361b91d2f67320d8d461089cdc538494f2f7 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Sun, 8 Nov 2026 12:11:50 +0200 Subject: [PATCH 58/58] feat(accountant): Say why an event is unpriced in the acc_fallback line (#939) reason=unpriced covered three causes and the log line named none. Add Event.UnpricedWhy and log it as `why`, as no_result already does: - no_usage: the request has no billed token counts - no_price: the model has no price - opID:reason (or reason alone for a refusal of the whole request, such as ratio_range): the first operation or effect the library could not price The cause is in the log line only. No stored column and no API field carries it, and nothing that is booked changes. Update docs/dashboard.md for the new field, and for the cacheinject mark that is no longer in the ops_dropped list. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- dash/accountant_price.go | 24 +++++++++++++ dash/accountant_switch_test.go | 62 +++++++++++++++++++++++++++++++-- dash/event.go | 2 ++ docs/dashboard.md | 13 +++++-- proxy/accountant_shadow.go | 6 +++- proxy/accountant_shadow_test.go | 5 +++ 6 files changed, 105 insertions(+), 7 deletions(-) diff --git a/dash/accountant_price.go b/dash/accountant_price.go index c7c05c04..96afd489 100644 --- a/dash/accountant_price.go +++ b/dash/accountant_price.go @@ -219,6 +219,23 @@ func (e *Event) AccountantEstimated() bool { return e.PriceEstimated } // An event with no usage or no price books no saving too, and its reason is FallbackUnpriced. func (e *Event) AccountantFallback() string { return e.accFallback } +// Causes behind FallbackUnpriced. They are for the log line only: no stored column and no API field +// carries them, so a new cause here changes nothing a client can read. +const ( + UnpricedNoUsage = "no_usage" // the request has no billed token counts (accounting is not complete) + UnpricedNoPrice = "no_price" // the model has no price +) + +// UnpricedWhy names the cause when AccountantFallback is FallbackUnpriced, and is "" for any other +// reason. It is UnpricedNoUsage or UnpricedNoPrice when the event itself cannot be priced, or the +// library's "opID:reason" for the first operation or effect it could not price. +func (e *Event) UnpricedWhy() string { + if e.accFallback != FallbackUnpriced { + return "" + } + return e.accUnpricedWhy +} + // PriceMode is Price, then an override of each component's SavedUSD, of BaselineCostUSD, of // KeepAliveSavedUSD and of CachesplitSavedUSD from the library's figure. CacheSavedUSD (the // provider's own cache, a measurement), CostUSD and CGLLMCostUSD always stay as Price set them. @@ -231,8 +248,13 @@ func (e *Event) AccountantFallback() string { return e.accFallback } // AccountantFallback says why: a figure nothing measured is not shown. func (e *Event) PriceMode(p modelinfo.Price, accountingComplete bool) { e.accFallback, e.PriceEstimated, e.AccountantPriced, e.EstimatedSavedUSD, e.EstimateHeld = "", false, false, 0, false + e.accUnpricedWhy = "" if !e.priceCost(p, accountingComplete) { e.accFallback = FallbackUnpriced + e.accUnpricedWhy = UnpricedNoUsage + if accountingComplete { + e.accUnpricedWhy = UnpricedNoPrice // usage is complete, so the zero price is the cause + } return } e.accFallback = e.accOverride(p) @@ -268,6 +290,8 @@ func (e *Event) accOverride(p modelinfo.Price) string { estimated := false for _, u := range fig.Unpriced { if !strings.HasSuffix(u, ":"+accountant.ReasonPenaltyUnknown) { + // "opID:reason". A request-level refusal has no operation, so its id is empty: drop the colon. + e.accUnpricedWhy = strings.TrimPrefix(u, ":") return FallbackUnpriced } estimated = true diff --git a/dash/accountant_switch_test.go b/dash/accountant_switch_test.go index 1927a7d1..b9e53318 100644 --- a/dash/accountant_switch_test.go +++ b/dash/accountant_switch_test.go @@ -143,7 +143,8 @@ func TestPriceMode_FallsBackToNoSaving(t *testing.T) { if got := e.AccountantFallback(); got != c.reason { t.Errorf("%s: fallback reason %q, want %q", name, got, c.reason) } - e.accFallback = "" // the copy carries no reason; every other field must match + e.accFallback = "" // the copy carries no reason; every other field must match + e.accUnpricedWhy = "" // the cause of an unpriced reason is part of the reason, not of the figures // A fallback row is marked, so the read views do not value it with the legacy rule. for i := range e.Components { if e.Components[i].PricedBy != PricedByNone { @@ -308,15 +309,70 @@ func TestPriceMode_AnUnpricedEventSaysWhyItBooksNoSaving(t *testing.T) { for name, tc := range map[string]struct { price modelinfo.Price complete bool + why string }{ - "no usage": {accTestPrice, false}, - "no price": {modelinfo.Price{}, true}, + "no usage": {accTestPrice, false, UnpricedNoUsage}, + "no price": {modelinfo.Price{}, true, UnpricedNoPrice}, } { e := switchEvent() e.PriceMode(tc.price, tc.complete) if fb := e.AccountantFallback(); fb != FallbackUnpriced { t.Errorf("%s: fallback = %q, want %q", name, fb, FallbackUnpriced) } + if w := e.UnpricedWhy(); w != tc.why { + t.Errorf("%s: UnpricedWhy = %q, want %q (#939)", name, w, tc.why) + } + } +} + +// #939. An operation the library could not price is the third cause of "unpriced", and it names itself. +// penalty_unknown is not one of them: it prices the event as an estimate and leaves no reason. +func TestPriceMode_UnpricedWhyNamesTheOperationTheLibraryCouldNotPrice(t *testing.T) { + e := switchEvent() + e.accRes.Unpriced = []accountant.Unpriced{ + {OpID: "op1", Reason: accountant.ReasonPenaltyUnknown}, + {OpID: "op2", Reason: "no_rate"}, + } + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb != FallbackUnpriced { + t.Fatalf("fallback = %q, want %q", fb, FallbackUnpriced) + } + if w := e.UnpricedWhy(); w != "op2:no_rate" { + t.Fatalf("UnpricedWhy = %q, want the operation that has no price", w) + } + + est := switchEvent() + est.accRes.Unpriced = []accountant.Unpriced{{OpID: "op1", Reason: accountant.ReasonPenaltyUnknown}} + est.PriceMode(accTestPrice, true) + if w := est.UnpricedWhy(); w != "" { + t.Fatalf("an estimated event has UnpricedWhy = %q, want empty", w) + } +} + +// UnpricedWhy is for the unpriced reason only, and a second PriceMode call on the same event clears it. +func TestUnpricedWhyIsEmptyForEveryOtherReason(t *testing.T) { + e := switchEvent() + e.PriceMode(modelinfo.Price{}, true) + if e.UnpricedWhy() == "" { + t.Fatal("setup: the no-price event has no cause") + } + e.PriceMode(accTestPrice, true) + if fb := e.AccountantFallback(); fb == FallbackUnpriced { + t.Fatalf("setup: the repriced event still falls back as unpriced (%q)", fb) + } + if w := e.UnpricedWhy(); w != "" { + t.Fatalf("after a repricing UnpricedWhy = %q, want empty", w) + } + + // A different reason leaves it empty too: the cause belongs to unpriced alone. + o := switchEvent() + o.SetAccountantContext(1, false) // a dropped operation: ops_dropped + o.PriceMode(accTestPrice, true) + if fb := o.AccountantFallback(); fb != FallbackOpsDropped { + t.Fatalf("setup: fallback = %q, want %q", fb, FallbackOpsDropped) + } + if w := o.UnpricedWhy(); w != "" { + t.Fatalf("ops_dropped event has UnpricedWhy = %q, want empty", w) } } diff --git a/dash/event.go b/dash/event.go index 0aa7de59..928fc886 100644 --- a/dash/event.go +++ b/dash/event.go @@ -129,6 +129,8 @@ type Event struct { accUnsupportedWire bool // accFallback is why PriceMode booked no saving (see AccountantFallback). accFallback string + // accUnpricedWhy names which cause lies behind accFallback == FallbackUnpriced (see UnpricedWhy). + accUnpricedWhy string ID int64 `json:"id"` TS int64 `json:"ts"` // epoch ms diff --git a/docs/dashboard.md b/docs/dashboard.md index ce0d63ed..300f36b2 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -139,6 +139,11 @@ measured is not shown. For `no_result` the line also carries `why`: `no_record` (no session, or the pipeline did not run), `no_usage`, `upstream_rounds=N` (an expand loop) or `compute_panicked`. +For `unpriced` the line carries `why` too: `no_usage` (the request has no billed token counts), `no_price` +(the model has no price), or the library's own cause for the first operation or effect it could not price, +as `opID:reason` (or `reason` alone when the refusal is for the whole request, as `ratio_range` is). The +cause is in the log line only. No stored column and no API field carries it. + `unsupported_wire` is the OpenAI Responses API. It keeps the conversation in `input`, and the proxy hashes `messages` and reports no operations there, so **every saving on that wire is $0**. Chat Completions and the Anthropic Messages API are priced. Until Responses is read, a Responses account sees no saving and the @@ -208,9 +213,11 @@ Limits: event books no saving. A request with no usage, or a model with no price, books no saving and logs `unpriced`. - A change the pipeline made that no operation can describe drops every operation of the request - (`ops_dropped`): a message whose only change is a `cache_control` mark, a cache anchor written - after a count-changing splice, and a splice that moves the later messages in a request the equal-count - writeback handled. Such a request books no saving. + (`ops_dropped`): a cache anchor written after a count-changing splice, and a splice that moves the + later messages in a request the equal-count writeback handled. Such a request books no saving. A + message whose only change is an added `cache_control` mark is not in this list: the mark is written at + its path and reported as an operation, so a `cacheinject` mark is priced (#938). It used to be in this + list. - A request whose response called `context_guru_expand` (more than one upstream round) books no saving (`no_result`): the usage is billed on the expanded body, which the library did not see. - The thread state lives in memory. After a restart, and for a thread dropped by the library's cap diff --git a/proxy/accountant_shadow.go b/proxy/accountant_shadow.go index d1b464b8..6e350bd6 100644 --- a/proxy/accountant_shadow.go +++ b/proxy/accountant_shadow.go @@ -12,7 +12,8 @@ import ( // books no saving, // with the reason (dash.Fallback*). // -// why is set for no_result only: it names which of that reason's causes applied (noResultWhy). +// why is set for no_result only: it names which of that reason's causes applied (noResultWhy). For +// unpriced the cause comes from the event itself (dash.Event.UnpricedWhy), so it needs no argument. func logFallback(e *dash.Event, why string) { r, _ := e.AccountantResult() attrs := []any{"session", e.SessionID, "thread", e.ThreadID, @@ -20,6 +21,9 @@ func logFallback(e *dash.Event, why string) { if why != "" && e.AccountantFallback() == dash.FallbackNoResult { attrs = append(attrs, "why", why) } + if w := e.UnpricedWhy(); w != "" { + attrs = append(attrs, "why", w) + } slog.Info("acc_fallback", attrs...) } diff --git a/proxy/accountant_shadow_test.go b/proxy/accountant_shadow_test.go index debd3238..8a17849e 100644 --- a/proxy/accountant_shadow_test.go +++ b/proxy/accountant_shadow_test.go @@ -39,4 +39,9 @@ func TestFallbackReasonIsLogged(t *testing.T) { if !strings.Contains(buf.String(), `"reason":"unpriced"`) { t.Fatalf("acc_fallback lacks the reason:\n%s", buf.String()) } + // #939. unpriced names its cause, as no_result does. This one is the library's request-level refusal, + // so the value is the bare reason with no operation id. + if n := strings.Count(buf.String(), `"why":"ratio_range"`); n != 2 { + t.Fatalf(`"why":"ratio_range" appears %d times, want 2 (one for each event):`+"\n%s", n, buf.String()) + } }