From cd241946dbaaa9888805754133ada1f6b03836eb Mon Sep 17 00:00:00 2001 From: Vijay Date: Wed, 28 Oct 2026 20:58:58 -0400 Subject: [PATCH 1/7] feat: add explicit agent attribution and recorded spend changes Compose bounded exact agent accounting with existing request accounting. Add shared opt-in request-tag transport, project-scoped Langfuse attribution, distinct-suffix coverage, sensitive-label acknowledgement, and frozen-source fixture provenance. Preserve default reports, projections, collector summaries and base accounting. Code authors: Vijay and Codex Signed-off-by: Vijay --- CHANGELOG.md | 10 + docs/AGENT_ATTRIBUTION.md | 74 ++++ docs/NATIVE_EXPORT.md | 18 +- docs/RELEASING.md | 2 +- docs/ROUTING.md | 7 +- examples/litellm-spend/README.md | 8 + examples/litellm-spend/TESTING.md | 8 + examples/litellm-spend/agent-tags.csv | 9 + examples/litellm-spend/agent-tags.csv.license | 5 + examples/litellm-spend/agents.csv | 7 + examples/litellm-spend/agents.csv.license | 5 + .../coverage_runtime_sources.tar.gz | Bin 0 -> 25726 bytes .../coverage_runtime_sources.tar.gz.license | 6 + examples/litellm-spend/export.sql | 11 +- examples/litellm-spend/test-schema.sql | 1 + examples/litellm-spend/test_native_export.py | 59 ++- examples/litellm-spend/test_row_coverage.py | 16 +- examples/litellm-spend/test_spend.py | 4 +- internal/cli/agents.go | 74 ++++ internal/cli/agents_test.go | 127 ++++++ internal/cli/database.go | 7 +- internal/cli/database_test.go | 11 +- internal/cli/routing_labels_test.go | 2 +- internal/cli/run.go | 54 ++- internal/cli/spend.go | 10 +- internal/exportdata/export.go | 1 + internal/langfuse/agents_test.go | 58 +++ internal/langfuse/directory.go | 33 +- internal/postgres/export.go | 2 +- internal/postgres/query.go | 9 +- internal/postgres/schema.go | 5 +- internal/postgres/tags_test.go | 31 ++ internal/spend/agent_spend_test.go | 97 +++++ internal/spend/agents.go | 391 ++++++++++++++++++ internal/spend/agents_test.go | 105 +++++ internal/spend/investigate.go | 44 +- internal/spend/reader.go | 13 +- internal/spend/report.go | 1 + internal/spend/routing.go | 2 +- internal/spend/routing_labels_test.go | 2 + internal/spend/selection.go | 7 +- 41 files changed, 1290 insertions(+), 46 deletions(-) create mode 100644 docs/AGENT_ATTRIBUTION.md create mode 100644 examples/litellm-spend/agent-tags.csv create mode 100644 examples/litellm-spend/agent-tags.csv.license create mode 100644 examples/litellm-spend/agents.csv create mode 100644 examples/litellm-spend/agents.csv.license create mode 100644 examples/litellm-spend/coverage_runtime_sources.tar.gz create mode 100644 examples/litellm-spend/coverage_runtime_sources.tar.gz.license create mode 100644 internal/cli/agents.go create mode 100644 internal/cli/agents_test.go create mode 100644 internal/langfuse/agents_test.go create mode 100644 internal/postgres/tags_test.go create mode 100644 internal/spend/agent_spend_test.go create mode 100644 internal/spend/agents.go create mode 100644 internal/spend/agents_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d38bca..1e9b036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ Notable user-visible changes are recorded here, newest first. ## [Unreleased] +- Add explicit agent attribution from LiteLLM `agent_id` or prefixed request + tags on LiteLLM and Langfuse, with exclusive known/unknown/ambiguous totals, + repeated/empty-tag coverage and an exact 1,024-agent bound. +- Add opt-in request-tag export through the shared psql/native projection, + and an agent table with exact recorded USD, coverage, period shares and + `--rank-by recorded-spend-change`. Default exports and reports are unchanged. +- Require `--acknowledge-sensitive-labels` together with `--show-labels` for + raw agent or routing labels. User/session/end-user/key rankings remain aliased. + See the [agent attribution contract](docs/AGENT_ATTRIBUTION.md). + ## [0.13.1] - 2026-10-27 See the [release notes](docs/releases/v0.13.1.md). diff --git a/docs/AGENT_ATTRIBUTION.md b/docs/AGENT_ATTRIBUTION.md new file mode 100644 index 0000000..31feb2a --- /dev/null +++ b/docs/AGENT_ATTRIBUTION.md @@ -0,0 +1,74 @@ +# Agent Attribution + +Development scope for the next minor release. The released v0.13.1 binary does +not support these options. Build this checkout to run the examples: + +```sh +go build -o bin/fleetdiff ./cmd/fleetdiff +bin/fleetdiff investigate --litellm-spend examples/litellm-spend/agent-tags.csv \ + --before-period 2026-10-07 --after-period 2026-10-08 \ + --group-by agent --agent-source request-tag --agent-tag-prefix 'agent:' \ + --rank-by recorded-spend-change +``` + +The fixture records 620 tokens and $6.20 after the change: known agents account +for 500 tokens/$5, unknown identity for 50/$0.50, and ambiguous identity for +70/$0.70. `agents.csv` is the three-agent fixture without the last two rows. +Agent alpha's tokens increase from 100 to 300. These are synthetic observations. + +## Source Selection + +`--group-by agent` requires an explicit `--agent-source`: + +- `agent_id`: the recorded LiteLLM spend-log ID, on CSV/JSON/JSONL or database. +- `request-tag`: request tags with a required `--agent-tag-prefix`. Supported on + LiteLLM sources and Langfuse exports. The prefix is exact and case-sensitive; + suffixes retain their source spelling. No fallback to another field occurs. + +For LiteLLM database investigation, replace the file option with `--litellm-db`; +the selected tag source enables tag export automatically. Standalone native +export requires `--request-tags`; psql requires `-v request_tags=true`. +See [export limits](NATIVE_EXPORT.md#limits-and-failures) and the adjacent +[optional column grant](NATIVE_EXPORT.md#administrator-setup). + +Zero matching nonblank suffixes means unknown identity. One distinct suffix +means a known agent, even if repeated. Two or more distinct suffixes means +ambiguous identity. Repeated matching tags and empty suffixes have separate +coverage counts; empty suffixes never match an agent. Unknown and ambiguous rows +retain all their accounting in separate buckets. Each analyzed row is counted +once. Langfuse identities remain project-scoped, and its unit stays GENERATION +observations. No request/observation join is inferred between sources. + +## Ranking And Disclosure + +The default agent table orders by signed recorded-token change. +`--rank-by recorded-spend-change` orders by signed change in usable recorded USD, +descending, with first appearance breaking ties. An unavailable cost delta sorts +after available deltas. Before/after USD, delta, cost coverage and each period's +share appear beside tokens. A period with no usable cost rows is unavailable, +not a recorded zero. Known agents outside `--top` are summed separately. + +Shares use usable recorded USD in all analyzed rows in that period, including +unknown, ambiguous and undisplayed agents. Missing/invalid costs remain +unquantified; zero-cost origin remains unknown. Shares are rounded to 12 decimal +places in JSON and two percentage places in text. Costs retain exact source +decimals. Recorded USD is source accounting, not an invoice. Unsupported +LiteLLM call types and partial Langfuse exports keep their existing boundaries. + +Default output uses keyed aliases. To include only the selected agent label, +add both `--show-labels --acknowledge-sensitive-labels`; either alone fails. +Reports identify disclosure and warn to review before sharing. Other tag values, +user/session/end-user/key identities and unrelated routing labels remain hidden. +Recorded IDs and tags can contain personal identifiers; no friendly-name lookup, +arbitrary metadata extraction or automatic agent discovery is performed. + +## Limits + +The agent table is exact and bounded to 1,024 known agents across both periods, +including project-scoped agents. Overflow fails and asks for a narrower prefix +or export; the error quotes the selected prefix. Malformed/non-string tag arrays +or tag data exceeding the existing 8 KiB field bound fail, without raw row data. +An old export without tags remains readable and reports column-absent unknown +identity. The optional `agents` JSON section is versioned independently as +`fleetdiff-agent-attribution/v1`; collector summaries and the base accounting +fingerprint are unchanged. Default export projections remain unchanged. diff --git a/docs/NATIVE_EXPORT.md b/docs/NATIVE_EXPORT.md index c11777f..a157f4e 100644 --- a/docs/NATIVE_EXPORT.md +++ b/docs/NATIVE_EXPORT.md @@ -97,6 +97,14 @@ GRANT SELECT ( ALTER ROLE fleetdiff_reader SET default_transaction_read_only = on; ``` +Only for opt-in request-tag exports, also grant the new column below. +This exports every request tag on a row, not only agent tags; keep the file private. + +```sql +GRANT SELECT (request_tags) + ON TABLE public."LiteLLM_SpendLogs" TO fleetdiff_reader; +``` + Only when daily counters are needed, also grant: ```sql @@ -201,14 +209,20 @@ fleetdiff issues no database writes or schema migrations. The v0.12.0 shared SQL orders spend rows by `"startTime", request_id COLLATE "C"` and daily rows by `date COLLATE "C"`. For the same database snapshot, bounds, -format and detail options, native and v0.12.0 `psql` exports must be -byte-for-byte identical. Sequential exports from a changing live database are +format and detail options, native and matching `psql` exports must be +byte-for-byte identical. Default projections retain the v0.12.0 recipe's bytes; +opt-in request tags require the matching recipe from the same build. +Sequential exports from a changing live database are not the same snapshot. This ordering is new relative to the v0.11.0 recipe: it can change bytes, first-seen aliases and tied report order, not recorded totals or identity hashing. Database sorting can also affect export time. ## Limits And Failures +- `--request-tags` includes every request tag on a row, not only agent tags. + It is opt-in; default exports omit the column. The manual recipe uses + `-v request_tags=true` for the same JSON-array-text projection. + | Native export bound | Value | | --- | --- | | Spend rows | 10,000,000 | diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 0b960cc..d9780ca 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -141,7 +141,7 @@ From an anonymous clone in a fresh directory outside any synced workspace: subset, and absent agents read as unknown. Keep the v0.8.0 import-refactor output goldens and legacy identity-hash tests unchanged; reports without these conditions must remain byte-identical. -11. Run the routing guide with default aliases and with `--show-labels`. Verify +11. Run the routing guide with default aliases and with `--show-labels --acknowledge-sensitive-labels`. Verify public model names, explicit configuration-label warnings, missing serving identity, classifier overhead and raw-identity suppression. Exercise a pinned-writer `openai/` export: price it only when the row provider diff --git a/docs/ROUTING.md b/docs/ROUTING.md index 51c255d..2814238 100644 --- a/docs/ROUTING.md +++ b/docs/ROUTING.md @@ -21,7 +21,7 @@ fleetdiff investigate --litellm-spend ./spend.csv \ ``` Add `--routing-by route-agent` to separate the known agents within each route. -Add `--show-labels` for a local report with the recorded route and agent names. +Add `--show-labels --acknowledge-sensitive-labels` for a local report with the recorded route and agent names. The usual `--group-by` still controls the existing contributor rankings; it does not change the routing table. `--format json` includes the same evidence as an additive `routing_audit` object. Default reports omit this object and section. @@ -96,13 +96,14 @@ other model and deployment names stay aliased, including under `--show-labels`. The same public-model rule applies to `investigate`'s model totals. A catalog match identifies a public name, not independent proof of the serving model. Routes and agents stay aliased by default; `--show-hashes` adds keyed digests. -`--show-labels` adds their recorded labels beside the aliases (JSON: `label`), +`--show-labels --acknowledge-sensitive-labels` adds their recorded labels beside the aliases (JSON: `label`), with terminal controls escaped in text. Reports made with it contain configuration labels: keep them local and review them before sharing. Unknown identities and groups with conflicting raw spellings remain aliased. Distinct source groups resolving to the same public model keep an alias suffix so they remain distinguishable; their accounting groups and hashes are not merged. -Selector arguments are not echoed; errors never expose configuration labels. +Routing selector arguments are not echoed. Agent-tag cardinality errors name +the selected prefix, with terminal controls escaped, to suggest a narrower one. An absent route stays unknown, even if `model_group` names a recovery route. The original route comes only from `metadata.original_model_group`. diff --git a/examples/litellm-spend/README.md b/examples/litellm-spend/README.md index 51976eb..4b174cf 100644 --- a/examples/litellm-spend/README.md +++ b/examples/litellm-spend/README.md @@ -216,6 +216,11 @@ default to zero. No missing-path `NULL` should be interpreted as zero cache use. ### Agent Identity And Rejection Evidence +For tag-based attribution, opt in with `-v request_tags=true` in the same psql +export, or `--request-tags` in native export. Both add the nullable `request_tags` +column as JSON-array text, without expanding rows. The read-only column grant +is documented in [Administrator Setup](../../docs/NATIVE_EXPORT.md#administrator-setup). + The projection also includes the pinned schema's `agent_id` and a SQL-derived `rejection_kind`. Both remain present with `usage_details=false`. Agent identity, like key/team/session identity, is sensitive: keep the export private. No raw @@ -276,6 +281,9 @@ the call types and asks for a supported request-level export. ## Data And Limits +- Opt-in tag export includes every request tag on a row, not only agent tags. + Keep the file private; tags can contain personal identifiers. + - Accepted inputs are request-level SQL projections as headered CSV, a JSON array, or JSONL. LiteLLM Usage dashboard CSV/JSON exports are daily aggregates: the importer recognizes them and requests row-level data. API page envelopes diff --git a/examples/litellm-spend/TESTING.md b/examples/litellm-spend/TESTING.md index d5f477c..027009e 100644 --- a/examples/litellm-spend/TESTING.md +++ b/examples/litellm-spend/TESTING.md @@ -3,6 +3,14 @@ # LiteLLM Spend Export Checks +The captured row-coverage CSVs and `coverage_runtime_expected.json` retain their +original observations and provenance. `coverage_runtime_sources.tar.gz` retains +the source bytes bound by that provenance; tests verify each original digest +without treating later source edits as a new historical run. Current native +export and psql recipe compatibility are exercised separately by the pinned +database checks below. The archive contains source only and is never unpacked +to the filesystem or executed by tests. + ## Documentation Checks Run the standard-library documentation checks without building or connecting to diff --git a/examples/litellm-spend/agent-tags.csv b/examples/litellm-spend/agent-tags.csv new file mode 100644 index 0000000..dc25b48 --- /dev/null +++ b/examples/litellm-spend/agent-tags.csv @@ -0,0 +1,9 @@ +request_id,call_type,startTime,endTime,status,model,prompt_tokens,completion_tokens,total_tokens,spend,agent_id,request_tags +before-a,completion,2026-10-07T12:00:00Z,2026-10-07T12:00:01Z,success,synthetic-model,100,0,100,1,different-id,"[""agent:alpha"",""other:private""]" +before-b,completion,2026-10-07T12:00:00Z,2026-10-07T12:00:01Z,success,synthetic-model,100,0,100,1,different-id,"[""agent:beta""]" +before-c,completion,2026-10-07T12:00:00Z,2026-10-07T12:00:01Z,success,synthetic-model,100,0,100,1,different-id,"[""agent:gamma""]" +after-a,completion,2026-10-08T12:00:00Z,2026-10-08T12:00:01Z,success,synthetic-model,300,0,300,3,different-id,"[""agent:alpha"",""agent:alpha"",""agent:""]" +after-b,completion,2026-10-08T12:00:00Z,2026-10-08T12:00:01Z,success,synthetic-model,100,0,100,1,different-id,"[""agent:beta""]" +after-c,completion,2026-10-08T12:00:00Z,2026-10-08T12:00:01Z,success,synthetic-model,100,0,100,1,different-id,"[""agent:gamma""]" +after-untagged,completion,2026-10-08T12:00:00Z,2026-10-08T12:00:01Z,success,synthetic-model,50,0,50,0.50,,"[""other:private""]" +after-ambiguous,completion,2026-10-08T12:00:00Z,2026-10-08T12:00:01Z,success,synthetic-model,70,0,70,0.70,,"[""agent:alpha"",""agent:beta""]" diff --git a/examples/litellm-spend/agent-tags.csv.license b/examples/litellm-spend/agent-tags.csv.license new file mode 100644 index 0000000..0f217b6 --- /dev/null +++ b/examples/litellm-spend/agent-tags.csv.license @@ -0,0 +1,5 @@ +SPDX-FileCopyrightText: 2026 Vijay and Codex +SPDX-License-Identifier: Apache-2.0 + +Newly authored deterministic synthetic data. No production records or private +source material are included. See the repository LICENSE for license terms. diff --git a/examples/litellm-spend/agents.csv b/examples/litellm-spend/agents.csv new file mode 100644 index 0000000..64d700c --- /dev/null +++ b/examples/litellm-spend/agents.csv @@ -0,0 +1,7 @@ +request_id,call_type,startTime,endTime,status,model,prompt_tokens,completion_tokens,total_tokens,spend,agent_id +before-a,completion,2026-10-07T12:00:00Z,2026-10-07T12:00:01Z,success,synthetic-model,100,0,100,1,alpha +before-b,completion,2026-10-07T12:00:00Z,2026-10-07T12:00:01Z,success,synthetic-model,100,0,100,1,beta +before-c,completion,2026-10-07T12:00:00Z,2026-10-07T12:00:01Z,success,synthetic-model,100,0,100,1,gamma +after-a,completion,2026-10-08T12:00:00Z,2026-10-08T12:00:01Z,success,synthetic-model,300,0,300,3,alpha +after-b,completion,2026-10-08T12:00:00Z,2026-10-08T12:00:01Z,success,synthetic-model,100,0,100,1,beta +after-c,completion,2026-10-08T12:00:00Z,2026-10-08T12:00:01Z,success,synthetic-model,100,0,100,1,gamma diff --git a/examples/litellm-spend/agents.csv.license b/examples/litellm-spend/agents.csv.license new file mode 100644 index 0000000..0f217b6 --- /dev/null +++ b/examples/litellm-spend/agents.csv.license @@ -0,0 +1,5 @@ +SPDX-FileCopyrightText: 2026 Vijay and Codex +SPDX-License-Identifier: Apache-2.0 + +Newly authored deterministic synthetic data. No production records or private +source material are included. See the repository LICENSE for license terms. diff --git a/examples/litellm-spend/coverage_runtime_sources.tar.gz b/examples/litellm-spend/coverage_runtime_sources.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f9446aa95a2c679d9f8e12d70ea7e984ac8df165 GIT binary patch literal 25726 zcmV(zK<2+6iwFP!000001MR(gU)xC1D7=63DS9>8wVl|;CXfqR&n^Kc!&!oP44E5n z^b@uW+Srm;l9Mnj?`MDO(%tIT#WtDDobxuGf5P2C&c z8~gE}zW%2JpJ&gW(O>xF|H8}dKXrGuo_Dt3zYdn`ZtrxT{^|NR=u6?VNK!9`CgUhd zmz7)fK0mnsCZC-xF^Xn0KTLbipN=OlM%&%7|MG>uv-5oC#n#L2cxUUy*3<6xxYK#M z{oLR7ww`bNpYHP?pRf1vKYO$J)K4~l!!~d~Jb%7Zz5n5RZvSsRgYrASVW0Hp|MC5= z*MAzM{&YHPC38O(i>kS zGjBQ-$!!Qf(qJU~&+{lw6ETUS8PpZtXyngRe=MTOWEzCNkhq3_+Z!A6TWBnNDq63_ z5^NN$R_u?Wcr1qRkKKdA;hOa}2I2IUx(S_(K0(*9C#s7$GGL2f0#Y%L0ZQnFBPbgC zV$S3A$4wDN^!D?um`2x08)so7m>~oxc|G^y#8?uOtdLXiDZ@ ziZ6tGaRNV8q2y+f2Gfm=;b3qkdi1L9!f2+x>o(f4pG4D-e!bC#HevlX_6P5e&-%kt zC|Y;CdEmzWzZZUzCQeg0$pYp*NhXV_^?2d|v~l*Rv9Yl+_9tQ-T*L1=cCN9zA>b4H zFo&TUM?1OkwsxLT0qBW0cCT(zKdCnw?Hm8ItgV32dzcFo+@~PDp|Q6kAU*Za|AffU zbDW0o5@;nHPkk+?42SDwEV3ZC?aH(_AM%EUhMLmfY@yM5LMOTR*e}skZ(Y&{( zkvC52vW(ai4pj6O({xi6Y-2H}!2?VE&oGn*LGltPkZOSE4x^46N1qbsQuKP_C6x<8 z*dNs3-N;M)a|i!*E_Vg~6_Y6DFF|O4!Y2Uao&ZdOI7yo#fzgBrMgoLV(G+GOh(ec& zIG4?`cU;f8G&@b@&mBOUc?6r@4aTy|T)6BFfV}b25gZjQwW(UV5gYmx7E!imb;JU90J@zV%W+mHw97rP41dRYZ%9xFslJ^&m|zh0e_S- zk<#<+OqnIX;boiv9U$sNhgWCu!q3*o$<9i6cW14g6U<&A;iI1|g@S7A^6aw@>qhxF zZv5%ok8!T~fu^!>yuK00YU42@ zH4T6%0dz(7DFE(j8b$Lf54gn8o54xrji6!}Wt^2DCM@6a(KH%;aDgtTaWu6$ZokQQ z9OgAa?v9btc^}TAF><(L0W1(qDdM0=0KaEmQ?heNF0g~l2zr3$G44pBIa1HNe_^u6Sy`EYzC~G*$$KW zU2%TL1b)||_R!Lo(eSTJx$VzVuFB|+qX=J-SP>p%C;qPJcADjVeuOcr&Y<#aI8?~N8+*63?#@shPYX?32{PpcCznuUg6R<`?M#VKoHaISOr6?L=T zy(spsTkKu8vUlAt?_KBqK2Jn6&x)N|%##$@rr@K2Z;s5PUr}Hzjt>Z$m7iXQ$ zF8;qOi>+*q2qQ2?X|6pdKlU7*bS1}#^Q5@Qa*{X=fMgP_at3%}24M)4`6JSQI50_0 z(8>s$zZQ!4Qy&ktL~+u%tk!)IvHZi&r$rs+sh*HHm#+9y7^emIy?Q0GcPA3zBnhJ_qKF z*#sOl$kj3bnn&}+RK7^40w4NIn}n{3S&$^qZ7Ebk*F$j?0Rxgi=V5$6jJ;c!YdRVe znnSE9+lP~R5?n82AK>tv_%R;G*!7tggcBQ^{wDIwFhAygDY7#6_cIk&A@2@nSYl>Z%lr;bGd_ zYKSMoxd^o?g;Zm8Dey_gF(>e?-G_qIhld^>-+Y)8>~qN^T(&=Y(*^2eGFej!h&FeC z_0O4+U{RbODAt#y4q|kAnJPyM7IvFjSkVBfI^*p+)uo7HSC`#R7qD;ZRN}nDT?IiK zNlQ~aR@yJC25B9zLsVF|ra6F|&OlEooVHE5FRht1E7FO4shFq29K$B91JrYa;9j=r zdxA_n4CA&zZ9ySyBkLx&ThcT9U}-6!<5vR!&oRj5)GAF2iRIW(CXhpU;DV=Ptf8iC z*V=Qk|3OGCoKxvbvt#^ZV-0Y|ROG?sJ6^d!Mr-;w78Pq*p_|f6VH~EHJRn*Ayqqbx zEMFj20@2Bf)Um2F_Q}jWs$M?cVQTpbJ(Z2wZ!BETBb!~#e1X)NN~FwO_p^`wZVSJ4`C|QgC zV7){_gFfM}HdivXoaw(|FX z+nt@~f87854xg`c|M!hJ2w-y&!>lwyEz}5UHYWnUjWi}BV7#ymZ%Pa>aDY+ji<>A( z+Z*Q^?PRsf4Sdnt{OHA-fQg%P);lHto^Bvx*W1~Cy0tOB!adrPRW^{6s2^@jS3?`o zoOsG!Yb2z~#WmP;D&=Aidr|oRn03?I`#+AZzU=mY)&Aew*?IOnzyE>X-2P+#{|=w8 zcK?5>MQg3zwY0#Y7sG*#A~OnXtxQ_ugPIkufNde1dP^?Sk()(?%-^Qd4LYhF+obP{ z)h+sKFY&)SJ3ARNU+^8>5ywq&c7vN9m>hh0N>vm{{)1QBGe7m9LC?CYm*1mg^{xHl z3WkL$E&b$He@#^f^aC6ffkciVQ2_!j&$FR1jp7C&OU56*x9W!KrJj_sE9UcvG3&! z#o6G^t}<+MZL1t8Qk zx^`h-Ca^2&i9ek*#bac^4Hdu~0B-2=`r!HiEbC9z`An^7>yS|zQ;F{-RCd6PoH+TI(Tt7gGIs1Z|8cRwzi|z zEC>VV-o(BlcH8JTx>0slrBy|)EzH?y~7j#mSj=}D`hLG9%-Y= zT~hW!{&h!D?7LJ-Kav;O$fv41ZJ^`OSQ#9esWxu|Wg_58F;W7_epLgwiJ}h)e@9gp z*+&5&W!Z#atOsR~Mj-z~BkHfq2G+!H%SX)GGTSV<vRUr%3MO=c%iOI08TPMr;if*4{ZAHi)WWwBX!?u273oG4 z_8>IYMxi@MA|RAAPs)~6T(e{h%UA^l*#?mq#`+8N9oxhnr@yF+=S*nivPw!#e+j}# z!~<4SiVPWJ35v+J5}^M0@g3?scLf~uUJ5%_%`qEv@uiwF?HUK!d&TLR@bv;gL%Bty zvKH!_g-sw_(!%bgMKc^;)x3qKD+^1MUDh<|SnS=x`(!i?)E0;rq0787UwYprZIJ9f zBHQH1s(5U)T5Q{Z-)%h#KW9Y>6+}fge$x_u2?r0w*cwAPsDV&7g}N`=kAi+G%;%?x;7-5<#x0s!53NJ*H3uXt| z;&^IzkTr60UX@BqS~FgO?0)02?CPa!Sk@^g1>`~e)V*4af!wH`NRxTnRtu0YN`OW9 z7{owCNebB?eBU4Lz3sbi_70EU5Bu)X;O*PPtpL6i>JapO!85eT`ITlEdOJ)a~OQya* zuXoxz_WLs2#r<##c$-Fx(M?XU`bO-*-b2a?YoqU<q*dJE`{SY8ETGnD-Psv38W&x!1Q>d9gpuh15b{@G(NSvSz zhbjfbj=Q@>2HKq`^e>-sY`a-fs%w@qmDqVpbW>3tr_B2Q0wdJrIAp9%kCSa^wyppT9J%!gv+t6uCc~#Xa5m~WD z<~&N645HiAyYi<^dD$2W*b`(R*F>KM_@aCORI~K*9=1u{VY@mkdEdyP{5D)cDs;piWeIcX*dcXhq>p9z)WL zOX)u)MrEo`d$EjFVHt<1g6<-zF^oc}g~k{6=o5w)@rXxs`vr29xfwDT_6_K{l%?1+ zIE-4BXoVZIm84Oup4=BGmAd7#ptcwDF<;Z+%O1m$2ENKF40Xk$>0<1=3#lch1{#X8 z=5~!d_^iG2vAcP^F=#NqwhG%>A5_^CFj)qB$*R!%M(#Y(eKq>2y2C)10cZ8GdV zn&&W=&ZVYI$e{+;qe|xiMJo|J+g* zg;c6VBx}hm(2g8~h$iRJL_cbX@s+WR$haY0kRau84S7!*{r+?DTTyZWI9dx>t^l_1Z&+_Q3L65 z2-Qo_PQElyt9n!9f*ahy>9L!SyHh7L-y}Q4#*zR$&?)J{sANJqB z!>83+%K#jTEw=!N0*A~*(`7rm6}LHN2|5W=;18rubhCr(^c=)IR3ejNPZ;hPZ3K3@ zm|YNlzhJ0;vgbPf*TQm?lO6lut<^wzd&z^r$)c7q`k2}IQdDS$?a+X zsK0+kmgENMMb|*nE5EV3dlg)x)^P7sm@jg5N`e+xRZHIIAmri}ShaKSpyHe1;2nob z8MPg0I^mKt`g@ov@{U6##}D82hkY8O__im~$OZ8vuJ6{E-CU%j+U_pO;P87kPk(8O z+E4Ily?fU>IH)lp+dDpB0Dm*;6VtJ}_S@m${fT(}69ItVgW*AcsD6w6!O;<#pga4{ zMnjwp#Oc|=;Qg8S;qdG`QGYWSzS}z!`={SG#drNZteD~z$Hv9A*k!h5lzb!x?f)J+ z`rq|aLH~xpKt!Wc@>14$<4;s@dN_tQMCL8|J~nM z%)I1-=sbUpEaSOw9)078)-)AQ#ijVqfAo{Peko6$wgQbwfCr% z0r~Vs6wy)~_kX~!pFq(E!=KPHXn5wLGjIR+fc~DJyme3e!|xCG`G0Znbho{$q*?M6#3Sr}gF|e&0HXJ( zzJGACclKRRjK%^^FXvGm9+LqM{pvUkNBs4#V*Y8|z?t|nS^wg!wLWrWuWvWU{>RM_ zBT{XB``2#OR2*_ZG_E6+j65kQ6j|fz;`s1lEc8F=drLiDS~CALoY+b5(Z>)W6a$3} zI0a5Gij;U$PAY5uLSebu0Fzn=Kt5DeAv$H06BMibxRv-*4k?FH1m3xMDFzLgj^R)b zyo4f}c~JfaxNn}2*$3#GXw3wW0G>46R|lpB6d1zW3wUsaz?!dlmo2JWk<`T?0ZP6fbSd7Pflf{@Q{? z`fF<=*La`WQ~}#?&1fg+6*L=DI8to%e-7WV^!)!-QeXC2e*Sm2x8cms#eaXk)BWT8 z|2;lmQT`*US5D$yiNyEC5SaOQ{q}53N~Wu55so>+qTFB~y*NOL5YCtja+wbNBwlk?u0D^VGM6x zKpDi9p^RAubOr=jAl=BS4d3iPefspJ#Eho!k`YcJ8odLO+<4PT3;C4xhLqsDVpHtz zo%T`KJ%%&Dly1fUCu;xuVtluK--3U)H0{i1y_e$x1R{pbH&u!%t9QEh{T z-%Q&5I?yU)Z;-Dm_z0gq1epxPV43669ebhDwZs+@GdUAt{0(`zUyVC6#Lt^R`JS$ zRnx1?1m(C%i=>9l^q&9V+#ay$aSI*NIGSWjIv=rE!$EDVqoYT(pMv+2sUbQ$N7KbD zOh~QyPR3DGCj~hh9lt+1Qf7$1_;IAyOV2w@q}2*w7`ih1AkokuAIayBlxDFh>gvWx zeDm$MqITzv#{rWEwsLSeCSa5AaOvJC2;mf#&Z_q{b0+l%Ca|Kym$t0TeIC?0+v@zv z=JPJQYI2+G&xZ&4s@`NJnT1iE(d)s;SF0)PA-aVv~ zbk5))Q4Qnrkk_BT(%PzLWiL1MBIog3Ax9~q16f659~85*_drZXepCE=5fRoa=^zmp zWFCdn7$b&}9!EOr{pF}ZH1&k{i*)Ez?qG`SQe<9z9M_*V2nKSkH4rcEJzqng@EFGcGZEEL7pUMM5I*s(8&F-g!bgfLwGp-KkILo+}kqM1wHUebEV zfMP{AXo>=ngsvDN4VtV*X=*DuB^g*f1EL$I_M>dsl=tt)ZoAvrZg-Tcjyh?HBJoiR z#7bH+GHD7zaqzlr0Yz$s@qJ<47a_($yoSlb#7;O)LNJ-MDVi`v3?o?4h-pkd4V5JnVvmo}GvQ0Kdski2I;O$0)S<*HL_XUqiTB#jFb$P0((| z-k7caU~b_R&7zqex3j(V((3zzdQUD94Ce$5GKSFEBkU_#^L{y8V^(kva0~2mRsxjG z2wk!xT(bdb1$$pB*xRUyZ`)(p6{n2Ng1bS6|332BsJ@VxE^0Z`8d|5C+6N^A_@BOJ z=&?!Unyg^|kFDK1@3Y9d@g-|lF$j{m;7kBEt(K^8lHxTjbYHf@ajAeBCt`bRt0{sQ zO%;*K;9bik2M$Gr5Jx;kTE(u5>;ljh`$VmohT(G^GuQ&mjG!(mF+lY|ac*jLx8a z3h;8a&SdW_ul&qxQp~!4TeqqU5c{Wv_&4_LM4%-{7GAatRfBJn}tKogp zKX7Tjx58`@ksu=|lpTeijDX7oPJ85qq}iJyw}mwX%L+-mOyi{Z*60(kfTT65#|>Wg z`mQ-T8g5KBJ6V0HT4GbIT=55)0cisuTS=eSFJ1&PAydwjQB~U?ynA4$N z759HB?uyXE^U9TXfnOH;7e7ny|GHaSo$Z~1{m=71>_2~(&sU58JCI3yrS^)Yw@o8a z99pGIpKD^m=$4%@Oj1R>C)jv%)bF1i9KLzuo}Qoy&%t2-pZ#I4E3y|yhsW=KbYCAH z_cpz&1f=}b2kZ?OpEoB{-%n94C;J5>5pZjU>334ZYP{^A8xYY#uCNUIsdj5FN;iRM zecaf1kDQ&hC1GOCiR928T?(tHnZF7`yd?p4aYmVf#+ePQ@Y0PZ`BNZ}x@6X<}UJ%j)zQ5c1(H#zcKw#aD3pkAbK`Y05NA8=$BWgt^*HcMp`3Lb_ zI!CR{M5{7|^`!Na{Jk+Iyl)cg*pz9AxrBPdzxchZy9i-{|6!vy`QS;4sYq6xX`pn< zgPpP;0m>x4Oh2aV@t8QNbP=TIrD$nD;ab3{Qf3KBO^K9Jka&cMO-kZgrw|)W@%ZtF zPq|imF z=m?c`E5?4G`X#%;arkxCqxYO=osJ5sE2kOV!$uV5Es&u%Wo0M1p~oY8`D@xN`W1*@ zsrXhnkGf43jM}&=q7x${1#&9yGWvsGIc8`ANwMmR zb5N1^Ih7&>W66R(QE>mlwsYvnQvf)?lQN~&Y?ey-t77Um3Oy){=x=(wdRB~RS7xxI z&Q62&7j!f;fXbRitF2d>S(^B<;2=3TIJq1gcvj6q7KndKWi7B`+i)l@6^0AV#pG0> z5jrpdcnhJU-N?UI-2J+YC)G5A-kj>M^v=v<4Q60*5lzfBda%!zQ~vmbAJj2|vU`qx@0Yr2@& zDwY(V6e2YRlZw5Yiy1A;ch}+5BZzn5FmBVTPI;QvtCk#_PSn76!A1QoR0KyoBnzd?@NXOM>MA=wIl;!9p#Q;t-?>ZPE`$X?KztMI+pgM1U z2{%YsD&s&=QPSpRlCPWiA+?Bj-|W>VK%pGs zN{O(fzQ+2poTC&yr1pS@^1=Qz-xS`~W1nJza$K)mH4Y3`cm4qojp$@$A4(_;hzmo~ zy*=+-vc7QW&!Yp1W`m*C&M&LtQgJkGAr+gb4(Av+c*xFB?O?1QM*mkS?KSho3UXHl zEKNvQEq{Y)eEA9(R_IC9u&V2-JANI(hWf;k2_}awSEO0JA73G#$5q;{%Q_IZZWr># zVVp<+*drIbz>o zupd^$VYb+^kofY7NNJ#GgWh=XB#dDDu~Jo~oGlU45j$t+&Vi2_DVF8Ay@GO$qm1F^ zg!025oXgVnL=>G-#%mtdJF>`jE*nkV)$GoPfaULfoW_Ptq4TkQzY*`WS+`*yE29}R z!x#Wr_EL81q3G!M)ZF?JUu%Vb;+5sNyxQ}aVHaL{sG@9XK$Vp zXps3#VdAN?gCCCj2WY{OVi-Yl*iZ4OeHSx(eMs7Vl zftXUEPQiy0ij{|X`j~*2nR33djgogc?8zzFRlK6OHW=yM&u-sd2iJ!vWFf}DmzB#W zm#CcO&$IJ&Ysl0UqA0L~TaQ!-0(`hAsm24uzD^+0Bxrfq>6> zclXk0mlCL&%`~i+GC+8Ztb*1OO;3SsJ1&?BHC|nW#&~gveCXC$dylGF6$?Cyd(MSU z>VQ|x*`}s(HA?&FTwJs-Ol@$<_N3y#v5-ejJ&U}aOP0mj9lTbg5Yl{hP&!S#S8d}z z+jR=zmvzLxZP3j+6gz{|_rw7{ry$ryE4ikL9bisC-R~7)#+=|Lp5#I4!)u(Ksre&J z>1vIEK7oM(uT(xl4O6O6WyE|PaiJ+5H^pR@%6z@FLcKTWY1dS)Ch8}rhbvFVlGzt< za14gFh^M{x!y}9=PS?{YG0ImQ5k}r0V^kP8FN$CR?PITpX=UfZ*n*6y=yPeTpb@&W z#c(kgU~+Sj#GAyjVj!Mf`bl5lA6$X&2JiY9oHd)uLI<3qeRuEZ2%+ths^w?=nkNT# znn)OQbzThU#2lB3wQS5I%Gh^Cc2#*xdS1eQ5fZBO zM^=s{3^rmSLy<{rwNO5CnmT@NtCdC$E!m|xYCxgtQPYJlvXp>-$ey#YutSJv>|Ovv z_*YU{1`vk2RKl?;NjzOzO_pg%5zq#}wmr+cpMj#%3Q!Ms9cK39%#y&zf5Ir7cG_Bd zbF4p4ib-k#IZWc%gfeI=M~BH!g=%$Lz$f10HV1HFD-0QyEzv|zYMU9GUNOuxxk#8 z&Z9?;yw!aLEd-Oav~9b6wRo!8!lnLsc0;b{_2c?Etu;NwKbMRt=mm~`$mb}SWJDcy zrn;ssY@){)&K}7$Aq(fn)4^c*Cx|_`&1Edu68YP>@0RnV#iEwp+>f&%OEZQ`;{dqv zM;|Z^32;mD6yP6C#c&$5lo=Fx*CQ|R>#De9%X=1&UXTx!4Uayi3F$euG=ORIZ5-Z@ z=-+-EmyA5ac)a(){Oe@RV_7%?4fC!;5QKwKQX;F$t{YdED5-WxqbUcE9)Y2{pUMHJ z5J)oYxV5wC`i|@8qn~rbtdTOt+vMcc#U_|RRnVm2o+;7F#Rwx;x(@o%H9E{-}&8@vIhZVyuISFjb3f-y8;;NJ{3Z-HQ z8CN)Lm7Fj+Q+j5~=Gw5|YLj7D!ZLo{T7>e@`enOLHXX%mW)?5z#%&XE)Uf5o!c*>? z{?dh$uIzeFey4>y7k?HlU#=V4!q9^w`QR(8S2jzqawFeZtf@aq$qOqV>C2oT5^=af zkv&D;)L{cWgE=+XzRz0us_gaXmUWiii~orFooT11Sq!G(7?9sMKsnkYQpA(^t) zQS!mH#_FZy_kq*Gk>0Htc~&taC9`7kcIJdUvnB*TJR2x2uHD1&?q7m@4{e|{)8{x9 zC_q%g9T8Z9V^=5dO*m#le@ZtWSD97r$#C%Q z=wreLX$?UX*oD&U!&33B`YAaIW9W2@OWiJMc~w?Hx{YMtEaD&1C^IU_!$(^NUL(Q+ z{Ham0S+?v+Auy9u;4!IOM)WLt_gM2_GLKFcJ(cBMWka^zL7P*VUdNcBY)_c`gO(H* z-q0sB%8wW-r*s6Sc0f0lb;2@a!w*+B`O7?KxfavJDwJwAC!<=q%}(=cUE!H(67zU5 zMkdF_S72~jI1QXHdDK`hS>$XAK6y+%4LQZJ=Z##+GQCE{I<31jC8xjBD!b9)k8|=? zM&(5|sJAISaW)HD#`(1?|HSBduK1P;x3>!(cGJg0xY| zo2Uq0nL(Lk0L&1nPWf?!NjGz$St(maE*xv4n)hEphRct#s_WO{P!uB?N8`oFAa_;L z5N!&X;QY(FR_K{MUzWdwp~k+?HVyf+U_chu0=df7mqSRwroWQ=A%RWnjg+ZjP7kVd z?%Zv2A2!a7hA*vkRe_(dt5T^+b{EBDTv8*nv>bk<*PU|ekbbwt%PplXTChpC(yYqA z<+DmEHOf=R)5(q}6{$)mk@o-M>w`5quRdgH!kJ1JS7`oH=cVaE4+?!%dSE#c=E<#= zm$ZZ*WeqCxoibfUE}vQ}fw#O1%k%G)SgW4fLc(QJF5)@=;I}1NbWSL45%yxdR&g4a(dU+-COI*9=Go4+x{qp5YtW7;yw|=)%?(^8~n#0JU%(|yX zHdjTNZS7NHIhAL=mHw?dVS+fxQz4DsMs;R;3w>3e1UNX#GyE&zSSU?>TSeFjS1UG% z`Y7H_4z?$|i?_Cf(0RRWB1Sh}cunR)DQRZxoiakw$hHV>8u+|7Fp35)ldB?y6z!Bw zv~?1Dv}+S3ZqCfi#HKBL+ul-HhNZvyN^6(*CkJTlvNWzt_Cc$NEU-&2i{NHFtm(wH ztxl&?lTmwqZo%klD^JO#z`6>Uw;nKx{A0wSDZODfg4XnEnFF)vZtXtZ-rafDe*WTR zF@{kVrKhaY^NLCoEU?$FY28Qvbbs%pf6#O=HX}$O7Z)LF4lgcZ_;zuTUcd*>z7kk6 zY+;YBjB>FHd$pPhaM*kQ*Q~vZi%!RXd2#XbneXLnSRTsAE<4H4=)*LJIS5ASuxJt^ zcPH~4T7haBd&%^=0Poz%7Sg$t%={<1O@hXXrjb;$r}T*d#yrO)!BcN`HTL+ln%aV8Av(L!+V@(7L>(Vt6yI zux!%NiiNdsc`FxNi@IdyC6i?+{WDN>v0pfp%nm5kpp@p8_hLy!HITInJL`2Qn;_H9 z=gzTKc2~f^_x460{W$G}XWQGj6P`Uo`zmLf%&f@TDz__&ObmIB<3A;AOSOsvoxvUH z@m-<}UwgpI*J5S+-?97pd0H4xBUeZ=v?D=yr~E$;u?RGdPz%WEYoEYZB4+ zk0|TzFK9T)D))zO|NUYA0Kf08x%mN$N_ISUkFw4}V*tK|zn#@HI;H7sB{~EE)-x}! zCXophf3E>`_JIm&t3h65d}gCRoG#;ljD~R!VAEX6G7D^G&C4sn{GRxt71wPy)|z$W z!e;P&e|UQMbKen<#SXA|zX9s$WErTVHfRcAoYYxUZQ!S4-wm!q3^JX^JXx>=`PzBa z_lh}?WnmhCXFAt2p_1QiaKN1GIeMX;ZH%wT62EZJZIJPze*7N5Y+ww+PrzODjC(P@ z58IGe5xv&POihy&WM_g6iEL~aDLLlDpkE+F4l|6&Y`y>{CPr$)mLH&!f?e_FTim*CoIx zFatwZHaX@5aXVQ|gf^}kR0@(Yj4ee^0oY2E=t-uIkUdw(lbaI4gwg{LrhxTiop$HEJ8_ffYl7J|%B*aoVi`Vd&wbv$W zs460hpik(uzDR8Hg`Y~LgPM@F6+A^XgBnK5=v0LYyz<=0?Uo0ymQ;QPds!#;QZDH3 ziiNGv_uvNwD)Z<>kqV6s@5(tW<4unHyDw2BEICunlhpX0pWBCvBGr?pIm^0J6h>f* zih9hdh@Kar6jJ28UxXx^*sE|6YU#u&@x`~-Cl?}g$%3spr*~h#i?9AvaqIWheKdDu zv5N*GWqtf%I5>W*rKKYmst3-8f_udFfl zVs$Fvl|`*>b++N(VjaE>-xbX1F9p%=bh*w@ef-bk!4Jpoo1?wAPSdPpE4L_qsy3u#2dBl1E+^}RGAlK^3O|(sgdjeQIA_+Y zIowA%MZV%I*(eNURuE1G-vKOnX?WkTk)3>gLwzw- z2$W6x6W@HJNAYI3_x2t9s5li??Pm?=-Qnr!;qlwD9lZu*9rN2>BMDS%N&a31tdg@3 zC5(iynGIpG6`As_qMbCIW6W~hMbsf~W`0Yn=Bt1)D^LLbD_XsA@MYr~g!Iw!azt9G z(@}|(0}ucWGzw~vHBG<`Vck+SX~zJ=&^psU6>dcG{H7Yj7u=8-4cgZvV=ug(awhV8 zbL!O>G?xeET_G7JH!y~2s=Kk2TQOw*`s;eoG}~Bqi<6tAthW{f>R0H0Dw>X0xn$^8 ztko{zR!iM5O60iAm!{f9!f1(=m5h=VTU6ZuYEL$488wf69k*Qi%;a^J+KPXbvmn05#I>T43plrWI*`ol)dk<3qDVqCn$;V^j6K(STa_Wk{(;Evh>nH zO&@OQrIL&GYBd>LN<9&WrxZ@ex^pU*1L`%_sqPPs-oHDxGK`)5bkY|_elg4-`yP)S ztCW>=>;kinUDWdSQ0unwfEIG+-P*&bKWkw8%=J68q9rS9EZ63kPK8!VN}^`#8iRi= zZ9-@CqdI7eRDfM9a#aL6?Bo57s-kh_Y*6G@TrT7k7Se`MSr37#28tb_ux0y$~U{y?IQ3IOJ{Vdw% z6!g*RvxKWFW0LgTl<82u)*#d*+gt;O#UJl640Gh@Z!?~*uhj`p_C9<5v{zRC<2 z(voaHTPs?$g(vM zW$9EBOG)&WzVW$*Uw3!$Uo))Y!)CxN-r(3k>^`HI6b=PS?u#tH@_hmGCo)}Ijww>w zC?p3dt596b?M0H1jFZ7htXzD9%@qvAzR*IQRzn?7J!?7XpmH(X!2V$GsDHZO*Xe6v zW$x~4yOcrms)4_<1ZUC}ktF@3n8CJo=Zr9a9~(UcA%^teQ`gzePOR5^2k!gRy|;ZQ zYsLa3<8@YA&fE6cYugPkL531%B)dxwH3BaYE7tddVh^@55D&Lt=8#;#5@STO3OnWx zVe~0B*t4~Gp3y?Nb4*Ef~S_B(yU~=wf6c`(So?}A@sto!rFh+UO#5~+H72&%@h)pIolp|G6DJq@YzwTI-``^5Rg_bx+ zA@4P2;ZBjKml7HRR=fv)=X4?txfXK1k8%^ovJ@R=pnc-Dv2mv<^TyPinhe(ffAEJH zqk;Qsaf8Z$P6MbrH5povHcg9?vQOvUxPHC6i?N8C;sqQsom!)4YhZnl_+h)04>EJ$ z;^Refr%WZEpS zXT$f$`^(Q^yTak|X@7VIf6fNWdQrcVtDZAgki)oBYkJS)QBI+VL;+AJU|jBW`qL+( z3kUhN1!sRaI6W2rYjAj6I(g}C)svk6dsF=+~h#$TRz*=b2HWh0aF$eTBTIz=M6p|~+o zYygDQJ-DJd2C9YxiE{=3fkHb$zb*dh`*Wga6gq+1R9E9Ser#BT(dcPA=JOy{-Cb~Q z%zW;1-+35Jy}So zm$n~>k8T<`gGpJ2%9a{r;dIPfRmwa>9L>M}xUp(~QjVhHZEqUaO?+<{18}CO<;OFL zX`Mb}5f$)&BgMNw7bnxknl=KnS!G7Aj$sa(MyZDi9O(s)ke22_YdB8H&4}6JGERYq zlVz2SFy@k}qrh;w^@)mMW)0yyu~;&2lUY*ih!|Xxu9smtvCb zFWD3;>5TeRs@~R$Ag84zvIsDeC}q^+>C?_b6?A22%f_kN zM@EH}t^qN|MpUVERCHKs!BEr$Ax!pk>Ru~5!zPP$K>N9WoBIxP@Ufa zXXu>thiD2&2J=8lrqMN;-g@JYL4uYX6wwItTDoi_PS!s6jaP=BT=~7KTr^LkQCx+JPk4IpS>R*p9)k>012_WdbD@^_Wj;l;9{opYh(|;AE0*>@UMHPOHu-j z7BN{B$V5f;TB%nSoQr?^R@9=fBx=TBxEzR1rXqo;$UakA({ZglvCweu@U$=bKkoO* zwM=9|D9(m^$ESNVS?+273?9Bct*w($h>{QjHIs|_gnsgrqDztgO644~nZxxiYWl}i z0**a#)c?MJ1ibYA`{Ci)Po=!>$>8X4|0hvgyHxz{z=)l%rR zYbOSGrU?L!T5Wofz30s0&pJQ}GT$*y%~(uojma6MykOIz(i61k=QGh6SgTTfahl<2 zDG4Td4ZECrsf(NBW02m0IHNEr7%PRX*UlSQ_G)*=qx8nuGShFTDILy~|9U9}^UH)2y4N2eYyo@A5++t##Ku?=}9-NXWw zyaKqo>4i6D`wDHu`>yrUm5niEXc1)Ur}P?##^@yC7enutzyu0Y8~~( zZ2f^aZ6zhoK(Dt#Yo;q#pV<4fPJDbB@LKldvY~PS$ROIKpfRb8_bHRJsulOYK0Lqt z)G+(HXeHvGhet;gn%@r0kNr`ukrjzPm-r-p0fGa&UYbeJ}EJ5jDK`*MzMA$!p8`bw9&=lbxJOy~nSK&m@V$d|4!rPCmWKSO%QJ z2;-y;i-b-ju$Z-Foud9oGsGQ9AucyrYD|1 z?R2^?9xm3hJix{PtG%}IlB%rwg9Coj2EoFXjDh@+Rb)-;Z2XiT>RM>eSg&#)1iQ3- z9F&RArZFzi=I&JN@176T)|RGa;izZ z5(H{u>3kOzG{B7n$&K1g4H+!E0Km2lsIZq`X%w*5^}IfpX6%`_qA zLrvqSk~>?L?)VGk;qv{nro?`5c)FIzu0GV!b<{M)9jle_tIsO&=bnAS0kd7stgSe> z3_!eYT@TwL5^=rtcW$`?0eobjT zrTRt&2<48d?BuHOLoI!Zd4M)3d|KHmY|l!5>ur1;2QmmXgG(*6v7P%^t=3Meq}InZl%Y6)-c$60Ha=PbRFK@ zuJJ4QbDec`x%uChvg*3-m)!-KAL!{Csl(K8mTFMcM;#7O%euCf;;pi-I6M%G(EA8e zNdD5xVOHl%TT|D=?BPm=mVG+Q~Ix1lhDyXO%3~aUp`} zMQq7MR6K6gm4Gzuk`c^=Y+eQC3Q}>+7N8(Zk@woBj1fnHMriWe#S?nThR#eFx;vfX z?*4)u{g7R}F`7b2@tqfk(3cbD&hao!>e_A_f9`uaVXEHr$ku7EWW!Yl?J8V9z<_0F zHJ#t$1)Z10C0Z#X55I?vgLXvjo?`_NHebCaKerr%=zkvh^Hl5+XjqdzN*{>mB!-kH zThrhjMPbnDiz?u8gZ-0zMV1t!x?mi*=#$C}X?mlhb!rKxhV~3~hNL~E@UF@mVRt!T z-bOGHF5L&YE?K?0?#v59*L4`6UKe}{8~smzeEz3?WR}Eaa|#0HbUJGhns2hC9er~1 z`54>t+XuJNfzPvN+w>PcxxYKlo;~mUsk^iFyt4)Wb)LcVt*1}7w*J)l4aQseEby8G znvA0;T~=<@`~2Ykn|y#9Iyv~Ubrg*JF!5W5I%ChS*qfvOU~8-0+4x56W0VRE)E33b zuJ}Iqmv>8UA^80B1{|D&dD;ScHV?v(qG?2*Z$;}{F^!@*>TcvN=5gF2+5;sNBO|SIlpr1L0HAdM%b=sAzrM7^uh-8L)i}3J-#| z3;fZ^`!kVvc-~In%mnsD8ORGCyBVSZc_cg$`kyj3sJ(FNITPOl)jmex_6ON>QHV%6pvU8wXBE-y=&^_GiV=BiXN;fFiA6 z{wfY}LOHQaHqLm*_LJnqPC-~)WoBv$;MGiFZqaVyyY*ms&e^hefw>V8>yl0Nj-M?8P! zq9Iapb2!*LJo@P?O(zE&!pWlwb9EXsgo|#DC@NL zvaIb`u_2)nhJeH@uU>3`cgzAK(M_CP$~UeGN|mc~*+8qDyG${MhBmt}rL#Jw$D*HB zUaSlN-|}4M6qIDBL%O;z4Lfmv4`a)lH&Lu`Cc||WUYg1U&jp$I8k1J3?4n9B$Sz{g z0tjpAIG9XyO6h7dL_E(vc^G+-oDQM7bR%q55PBCu^twOM!K zD~H5=J|>IGMB7z_?kMZq^19;U^Ycmn=#4|11$57&C9X64CO-Mm9sF}iSVkqQt0Q*q z#dt5A`G|%B)o}|p3_c3N#b*Hx$`KTWyTNI{M40n1Mz$^Y)s=?}jz{Er!O&l+fCK`w z5umR+3S)v!RVyoee)*kNi@uXhC|yniz;QE)N4+}+ecv2od_%6dsS&h^ouNw|#vrAU zlpe5j?)e7OV`=JQyeJRMhKkz2I4$`7?~NxYONYU!E3FuhtmYEiJUNpJB2^ zACC3$nD9S=fjn<_;D4pAeq1HG&Xm3xKIwO}KnBA#O3l467+J}wy)nAIm$~^p5zn%F zrcXC$xQb-X-ty`Kpwf_2ZUnx2a5l00W_J_KSe?uKExk9sT?uG@?FI=IUB`+%*MwoA zYFm*qmDO;F%u*w{P=uW#*}j5uA4+vvg&F;uo^QNtl{CF`W8C-E@BG6ILCbeEqrQeGXgoF5;B?*Jth+6 zz$G2ox@I^AsoC0jwkBl}Uyf2Y=0TtBdCdt{n225?}(CJ9l85R zL*MP45ciY;Q$^w5J`mU8PkgDq4EpKWVAz*X508;@%PM-V*w{#h#CF*%1ZlQ!n3yu! zwt*W)d%C);s&yl8Krnj`%^VHhzCApCOMWpq*4VgT{6@S1#21S>OX>2$qN!ZuaDPuG zM0)p+4vF6Q;WY*zX#>wAZr}li+kl>kgV(Zh+t}}h$Y8&GQ^kaFn0wfLB};`x@}{;YRWvg&lzAxX>*jMR|w)iS}} z!^}F_P#TT%&gHptni7hZ+YI8nb7!~+L(JWIS?G`dnbN*xZn9(C#mTax+c>$|;f7B? z*(rKgFypM)D3z3;ux5$_vsQ?PK-;RwDV6V)A_HXX)ePK{U25#gJoRE$eUkP2a;?HR zeU~MhJy#=>?x-&REJT~`ltr7CD~=6~mC9noVFWU&pe}TWTzX$l^6DF1zM;0v7y-}N-xb}^I%>UC=>W&mu#|=EL~Vdfx7BVxgA6f zwYbfe)O1S`JhSbWdyyg-EZwTQQQWJAHP&Fxo$r9_*(ZAd2ln(avzhN;Rh>(^$|c^c z)SlN1Z|k#j=ODA;kW~tb1C|UZUw@ zjO3VGA;scCn`Dt$YD&!&GndA>Dua?^FbULgW{6~|I5YLKvcN&rs@+Aj&6Q!%Gvi#V z_$p)EOv!B;{}x-xTL3?(k#dXQUq6(bDjJ(IHLhQEL?vyN59?LI;fzgqa*9a3OC&bf z;;wD|^PluT6a3F<77$X}L}xOUz`X!O)of|`Ib`6UAChe;Mm@4d;GFE8eb?JuB=IIC zBt#}-mwwV0_@^MC9PPUx7xwa!ho!V-VWL!60#PZKYDDK#smhB|ObuhVY%oQ&#mq}S zh)$>Tigc^OdGrr5QIvlAa(43&iZq925lI3oikK%PW)is{)9u)X)N0{DfF(;3T>UKN z;>ddc(tKZlMmD{q9cA-i%wml zm~}5y03>X%lskXW6j7>rdv>0c&d!n}!%{k!y@XVjL$WTCpPKTPsYTjOF<-6lcCm?1V6vuE+3)*WzcdZa*7q*k%Dfe5{Mti%YayF(*j;(MJ-H%Fic=S~RZaEOU&d@Wjiz^J|Oj${8uEKF!M-^(%SA{8ZSgQ5rgi;KFO7wV{u9 z{}NO-P*A?golDv#4^_vg^-hqpscQ@^bIi=PJXW*!xiPIvaGEBsnZqebsvkcd2cxu& z{N@M$HerN>!66&nxL*R4OyYdZ35?JI*U3mOkIx0-z&$Q*)AIgT*THRk*~k=}>}NEO z{f|MkNGR(BO%zhEX%wWKK?jMK((AKJjan8E0MMojsE)yy{Y$AcjV*!FdQAS>W}d@o z;3X{vMU=?eK~wJvjWoUZniy*G!^`F5a}bGK0t1V3j^a2+0K?rRo3uD*{nE@_U+hO- zIazlkFiI@^E82>1@+z80(Afr%Vgwrju+45{bL@ZIoQyZ0I>k1!cPoKn8fc8^?f&A& z(ZgHllr_O+*+crl5Vu#kxU%bYVW!P)6jzp;V#kJ1;uy6R#l&(O6|kjdRt*=z!nn}Rlp zcof=Mm6e%#ST|LrRn$gT2S7=Y3Z4!>8GYu8xINmERx!6xb#u0yFm7i=`eDpn<%pq1AR zk`1fG1?vKsrq^H-4qihaz3VVa(qNPT5yQ(6GFuxf^&f_0BJ(yNeDi?Rp=re0I1=E6 zOci4!tzk?23QZEfZAn4b)F^>O9{R{jWT>Rl7Avt>70Rkrpl7}h2BIn$t7d*FYOafx z1(Z~+uSUv3)%=)NgUHr)4LPH*Y^3*FOIj=+SJwI$5%!|=vYe$TLA>%2nA#rV>>r@4 zu(8)Z*{p-DbnxE9Cq?4jDh0G~mg)v870wjsX^Q69z#Kv;DeQo(gIE8fR5|RkoU#mz zVgpdmV9Q#rQ`Kll4?$QK;|;U<$PCFYB+L!mA87z9#378Hnfu8NDxPvwdQE5 zjK=dLKOgOYWBn~cN^+dk$)7Sqoux_gezTtEEJj^FL@n@mMb}GRKpVumQ}zEl#v<>; zzj!dJz8TprtHxJ3^u6GdW;7;gQBTyD_;TeOo`DGf;+2D0vWb>@n&n|OI87R5yjVp+ zr4UZ~aniu_I??k^N3ku5%(+Sg#f?7X)2?dIY$Tn*j1trBO21<#H!OIOdr}pekfLm< zh%LK14ZHGLMVm(C_L1Pe5@95aorr3#(M{FWDy<i4l^B&3Y$BEU?ZeD#u>q%1$y)QdjfKt~?SYZAw9Z^JicI#0Ae5e+nUOW8UMy4i zRWvMfJS8_Cx+z?qBT&@|Y5GFuA2&y)VG8 za(q-D9gJDeI<(3$2r~L(gve+prEp0xBlJT_Pw)eKCu^`rn71T&yn&^RpV?1j_EzCL zeu}ZQmb6fc?f6k4j(deT3Z=z4(>hG(2mf|gRX&Fwm!hSqHvgsr)4wIM%H=92u5R@f zQJ_3oTvq1Wo_N72p1DJrt=3XB#2V%Mtz_1-1a;WlRwr*0XMT#Vrt0%RM;e1sQ&Ibv zcveL2R1{9GT4|Xo50|Wqd?L_XGN2;-oqC%v&7>|Yw_=SCj zW#M-C6zf(sV^)%}dn#y3)*mE1x+`4C9it;dNe&rZOfC~ma^;#C+>e~v;)$XeJm%Oh z)tv{oRO|t#btL{}yqL|C`W;Rmst!t|4weJp1@jlgDv(=nqYJbMT@K67CPiP7h!H6Lc68B zfK+6Zq+$YapP3aPheKXL=WI`=FOmGQL&1>61ifyf7fUYIVzl04BBI+M8BluyX66VWCWmfC>KD74&8w849t8M zM#l`w1)b!-fW{@{I!`;$w~0C{ViQ4-Ldhqlg@ri!*mm~)oXX;Nf%gW=$)pU?DpG$>7~^me#+ zd?wzXV*Y}`vEax9>Q1nbFSM$@9Nf}0EDwfO=O}6qkM35^9hP)1AHS{G!$OjliXe)) zgJM~fWv3(GX(jj%J9z^Za6qBeO4(bv#nf1Wl*YV>U zRk||S1CqFr??MhlwYNqc_v)6@%E1_JnLCCrS%A1I0tbZ}<_;`xlI9l*L21@3W@OQ_ zc81MnpHiiqv2LST5$Z0d8m;tP*7tozCJoDbqu{r(d?Fbhv%P#ZWfp&Un7Cp?mOnK& zl!X|#9k|OS+co^T%a@RgNbBm=_-jC{Mqb&F>s@<6xCT(8&y_%{Y@}6VlwqIsBRi|2+&6_-PdZpmoAYDsly>6Uz zXaZoG)0PVrUmV~OkM(joprc2sewWKL>a7Y-`Gic&=(H;`|x9hs;JJUorE% zT~?nOZ{a!Issd#?_$~wW>(wBCq-6Nx^T+3p&mW&ZK7V}v`2234{~xFp0Db_F0Ra7) BihBS6 literal 0 HcmV?d00001 diff --git a/examples/litellm-spend/coverage_runtime_sources.tar.gz.license b/examples/litellm-spend/coverage_runtime_sources.tar.gz.license new file mode 100644 index 0000000..35b3eb2 --- /dev/null +++ b/examples/litellm-spend/coverage_runtime_sources.tar.gz.license @@ -0,0 +1,6 @@ +SPDX-FileCopyrightText: 2026 Vijay and Codex +SPDX-License-Identifier: Apache-2.0 + +Historical source for the unchanged captured row-coverage fixture, retained from +commit 73df8c41de98e55758291d5282314d00347e4a27 with git archive. Source hashes +match coverage_runtime_expected.json. No new observations or private records. diff --git a/examples/litellm-spend/export.sql b/examples/litellm-spend/export.sql index 07342c0..c6377b7 100644 --- a/examples/litellm-spend/export.sql +++ b/examples/litellm-spend/export.sql @@ -13,6 +13,10 @@ \else \set routing_details false \endif +\if :{?request_tags} +\else +\set request_tags false +\endif BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SET LOCAL TIME ZONE 'UTC'; SET LOCAL search_path = pg_catalog; @@ -76,6 +80,11 @@ SELECT CASE WHEN :'routing_details'::boolean THEN $routing$ $routing$ ELSE '' END AS routing_projection \gset +SELECT CASE WHEN :'request_tags'::boolean THEN $tags$ + , request_tags::text AS request_tags +$tags$ ELSE '' END AS tag_projection +\gset + -- Define the projection once, quoting bounds as SQL literals with format(%L). -- Pinned Prisma DateTime columns are timestamp(3) without time zone, stored UTC. -- Do not copy this interpretation to a differently configured database. @@ -118,7 +127,7 @@ SELECT format($projection$ WHERE "startTime" >= (%L::timestamptz AT TIME ZONE 'UTC') AND "startTime" < (%L::timestamptz AT TIME ZONE 'UTC') ORDER BY "startTime", request_id COLLATE "C" -$projection$, :'detail_projection' || :'routing_projection', :'start_utc', :'end_utc') AS export_query +$projection$, :'detail_projection' || :'routing_projection' || :'tag_projection', :'start_utc', :'end_utc') AS export_query \gset -- Write one client-local file in one pass; JSONL replaces the default CSV. diff --git a/examples/litellm-spend/test-schema.sql b/examples/litellm-spend/test-schema.sql index 4716182..687ab5b 100644 --- a/examples/litellm-spend/test-schema.sql +++ b/examples/litellm-spend/test-schema.sql @@ -21,6 +21,7 @@ CREATE TABLE public."LiteLLM_SpendLogs" ( status text, session_id text, agent_id text, + request_tags jsonb DEFAULT '[]', "startTime" timestamp(3) NOT NULL, "endTime" timestamp(3) NOT NULL, metadata jsonb DEFAULT '{}', diff --git a/examples/litellm-spend/test_native_export.py b/examples/litellm-spend/test_native_export.py index d18993e..71fbc16 100644 --- a/examples/litellm-spend/test_native_export.py +++ b/examples/litellm-spend/test_native_export.py @@ -112,8 +112,8 @@ def apply_documented_grants(container): grants = re.findall( r'GRANT SELECT\s*\([^;]+\)\s+ON TABLE public\."LiteLLM_(?:SpendLogs|DailyUserSpend)"' r"\s+TO fleetdiff_reader;", "\n".join(blocks)) - require(len(grants) == 2 and - sum('public."LiteLLM_SpendLogs"' in grant for grant in grants) == 1, + require(len(grants) == 3 and + sum('public."LiteLLM_SpendLogs"' in grant for grant in grants) == 2, "documented export column grants are missing or ambiguous") psql(container, "-c", '''REVOKE SELECT ON TABLE public."LiteLLM_SpendLogs", public."LiteLLM_DailyUserSpend" FROM fixture_reader;''' + @@ -188,7 +188,7 @@ def setUp(self): def destination(self): return self.parent + "/out-" + uuid.uuid4().hex - def command(self, directory, *, fmt=None, daily=False, routing=False, usage=True, + def command(self, directory, *, fmt=None, daily=False, routing=False, usage=True, tags=False, credential_file=None, url=URL, environment=None, start=START, end=END, extra=(), pid_file=None, user=None): env = {"PATH": "/usr/local/bin:/usr/bin:/bin", "HOME": self.parent, @@ -204,6 +204,8 @@ def command(self, directory, *, fmt=None, daily=False, routing=False, usage=True args.append("--daily") if routing: args.append("--routing-details") + if tags: + args.append("--request-tags") if usage is not None: args.append("--usage-details=" + str(usage).lower()) if credential_file is not None: @@ -295,6 +297,16 @@ def test_all_format_routing_usage_combinations_exact_bytes(self): default = self.export(usage=None) assert_bytes(default["spend.csv"], spend.export_database(self.db)["csv"], "default spend") + def test_request_tag_column_type_is_opt_in(self): + psql(self.db, data=b'''ALTER TABLE public."LiteLLM_SpendLogs" + ALTER COLUMN request_tags DROP DEFAULT, + ALTER COLUMN request_tags TYPE text USING request_tags::text;''') + self.export(tags=True, success=False) + self.export() + psql(self.db, data=b'''ALTER TABLE public."LiteLLM_SpendLogs" DROP COLUMN request_tags;''') + self.export(tags=True, success=False) + self.export() + def test_exact_documented_column_grants_preserve_export_bytes(self): expected = spend.export_database(self.db, details=True, routing=True) daily = self.daily_oracle()["daily.csv"] @@ -304,6 +316,47 @@ def test_exact_documented_column_grants_preserve_export_bytes(self): assert_bytes(actual["spend." + fmt], expected[fmt], "column-granted spend") assert_bytes(actual["daily.csv"], daily, "column-granted daily") + def test_request_tags_exact_bytes_and_optional_grant(self): + psql(self.db, data=b'''UPDATE public."LiteLLM_SpendLogs" SET + request_tags='["agent:alpha", "unrelated-private-tag"]'::jsonb;''') + apply_documented_grants(self.db) + for routing in (False, True): + for usage in (False, True): + oracle = spend.export_database(self.db, details=usage, routing=routing, tags=True) + for fmt in ("csv", "jsonl"): + actual = self.export(fmt=fmt, routing=routing, usage=usage, tags=True) + assert_bytes(actual["spend." + fmt], oracle[fmt], "request tags") + if fmt == "csv": + rows = list(csv.DictReader(io.StringIO(actual["spend.csv"].decode()))) + else: + rows = [json.loads(line) for line in actual["spend.jsonl"].splitlines()] + require(all(json.loads(row["request_tags"]) == + ["agent:alpha", "unrelated-private-tag"] for row in rows), + "tag export lost or expanded tags") + offline = self.parent + "/agent-report.csv" + private_file(self.db, offline, oracle["csv"]) + for source in ("agent_id", "request-tag"): + flags = ["--before-period", "2026-10-07", "--after-period", "2026-10-08", + "--group-by", "agent", "--agent-source", source, + "--rank-by", "recorded-spend-change", "--format", "json", + "--hash-secret-env", "AGENT_TEST_SECRET"] + if source == "request-tag": + flags.extend(("--agent-tag-prefix", "agent:")) + prefix = ["docker", "exec", self.db, "env", "-i", + "PATH=/usr/local/bin:/usr/bin:/bin", "GOMAXPROCS=1", + "FLEETDIFF_LITELLM_DATABASE_URL=" + URL, + "AGENT_TEST_SECRET=synthetic-agent-test-secret-at-least-32-bytes", + "/native/fleetdiff", "investigate"] + local = run([*prefix, "--litellm-spend", offline, *flags]) + native = run([*prefix, "--litellm-db", *flags]) + self.check_result(local, True) + self.check_result(native, True) + assert_bytes(native.stdout, local.stdout, "agent database/file report") + psql(self.db, data=b'''REVOKE SELECT (request_tags) + ON public."LiteLLM_SpendLogs" FROM fixture_reader;''') + self.export() # Default projection still works without the optional grant. + self.export(tags=True, success=False) + def test_half_open_bounds_empty_windows_and_repeatability(self): for start, end in ((START, "2026-10-08T00:00:00Z"), ("2026-10-08T00:00:00Z", END), diff --git a/examples/litellm-spend/test_row_coverage.py b/examples/litellm-spend/test_row_coverage.py index fd1f35f..af4b632 100644 --- a/examples/litellm-spend/test_row_coverage.py +++ b/examples/litellm-spend/test_row_coverage.py @@ -2,10 +2,12 @@ # Code authors: Vijay and Codex """Pinned dropped-row fixture through the built local CLI; no runtime required.""" import csv +import hashlib import json import os from pathlib import Path import tempfile +import tarfile import unittest import coverage_runtime_fixture @@ -67,10 +69,16 @@ class RowCoverageTests(unittest.TestCase): def test_recorded_fixture(self): coverage_runtime_fixture.check() expected = json.loads((ROOT / "coverage_runtime_expected.json").read_text()) - for name, digest in expected["provenance"]["export_source_sha256"].items(): - self.assertEqual(coverage_runtime_fixture.digest(ROOT / name), digest) - for name, digest in expected["provenance"]["runtime_harness_sha256"].items(): - self.assertEqual(coverage_runtime_fixture.digest(ROOT / name), digest) + # Captured provenance describes the original run, not evolving source. + # Current SQL/native parity is exercised by test_native_export.py. + archive = ROOT / "coverage_runtime_sources.tar.gz" + self.assertEqual(coverage_runtime_fixture.digest(archive), + "8c52a2b7c032e34a4232c08c806d6a09eaeb61e463770eb25592b0f1de14695c") + with tarfile.open(archive, "r:gz") as sources: + for group in ("export_source_sha256", "runtime_harness_sha256"): + for name, digest in expected["provenance"][group].items(): + with sources.extractfile("examples/litellm-spend/" + name) as source: + self.assertEqual(hashlib.sha256(source.read()).hexdigest(), digest) @unittest.skipUnless(BIN, "set FLEETDIFF_BIN for built CLI tests") def test_collision_fixture_and_safe_output(self): diff --git a/examples/litellm-spend/test_spend.py b/examples/litellm-spend/test_spend.py index 4b13405..7a2cdb2 100644 --- a/examples/litellm-spend/test_spend.py +++ b/examples/litellm-spend/test_spend.py @@ -318,7 +318,7 @@ def seed_database(container, records): def export_database(container, start="2026-10-07T00:00:00Z", end="2026-10-09T00:00:00Z", details=None, - routing=None): + routing=None, tags=None): exports = {} for fmt in ("csv", "jsonl"): directory = "/tmp/export-" + uuid.uuid4().hex @@ -332,6 +332,8 @@ def export_database(container, start="2026-10-07T00:00:00Z", end="2026-10-09T00: arguments.extend(("-v", "usage_details=" + str(details).lower())) if routing is not None: arguments.extend(("-v", "routing_details=" + str(routing).lower())) + if tags is not None: + arguments.extend(("-v", "request_tags=" + str(tags).lower())) output = psql(container, *arguments, "-f", "/example/export.sql", reader=True) require(not output.strip(), "SQL export leaked output outside its selected file") files = docker("exec", container, "ls", "-1", directory).splitlines() diff --git a/internal/cli/agents.go b/internal/cli/agents.go new file mode 100644 index 0000000..8d31ba8 --- /dev/null +++ b/internal/cli/agents.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package cli + +import ( + "fmt" + "io" + "math/big" + "strconv" + "text/tabwriter" + + "github.com/llm-measurement/fleetdiff/internal/spend" +) + +func agentUSD(v *string) string { + if v == nil { + return "unavailable" + } + return *v +} + +func agentShare(v *string) string { + if v == nil { + return "unavailable" + } + n, ok := new(big.Rat).SetString(*v) + if !ok { + return "unavailable" + } + return new(big.Rat).Mul(n, big.NewRat(100, 1)).FloatString(2) + "%" +} + +func agentCostCoverage(c spend.AgentCostCoverage) string { + return fmt.Sprintf("%d/%d/%d/%d", c.Usable, c.Missing, c.Invalid, c.ZeroUnknown) +} + +func renderAgents(out io.Writer, a *spend.AgentReport) { + fmt.Fprintf(out, "\nAgent attribution (%s); ranking: %s.\n", a.Source, a.RankBy) + if a.LabelsIncluded { + fmt.Fprintln(out, " Sensitive agent labels included; review before sharing.") + } + for _, side := range []struct { + name string + w spend.AgentWindow + }{{"Before", a.Before}, {"After", a.After}} { + c := side.w.Coverage + fmt.Fprintf(out, " %s: %d known, %d unknown, %d ambiguous rows; %d column-absent, %d empty/null, %d without a matching tag.\n", side.name, c.KnownRows, c.UnknownRows, c.AmbiguousRows, c.ColumnAbsentRows, c.EmptyOrNullRows, c.NoMatchingTagRows) + fmt.Fprintf(out, " Repeated agent tags: %d in %d rows; empty suffixes: %d in %d rows.\n", c.RepeatedAgentTags, c.RepeatedAgentTagRows, c.EmptySuffixTags, c.EmptySuffixRows) + usd := func(w spend.Window) string { + if w.Requests > 0 && w.Quality.SpendMissing+w.Quality.SpendInvalid == w.Requests { + return "unavailable" + } + return w.RecordedSpend + } + fmt.Fprintf(out, " Unknown: %d tokens / %s recorded USD; ambiguous: %d tokens / %s recorded USD.\n", side.w.Unknown.Tokens, usd(side.w.Unknown), side.w.Ambiguous.Tokens, usd(side.w.Ambiguous)) + } + fmt.Fprintln(out, " Shares use all usable recorded USD in each period, including unknown, ambiguous and undisplayed agents. Missing/invalid costs remain unquantified; recorded USD is not an invoice.") + fmt.Fprintln(out, " Cost coverage columns: usable/missing/invalid/zero-origin-unknown rows.") + w := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0) + fmt.Fprintln(w, " Agent\tTokens before\tTokens after\tUSD before\tUSD after\tUSD change\tCost coverage before\tCost coverage after\tShare before\tShare after") + for _, c := range a.Contributors { + label := c.Item + if c.Label != "" { + label += " " + strconv.QuoteToASCII(c.Label) + } + if c.Hash != "" { + label += " (" + c.Hash + ")" + } + fmt.Fprintf(w, " %s\t%d\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", label, c.Before.Tokens, c.After.Tokens, agentUSD(c.Before.RecordedUSD), agentUSD(c.After.RecordedUSD), agentUSD(c.Delta), agentCostCoverage(c.Before.CostCoverage), agentCostCoverage(c.After.CostCoverage), agentShare(c.Before.Share), agentShare(c.After.Share)) + } + _ = w.Flush() + fmt.Fprintf(out, " Known agents not displayed: %d -> %d rows; %d -> %d tokens; %s -> %s recorded USD.\n", a.UndisplayedBefore.Requests, a.UndisplayedAfter.Requests, a.UndisplayedBefore.Tokens, a.UndisplayedAfter.Tokens, agentUSD(a.UndisplayedBefore.RecordedUSD), agentUSD(a.UndisplayedAfter.RecordedUSD)) +} diff --git a/internal/cli/agents_test.go b/internal/cli/agents_test.go new file mode 100644 index 0000000..2cb67bf --- /dev/null +++ b/internal/cli/agents_test.go @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package cli + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/llm-measurement/fleetdiff/internal/exportdata" + "github.com/llm-measurement/fleetdiff/internal/spend" +) + +func agentCLIArgs(path string) []string { + return []string{"investigate", "--litellm-spend", path, "--before-period", "2026-10-07", "--after-period", "2026-10-08", "--group-by", "agent", "--agent-source", "request-tag", "--agent-tag-prefix", "agent:", "--rank-by", "recorded-spend-change", "--hash-secret-env", "AGENT_CLI_SECRET"} +} + +func TestAgentCLIFormatsAndPrivacy(t *testing.T) { + t.Setenv("AGENT_CLI_SECRET", "synthetic-agent-secret-for-tests-at-least-32-bytes") + data, err := os.ReadFile("../../examples/litellm-spend/agent-tags.csv") + if err != nil { + t.Fatal(err) + } + rows, err := csv.NewReader(bytes.NewReader(data)).ReadAll() + if err != nil { + t.Fatal(err) + } + var records []map[string]string + for _, row := range rows[1:] { + r := map[string]string{} + for i, k := range rows[0] { + r[k] = row[i] + } + records = append(records, r) + } + var jsonl bytes.Buffer + for _, r := range records { + if json.NewEncoder(&jsonl).Encode(r) != nil { + t.Fatal("encode fixture") + } + } + array, _ := json.Marshal(records) + var baseline string + for _, input := range []struct { + ext string + data []byte + }{{"csv", data}, {"json", array}, {"jsonl", jsonl.Bytes()}} { + path := filepath.Join(t.TempDir(), "agents."+input.ext) + if err := os.WriteFile(path, input.data, 0600); err != nil { + t.Fatal(err) + } + for _, format := range []string{"text", "json"} { + for _, disclose := range []bool{false, true} { + args := append(agentCLIArgs(path), "--format", format) + if disclose { + args = append(args, "--show-labels", "--acknowledge-sensitive-labels", "--routing-audit") + } + code, out, errs := databaseRun(args, nil) + if code != 0 || errs != "" { + t.Fatal("agent report failed", code, errs) + } + for _, secret := range []string{"other:private", "different-id", "before-a", path} { + if strings.Contains(out+errs, secret) { + t.Fatal("unselected identity leaked") + } + } + if strings.Contains(out, "alpha") != disclose { + t.Fatal("label disclosure mismatch") + } + if format == "json" { + var r spend.Report + if json.Unmarshal([]byte(out), &r) != nil || r.Agents.After.Coverage.RepeatedAgentTagRows != 1 || r.Agents.After.Coverage.EmptySuffixRows != 1 || r.After.RecordedSpend != "6.2" { + t.Fatal("partition/coverage mismatch") + } + if !disclose { + if baseline == "" { + baseline = out + } else if out != baseline { + t.Fatal("source format changed report") + } + } + if disclose && r.RoutingAudit.ConfigurationLabelsIncluded { + t.Fatal("agent disclosure expanded to routing labels") + } + } else if !strings.Contains(out, "USD change") || !strings.Contains(out, "Share after") || disclose && !strings.Contains(out, "review before sharing") { + t.Fatal("required columns or warning missing") + } + } + } + } +} + +func TestAgentCLIRejectsBeforeDatabaseDispatch(t *testing.T) { + for _, flags := range [][]string{ + {"--group-by", "agent"}, + {"--group-by", "agent", "--agent-source", "request-tag"}, + {"--group-by", "agent", "--agent-source", "end-user"}, + {"--group-by", "agent", "--agent-source", "agent_id", "--agent-tag-prefix", "PRIVATE_PREFIX"}, + {"--group-by", "agent", "--agent-source", "agent_id", "--rank-by", "recorded-spend"}, + {"--group-by", "agent", "--agent-source", "request-tag", "--agent-tag-prefix", "agent:", "--request-tags=false"}, + {"--show-labels"}, {"--acknowledge-sensitive-labels"}, + {"--show-labels", "--acknowledge-sensitive-labels", "--group-by", "end-user"}, + } { + called := false + code, out, errs := databaseRun(append([]string{"investigate", "--litellm-db"}, flags...), func(context.Context, exportdata.Options) (exportdata.Result, error) { + called = true + return exportdata.Result{}, nil + }) + if code != 2 || called || out != "" || strings.Contains(errs, "PRIVATE_PREFIX") { + t.Fatal("invalid selector reached DB or leaked", code, errs) + } + } +} + +func TestAgentTextEscapesSelectedLabel(t *testing.T) { + var b bytes.Buffer + renderAgents(&b, &spend.AgentReport{Contributors: []spend.AgentContributor{{Item: "agent-1", Label: "name\n\x1b[31m\u202eend"}}}) + if !strings.Contains(b.String(), `"name\n\x1b[31m\u202eend"`) || strings.ContainsAny(b.String(), "\x1b\u202e") { + t.Fatal("label changed terminal structure") + } +} diff --git a/internal/cli/database.go b/internal/cli/database.go index 90daf34..b132db8 100644 --- a/internal/cli/database.go +++ b/internal/cli/database.go @@ -31,6 +31,7 @@ Options: --format csv|jsonl Spend file format (default csv) --daily Include daily row counters --routing-details Include original-route and agent details + --request-tags Include every request tag as private JSON-array text --usage-details=false Omit default-on cache and usage details --database-url-file FILE Read credentials from an owner-only regular file @@ -45,6 +46,7 @@ private temporary data behind. type databaseFlags struct { enabled, daily, routing, usage bool + requestTags bool keep, credentials string } @@ -57,13 +59,14 @@ func (d *databaseFlags) register(f *flag.FlagSet) { f.BoolVar(&d.daily, "daily", false, "") f.BoolVar(&d.routing, "routing-details", false, "") f.BoolVar(&d.usage, "usage-details", true, "") + f.BoolVar(&d.requestTags, "request-tags", false, "") } func (d databaseFlags) validate(f *flag.FlagSet) error { var err error f.Visit(func(v *flag.Flag) { switch v.Name { - case "keep-export", "database-url-file", "daily", "routing-details", "usage-details": + case "keep-export", "database-url-file", "daily", "routing-details", "usage-details", "request-tags": if !d.enabled { err = errors.New("database export options require --litellm-db") } else if (v.Name == "keep-export" || v.Name == "database-url-file") && strings.TrimSpace(v.Value.String()) == "" { @@ -165,6 +168,7 @@ func (d databaseFlags) export(ctx context.Context, p spend.Period, exporter expo r, err := exporter(ctx, exportdata.Options{ Directory: directory, CredentialFile: d.credentials, Start: p.Start, End: p.End, Format: "csv", Daily: d.daily, Routing: true, UsageDetails: d.usage, + RequestTags: d.requestTags, }) if err != nil || ctx.Err() != nil { cleanup() @@ -280,6 +284,7 @@ func runExport(ctx context.Context, args []string, out, errout io.Writer, export f.BoolVar(&opts.Daily, "daily", false, "") f.BoolVar(&opts.Routing, "routing-details", false, "") f.BoolVar(&opts.UsageDetails, "usage-details", true, "") + f.BoolVar(&opts.RequestTags, "request-tags", false, "") if err := f.Parse(args[1:]); err != nil { if errors.Is(err, flag.ErrHelp) { if _, err := io.WriteString(out, exportHelp); err != nil { diff --git a/internal/cli/database_test.go b/internal/cli/database_test.go index 4ed29bd..7b6b9bb 100644 --- a/internal/cli/database_test.go +++ b/internal/cli/database_test.go @@ -78,7 +78,7 @@ func TestDatabaseExportOptions(t *testing.T) { t.Run(map[bool]string{false: "defaults", true: "explicit"}[explicit], func(t *testing.T) { args := []string{"export", "litellm", "--out", filepath.Join(t.TempDir(), "SENTINEL_DIRECTORY")} if explicit { - args = append(args, "--from", "2026-10-01", "--to", "2026-10-15T00:00:00Z", "--format", "jsonl", "--daily", "--routing-details", "--usage-details=false", "--database-url-file", "SENTINEL_CREDENTIAL_FILE") + args = append(args, "--from", "2026-10-01", "--to", "2026-10-15T00:00:00Z", "--format", "jsonl", "--daily", "--routing-details", "--request-tags", "--usage-details=false", "--database-url-file", "SENTINEL_CREDENTIAL_FILE") } var got exportdata.Options calls := 0 @@ -95,10 +95,10 @@ func TestDatabaseExportOptions(t *testing.T) { t.Fatalf("incorrect range/destination: %+v", got) } if explicit { - if got.Format != "jsonl" || !got.Daily || !got.Routing || got.UsageDetails || got.CredentialFile != "SENTINEL_CREDENTIAL_FILE" || got.Start.Format(time.DateOnly) != "2026-10-01" { + if got.Format != "jsonl" || !got.Daily || !got.Routing || !got.RequestTags || got.UsageDetails || got.CredentialFile != "SENTINEL_CREDENTIAL_FILE" || got.Start.Format(time.DateOnly) != "2026-10-01" { t.Fatalf("explicit options lost: %+v", got) } - } else if got.Format != "csv" || got.Daily || got.Routing || !got.UsageDetails || !got.End.Equal(before.UTC().Truncate(24*time.Hour)) && !got.End.Equal(time.Now().UTC().Truncate(24*time.Hour)) { + } else if got.Format != "csv" || got.Daily || got.Routing || got.RequestTags || !got.UsageDetails || !got.End.Equal(before.UTC().Truncate(24*time.Hour)) && !got.End.Equal(time.Now().UTC().Truncate(24*time.Hour)) { t.Fatalf("wrong defaults: %+v", got) } }) @@ -269,6 +269,8 @@ func TestDatabaseReportsMatchExistingFilePaths(t *testing.T) { flags []string }{ {"spend", "investigate", "litellm-spend/synthetic.csv", "", []string{"--before-period", "2026-10-07", "--after-period", "2026-10-08"}}, + {"agent-id", "investigate", "litellm-spend/agents.csv", "", []string{"--before-period", "2026-10-07", "--after-period", "2026-10-08", "--group-by", "agent", "--agent-source", "agent_id", "--rank-by", "recorded-spend-change"}}, + {"agent-tags", "investigate", "litellm-spend/agent-tags.csv", "", []string{"--before-period", "2026-10-07", "--after-period", "2026-10-08", "--group-by", "agent", "--agent-source", "request-tag", "--agent-tag-prefix", "agent:", "--show-labels", "--acknowledge-sensitive-labels"}}, {"coverage", "investigate", "litellm-spend/coverage_runtime_spend.csv", "litellm-spend/coverage_runtime_daily.csv", []string{"--before-period", "2026-10-07", "--after-period", "2026-10-08"}}, {"routing", "investigate", "routing/synthetic.csv", "", []string{"--before-period", "2026-10-01", "--after-period", "2026-10-02", "--routing-audit"}}, {"scope", "investigate", "routing/synthetic.csv", "", []string{"--before-period", "2026-10-01", "--after-period", "2026-10-02", "--scope-route", "coding-default"}}, @@ -308,6 +310,9 @@ func TestDatabaseReportsMatchExistingFilePaths(t *testing.T) { if !opts.Routing || !opts.UsageDetails || opts.Daily != (tc.daily != "") { t.Fatalf("wrong options: %+v", opts) } + if opts.RequestTags != (tc.name == "agent-tags") { + t.Fatal("tag source did not control tag export") + } if _, err := os.Stat(filepath.Dir(opts.Directory)); !errors.Is(err, os.ErrNotExist) { t.Fatalf("temporary parent remains: %v", err) } diff --git a/internal/cli/routing_labels_test.go b/internal/cli/routing_labels_test.go index e13cd4c..857b6a7 100644 --- a/internal/cli/routing_labels_test.go +++ b/internal/cli/routing_labels_test.go @@ -17,7 +17,7 @@ func TestRoutingShowLabelsPrivacyBoundary(t *testing.T) { path := routingCLIInput(t, true) for _, format := range []string{"text", "json"} { var out, errs bytes.Buffer - args := append(routingCLIArgs(path), "--routing-audit", "--routing-by", "route-agent", "--show-labels", "--show-hashes", "--format", format) + args := append(routingCLIArgs(path), "--routing-audit", "--routing-by", "route-agent", "--show-labels", "--acknowledge-sensitive-labels", "--show-hashes", "--format", format) if code := Run(args, &out, &errs); code != 0 || errs.Len() != 0 { t.Fatal("opted-in report failed", code, errs.String()) } diff --git a/internal/cli/run.go b/internal/cli/run.go index c41f369..2652ba4 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -82,10 +82,14 @@ Options: --provider-counts-contract FILE UTC scope and primary counting-basis JSON --before-period PERIOD UTC date or RFC3339 start/end (requires after-period) --after-period PERIOD UTC date or RFC3339 start/end (equal duration) - --group-by KIND key (default), team, user, end-user or session + --group-by KIND key (default), team, user, end-user, session or agent + --agent-source SOURCE agent_id (LiteLLM) or request-tag (explicit) + --agent-tag-prefix PREFIX Exact, case-sensitive prefix for request-tag attribution + --rank-by KIND recorded-spend-change for agent ranking --routing-audit Show original-route distributions and attribution coverage --routing-by KIND route (default) or route-agent; requires routing-audit - --show-labels Include route/agent configuration labels; requires routing-audit + --show-labels Include selected agent labels (or routing-audit labels) + --acknowledge-sensitive-labels Required with show-labels; review before sharing --scope-route ID Investigate one recorded original route (explicit periods) --scope-agent ID Investigate one recorded agent (explicit periods) --period PERIOD Single-period session review: UTC date or RFC3339 start/end @@ -175,6 +179,9 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro beforePeriod := flags.String("before-period", "", "") afterPeriod := flags.String("after-period", "", "") group := flags.String("group-by", "key", "") + agentSource := flags.String("agent-source", "", "") + agentPrefix := flags.String("agent-tag-prefix", "", "") + rankBy := flags.String("rank-by", "", "") secret := flags.String("hash-secret-env", "", "") period := flags.String("period", "", "") budget := flags.String("session-budget-usd", "", "") @@ -182,6 +189,7 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro routingAudit := flags.Bool("routing-audit", false, "") routingBy := flags.String("routing-by", "", "") showLabels := flags.Bool("show-labels", false, "") + ackLabels := flags.Bool("acknowledge-sensitive-labels", false, "") var scopeRoute, scopeAgent string var routeSet, agentSet bool flags.Func("scope-route", "", func(value string) error { @@ -218,6 +226,12 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro if err := database.validate(flags); err != nil { return fail(2, err.Error()) } + if *showLabels != *ackLabels { + return fail(2, "show-labels and acknowledge-sensitive-labels must be supplied together") + } + if *showLabels && !*routingAudit && *group != "agent" { + return fail(2, "show-labels requires agent grouping or routing-audit; identity rankings stay alias-only") + } spendMode, incompatible := database.enabled, false sessionMode, windowSet, emptySessionOption, twoPeriods := false, false, false, false groupExplicit := false @@ -226,8 +240,8 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro routingSet, routingBySet := false, false flags.Visit(func(f *flag.Flag) { switch f.Name { - case "litellm-db", "keep-export", "database-url-file", "daily", "routing-details", "usage-details": - case "routing-audit", "routing-by", "scope-route", "scope-agent", "show-labels": + case "litellm-db", "keep-export", "database-url-file", "daily", "routing-details", "usage-details", "request-tags": + case "routing-audit", "routing-by", "scope-route", "scope-agent": spendMode, routingSet = true, true routingBySet = routingBySet || f.Name == "routing-by" case "litellm-daily-spend": @@ -243,6 +257,8 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro spendMode, windowSet = true, true case "group-by": spendMode, groupExplicit = true, true + case "agent-source", "agent-tag-prefix", "rank-by", "show-labels", "acknowledge-sensitive-labels": + spendMode = true case "litellm-spend", "langfuse-export", "langfuse-tag", "before-period", "after-period", "hash-secret-env": spendMode = true case "format", "top", "show-hashes": @@ -254,6 +270,22 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro } }) if spendMode { + agentOpts := spend.Options{GroupBy: *group, AgentSource: *agentSource, AgentTagPrefix: *agentPrefix, RankBy: *rankBy, ScopeAgent: scopeAgent} + if err := spend.ValidateAgentOptions(agentOpts); err != nil { + return fail(2, err.Error()) + } + if *group == "agent" && (sessionMode || windowSet) { + return fail(2, "agent grouping requires a two-period investigation") + } + tagOptOut := false + flags.Visit(func(f *flag.Flag) { + if f.Name == "request-tags" && !database.requestTags && *agentSource == "request-tag" { + tagOptOut = true + } + }) + if tagOptOut { + return fail(2, "request-tag attribution requires tag export; do not combine with request-tags=false") + } langfuseSet := false flags.Visit(func(f *flag.Flag) { if f.Name == "langfuse-export" || f.Name == "langfuse-tag" { @@ -267,7 +299,7 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro if !groupExplicit { *group = "user_id" } - return runLangfuse(*langfusePath, *group, *langfuseTag, *format, spend.Options{BeforePeriod: *beforePeriod, AfterPeriod: *afterPeriod, SecretEnv: *secret, Top: *top, ShowHashes: *hashes}, out, errout) + return runLangfuse(*langfusePath, *group, *langfuseTag, *format, spend.Options{BeforePeriod: *beforePeriod, AfterPeriod: *afterPeriod, SecretEnv: *secret, Top: *top, ShowHashes: *hashes, AgentSource: *agentSource, AgentTagPrefix: *agentPrefix, RankBy: *rankBy, ShowLabels: *showLabels, AcknowledgeSensitiveLabels: *ackLabels}, out, errout) } if providerSet && (emptyProviderOption || *providerCountsPath == "" || sessionMode || windowSet || routeSet || agentSet) { return fail(2, "provider coverage needs a nonempty aggregate CSV and an unscoped two-period spend investigation") @@ -285,9 +317,6 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro if routingBySet && (!*routingAudit || *routingBy != "route" && *routingBy != "route-agent") { return fail(2, "routing-by must be route or route-agent and requires routing-audit") } - if *showLabels && !*routingAudit { - return fail(2, "show-labels requires routing-audit") - } if routeSet && agentSet || routeSet && (strings.TrimSpace(scopeRoute) == "" || len(scopeRoute) > spend.MaxFieldBytes || database.enabled && !utf8.ValidString(scopeRoute)) || agentSet && (strings.TrimSpace(scopeAgent) == "" || len(scopeAgent) > spend.MaxFieldBytes || database.enabled && !utf8.ValidString(scopeAgent)) { return fail(2, "choose one nonempty route or agent scope of at most 8192 bytes") } @@ -303,7 +332,7 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro } } if database.enabled { - if *format != "text" && *format != "json" || *top < 1 || *top > 100 || !slices.Contains([]string{"key", "team", "user", "end-user", "session"}, *group) { + if *format != "text" && *format != "json" || *top < 1 || *top > 100 || !slices.Contains([]string{"key", "team", "user", "end-user", "session", "agent"}, *group) { return fail(2, "use text or json output, top between 1 and 100, and a supported spend group") } key, _, err := attribution.Secret(*secret) @@ -336,6 +365,9 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro if err := validateDatabaseDaily(p, database.daily); err != nil { return fail(2, err.Error()) } + if *group == "agent" && *agentSource == "request-tag" { + database.requestTags = true + } r, cleanup, err := database.export(ctx, p, exporter) defer cleanup() if err != nil { @@ -350,7 +382,9 @@ func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, erro BeforePeriod: *beforePeriod, AfterPeriod: *afterPeriod, GroupBy: *group, SecretEnv: *secret, Top: *top, ShowHashes: *hashes, DailySpendPath: *dailySpendPath, RoutingAudit: *routingAudit, RoutingBy: *routingBy, ScopeRoute: scopeRoute, ScopeAgent: scopeAgent, - ShowLabels: *showLabels, + ShowLabels: *showLabels, + AcknowledgeSensitiveLabels: *ackLabels, + AgentSource: *agentSource, AgentTagPrefix: *agentPrefix, RankBy: *rankBy, ProviderCountsPath: *providerCountsPath, ProviderModelMapPath: *providerModelMapPath, ProviderContractPath: *providerContractPath, }, out, errout) diff --git a/internal/cli/spend.go b/internal/cli/spend.go index 22ce7f3..e6a31c3 100644 --- a/internal/cli/spend.go +++ b/internal/cli/spend.go @@ -113,14 +113,20 @@ func renderSpend(out io.Writer, r spend.Report) { } fmt.Fprintln(out) } + if r.Agents != nil { + renderAgents(out, r.Agents) + } for _, m := range r.Rankings.Movers { + if r.Agents != nil { + break + } label := m.Item if m.Hash != "" { label += " (" + m.Hash + ")" } fmt.Fprintf(out, " %s: %s -> %s tokens; change %s.\n", label, spendInterval(m.Before), spendInterval(m.After), signedInterval(m.Delta)) } - if c := r.Increase; c != nil && !r.Incomplete { + if c := r.Increase; c != nil && !r.Incomplete && r.Agents == nil { kind := r.GroupBy verb := "accounts" if c.Count != 1 { @@ -149,7 +155,7 @@ func renderSpend(out io.Writer, r spend.Report) { fmt.Fprintf(out, " %s leading tracked %s %s for %s of the net recorded increase.\n", spendNumber(c.Count), kind, verb, share) } } - if len(r.Rankings.Movers) == 0 { + if len(r.Rankings.Movers) == 0 && r.Agents == nil { fmt.Fprintln(out, " No tracked contributors with attributable recorded tokens.") } fmt.Fprintf(out, " Attributed tokens: %s -> %s.\n", spendNumber(r.Before.AttributedTokens), spendNumber(r.After.AttributedTokens)) diff --git a/internal/exportdata/export.go b/internal/exportdata/export.go index 413b146..4a91d59 100644 --- a/internal/exportdata/export.go +++ b/internal/exportdata/export.go @@ -14,6 +14,7 @@ type Options struct { Start, End time.Time Format string Daily, Routing, UsageDetails bool + RequestTags bool } type Result struct { diff --git a/internal/langfuse/agents_test.go b/internal/langfuse/agents_test.go new file mode 100644 index 0000000..5be37a7 --- /dev/null +++ b/internal/langfuse/agents_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package langfuse + +import ( + "strings" + "testing" +) + +func TestAgentTagsProjectScopeAndCoverage(t *testing.T) { + root := t.TempDir() + for _, project := range []string{"one", "two"} { + body := strings.ReplaceAll(rowJSON("a", project, "2026-10-10"), `["PRIVATE_TAG"]`, `["agent:alpha","agent:alpha","agent:"]`) + body += strings.ReplaceAll(rowJSON("b", project, "2026-10-17"), `["PRIVATE_TAG"]`, `["agent:alpha","agent:beta"]`) + addRun(t, root, project, "run", "2026-10-10T00:00:00Z", "2026-10-18T00:00:00Z", body, 2) + } + o := options() + o.AgentSource, o.AgentTagPrefix = "request-tag", "agent:" + r, err := Investigate(root, "agent", "", o) + if err != nil { + t.Fatal(err) + } + if r.Before.Tokens != 280 || r.After.Tokens != 280 || r.Agents.Before.Known.Requests != 2 || r.Agents.Before.Coverage.RepeatedAgentTagRows != 2 || r.Agents.Before.Coverage.EmptySuffixTags != 2 || r.Agents.After.Ambiguous.Requests != 2 || len(r.Rankings.Movers) != 2 { + t.Fatal("project or tag attribution lost", r.Agents) + } + if len(r.Agents.Contributors) != 2 || r.Agents.Contributors[0].Item == r.Agents.Contributors[1].Item { + t.Fatal("projects merged in exact table") + } + o.ShowLabels, o.AcknowledgeSensitiveLabels = true, true + shown, err := Investigate(root, "agent", "", o) + if err != nil || shown.Agents.Contributors[0].Label != "alpha" || shown.Agents.Contributors[1].Label != "alpha" || shown.Before.Tokens != r.Before.Tokens { + t.Fatal("Langfuse disclosure or scope changed", err) + } + o.AgentSource, o.AgentTagPrefix = "agent_id", "" + if _, err := Investigate(root, "agent", "", o); err == nil { + t.Fatal("Langfuse agent_id inference accepted") + } +} + +func TestAgentTagDuplicateNormalizationAndValidation(t *testing.T) { + root := t.TempDir() + body := strings.ReplaceAll(rowJSON("a", "project", "2026-10-10"), `["PRIVATE_TAG"]`, `["agent:alpha", "other:b", "agent:alpha"]`) + duplicate := strings.ReplaceAll(body, `["agent:alpha", "other:b", "agent:alpha"]`, `["other:b","agent:alpha","agent:alpha"]`) + addRun(t, root, "project", "run", "2026-10-10T00:00:00Z", "2026-10-18T00:00:00Z", body+duplicate, 2) + o := options() + o.AgentSource, o.AgentTagPrefix = "request-tag", "agent:" + r, err := Investigate(root, "agent", "", o) + if err != nil || r.Before.Requests != 1 || r.Coverage.Duplicates != 1 || r.Agents.Before.Coverage.RepeatedAgentTags != 1 { + t.Fatal("duplicate tag normalization changed accounting", err) + } + root = t.TempDir() + body = strings.ReplaceAll(rowJSON("a", "project", "2026-10-10"), `["PRIVATE_TAG"]`, `[null]`) + addRun(t, root, "project", "run", "2026-10-10T00:00:00Z", "2026-10-18T00:00:00Z", body, 1) + if _, err := Investigate(root, "agent", "", o); err == nil { + t.Fatal("malformed observation tag array accepted") + } +} diff --git a/internal/langfuse/directory.go b/internal/langfuse/directory.go index 5fe19b0..0387d25 100644 --- a/internal/langfuse/directory.go +++ b/internal/langfuse/directory.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "io/fs" + "maps" "math/big" "os" "path" @@ -453,6 +454,12 @@ func mappedRow(p projection, group, tag string) (spend.Row, [32]byte, projection } } // All group identities are project-scoped; raw labels never reach output. + if group == "agent" { + row.AgentNamespace = project + if raw := p.fields["tags"]; len(raw) > 0 { + v["request_tags"] = string(raw) + } + } v["api_key"] = framed("langfuse", project) if group == "user_id" { if v["user"] != "" { @@ -478,6 +485,19 @@ func mappedRow(p projection, group, tag string) (spend.Row, [32]byte, projection } // Compare normalized approved fields, not encoding-specific raw JSON/CSV. canonical := map[string]any{"accounting/grouping": v, "type": kind, "level": row.ObservationLevel, "start_time": v["startTime"], "end_time": v["endTime"], "total_cost": v["spend"]} + if group == "agent" { + grouping := maps.Clone(v) + if raw, ok := v["request_tags"]; ok { + tags, err := spend.ParseRequestTags(raw) + if err != nil { + return row, digest, projectionFingerprint{}, err + } + slices.Sort(tags) + normalized, _ := json.Marshal(tags) + grouping["request_tags"] = string(normalized) + } + canonical["accounting/grouping"] = grouping + } for _, name := range []string{"user_id", "session_id", "provided_model_name", "environment", "release", "trace_name", "prompt_name"} { s, _ := text(p.fields[name]) canonical[name] = s @@ -532,9 +552,17 @@ func Investigate(dir, group, tag string, opts spend.Options) (Report, error) { if group == "" { group = "user_id" } - if !slices.Contains([]string{"user_id", "session_id", "provided_model_name", "environment", "release", "trace_name", "prompt_name", "tag"}, group) || group == "tag" && (strings.TrimSpace(tag) == "" || len(tag) > MaxFieldBytes) || group != "tag" && tag != "" { + if !slices.Contains([]string{"user_id", "session_id", "provided_model_name", "environment", "release", "trace_name", "prompt_name", "tag", "agent"}, group) || group == "tag" && (strings.TrimSpace(tag) == "" || len(tag) > MaxFieldBytes) || group != "tag" && tag != "" { return Report{}, errors.New("unsupported Langfuse group; tag grouping requires --langfuse-tag") } + if group == "agent" && opts.AgentSource != "request-tag" { + return Report{}, errors.New("Langfuse agent grouping requires agent-source request-tag") + } + check := opts + check.GroupBy = group + if err := spend.ValidateAgentOptions(check); err != nil { + return Report{}, err + } snap, err := loadDirectory(dir, group, tag) if err != nil { return Report{}, err @@ -543,6 +571,9 @@ func Investigate(dir, group, tag string, opts spend.Options) (Report, error) { if opts.GroupBy == "" { opts.GroupBy = "key" } + if group == "agent" { + opts.GroupBy = "agent" + } opts.ModelAliases = group == "provided_model_name" r, err := spend.InvestigateRows(func(visit func(spend.Row) error) error { for _, row := range snap.rows { diff --git a/internal/postgres/export.go b/internal/postgres/export.go index e7402ba..33a3f86 100644 --- a/internal/postgres/export.go +++ b/internal/postgres/export.go @@ -70,7 +70,7 @@ SET LOCAL idle_in_transaction_session_timeout = '120s'; SET LOCAL client_encoding = 'UTF8';`); err != nil { return result, err } - if err = checkSchema(ctx, conn, o.Daily); err != nil { + if err = checkSchema(ctx, conn, o.Daily, o.RequestTags); err != nil { return result, err } name := "spend." + o.Format diff --git a/internal/postgres/query.go b/internal/postgres/query.go index 901fb14..9ebcbcc 100644 --- a/internal/postgres/query.go +++ b/internal/postgres/query.go @@ -40,16 +40,23 @@ func queries(o exportdata.Options) (string, string, error) { if err != nil { return "", "", err } + tags, err := recipeBlock("$tags$") + if err != nil { + return "", "", err + } if !o.UsageDetails { details = "" } if !o.Routing { routing = "" } + if !o.RequestTags { + tags = "" + } if strings.Count(query, "%s") != 1 || strings.Count(query, "%L") != 2 { return "", "", errProjection } - query = strings.Replace(query, "%s", details+routing, 1) + query = strings.Replace(query, "%s", details+routing+tags, 1) query = strings.Replace(query, "%L", sqlTime(o.Start), 1) query = strings.Replace(query, "%L", sqlTime(o.End), 1) daily := litellmsql.DailyRecipe diff --git a/internal/postgres/schema.go b/internal/postgres/schema.go index 7ef7f2d..51bc86a 100644 --- a/internal/postgres/schema.go +++ b/internal/postgres/schema.go @@ -13,7 +13,7 @@ import ( var errSchema = exportdata.ErrSchema -func checkSchema(ctx context.Context, c *pgconn.PgConn, daily bool) error { +func checkSchema(ctx context.Context, c *pgconn.PgConn, daily, requestTags bool) error { spend := map[string]uint32{} for _, name := range []string{"request_id", "call_type", "api_key", "user", "team_id", "organization_id", "end_user", "model", "model_id", "model_group", "custom_llm_provider", "status", "session_id", "agent_id"} { spend[name] = 25 @@ -22,6 +22,9 @@ func checkSchema(ctx context.Context, c *pgconn.PgConn, daily bool) error { spend[name] = 23 } spend["spend"], spend["metadata"], spend["startTime"], spend["endTime"] = 701, 3802, 1114, 1114 + if requestTags { + spend["request_tags"] = 3802 + } notNull := map[string]bool{} for _, name := range []string{"request_id", "call_type", "api_key", "model", "prompt_tokens", "completion_tokens", "total_tokens", "spend", "startTime", "endTime"} { notNull[name] = true diff --git a/internal/postgres/tags_test.go b/internal/postgres/tags_test.go new file mode 100644 index 0000000..c788f43 --- /dev/null +++ b/internal/postgres/tags_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "strings" + "testing" + "time" + + "github.com/llm-measurement/fleetdiff/internal/exportdata" +) + +func TestRequestTagsProjectionOptIn(t *testing.T) { + o := exportdata.Options{Start: time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC), End: time.Date(2026, 10, 9, 0, 0, 0, 0, time.UTC)} + for _, usage := range []bool{false, true} { + for _, routing := range []bool{false, true} { + o.UsageDetails, o.Routing, o.RequestTags = usage, routing, false + base, daily, err := queries(o) + if err != nil || strings.Contains(base, "request_tags") { + t.Fatal("default projection changed", err) + } + o.RequestTags = true + with, withDaily, err := queries(o) + tags, _ := recipeBlock("$tags$") + if err != nil || strings.Replace(with, tags, "", 1) != base || withDaily != daily || strings.Contains(with, "jsonb_array_elements") { + t.Fatal("tag option changes rows or other projection bytes", err) + } + } + } +} diff --git a/internal/spend/agent_spend_test.go b/internal/spend/agent_spend_test.go new file mode 100644 index 0000000..d519a6d --- /dev/null +++ b/internal/spend/agent_spend_test.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package spend + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +func TestAgentSpendChangeAndShares(t *testing.T) { + rows := agentFixtureRows() + o := agentOptions(t) + o.RankBy = "recorded-spend-change" + r := routingReport(t, rows, o) + first := r.Agents.Contributors[0] + if first.Item != "agent-1" || *first.Before.RecordedUSD != "1" || *first.After.RecordedUSD != "3" || *first.Delta != "2" || *first.Before.Share != "0.333333333333" || *first.After.Share != "0.483870967742" { + t.Fatal("USD change or denominator wrong", first) + } + o.Top = 1 + top := routingReport(t, rows, o) + if *top.Agents.Contributors[0].After.Share != *first.After.Share || top.Agents.UndisplayedAfter.Requests != 2 || *top.Agents.UndisplayedAfter.RecordedUSD != "2" { + t.Fatal("top changes accounting or share") + } + // A low-token agent has the largest cost increase, not the largest after cost. + rows = rows[:6] + for i, row := range rows { + row["spend"] = []string{"100", "1", "1", "100", "3", "2"}[i] + } + r = routingReport(t, rows, o) + if r.Agents.Contributors[0].Item != "agent-2" || *r.Agents.Contributors[0].Delta != "2" { + t.Fatal("ranked by after spend or token candidates") + } + for i, row := range rows { + row["spend"] = []string{"0", "0", "0", "94", "3", "3"}[i] + } + r = routingReport(t, rows, o) + if *r.Agents.Contributors[0].After.Share != "0.940000000000" || r.Agents.Contributors[0].Before.Share != nil { + t.Fatal("94 percent or zero denominator wrong") + } +} + +func TestAgentSpendMissingAndDisclosure(t *testing.T) { + rows := agentFixtureRows() + delete(rows[0], "spend") + rows[1]["spend"] = "invalid" + o := agentOptions(t) + r := routingReport(t, rows, o) + for _, c := range r.Agents.Contributors { + if c.Item == "agent-1" && (c.Before.RecordedUSD != nil || c.Delta != nil || c.Before.CostCoverage.Missing != 1) { + t.Fatal("missing cost became zero") + } + } + for _, flags := range [][2]bool{{true, false}, {false, true}} { + o.ShowLabels, o.AcknowledgeSensitiveLabels = flags[0], flags[1] + if _, err := Investigate(writeRows(t, rows), o); err == nil { + t.Fatal("one disclosure flag accepted") + } + } + o.ShowLabels, o.AcknowledgeSensitiveLabels = true, true + r = routingReport(t, rows, o) + b, _ := json.Marshal(r) + if !r.Agents.LabelsIncluded || !strings.Contains(string(b), `"label":"alpha"`) { + t.Fatal("selected label missing") + } + for _, secret := range []string{"different-id", "unrelated-private-tag", "PRIVATE_KEY", "PRIVATE_ROUTE"} { + if strings.Contains(string(b), secret) { + t.Fatal("unselected value leaked") + } + } +} + +func TestAgentCapNamesPrefix(t *testing.T) { + var rows []map[string]string + for i := 0; i <= MaxAgents; i++ { + r := routingSample(fmt.Sprint(i), "2026-09-01", 1, 0) + r["request_tags"] = fmt.Sprintf(`["agent:%d"]`, i) + rows = append(rows, r) + } + _, err := Investigate(writeRows(t, rows), agentOptions(t)) + if err == nil || !strings.Contains(err.Error(), `prefix "agent:"`) || !strings.Contains(err.Error(), "narrower") { + t.Fatal("agent cap missing actionable selector", err) + } +} + +func TestAgentCostAggregateExceedsPerRowBound(t *testing.T) { + rows := agentFixtureRows()[:6] + for _, row := range rows { + row["spend"] = "1000000000000000000" + } + r := routingReport(t, rows, agentOptions(t)) + if r.After.RecordedSpend != "3000000000000000000" || *r.Agents.Contributors[0].After.Share != "0.333333333333" { + t.Fatal("aggregate cost reapplied per-row validation") + } +} diff --git a/internal/spend/agents.go b/internal/spend/agents.go new file mode 100644 index 0000000..6304e9f --- /dev/null +++ b/internal/spend/agents.go @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package spend + +import ( + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "math/big" + "slices" + "strconv" + "strings" + "unicode/utf8" + + "github.com/llm-measurement/fleetdiff/internal/accounting" +) + +const MaxAgents = 1024 + +type AgentCoverage struct { + KnownRows uint64 `json:"known_rows"` + UnknownRows uint64 `json:"unknown_rows"` + AmbiguousRows uint64 `json:"ambiguous_rows"` + ColumnAbsentRows uint64 `json:"column_absent_rows"` + EmptyOrNullRows uint64 `json:"empty_or_null_rows"` + NoMatchingTagRows uint64 `json:"no_matching_tag_rows"` + RepeatedAgentTagRows uint64 `json:"repeated_agent_tag_rows"` + RepeatedAgentTags uint64 `json:"repeated_agent_tags"` + EmptySuffixRows uint64 `json:"empty_suffix_rows"` + EmptySuffixTags uint64 `json:"empty_suffix_tags"` +} + +type AgentWindow struct { + Known Window `json:"known"` + Unknown Window `json:"unknown"` + Ambiguous Window `json:"ambiguous"` + Coverage AgentCoverage `json:"coverage"` +} + +type AgentReport struct { + Schema string `json:"schema"` + Source string `json:"label_source"` + Before AgentWindow `json:"before"` + After AgentWindow `json:"after"` + RankBy string `json:"rank_by"` + LabelsIncluded bool `json:"sensitive_labels_included"` + Contributors []AgentContributor `json:"contributors"` + UndisplayedBefore AgentValues `json:"known_undisplayed_before"` + UndisplayedAfter AgentValues `json:"known_undisplayed_after"` + ShareBasis string `json:"spend_share_basis"` +} + +type AgentCostCoverage struct { + Usable uint64 `json:"usable_rows"` + Missing uint64 `json:"missing_rows"` + Invalid uint64 `json:"invalid_rows"` + ZeroUnknown uint64 `json:"zero_origin_unknown_rows"` +} + +type AgentValues struct { + Requests uint64 `json:"logged_rows"` + Tokens uint64 `json:"recorded_tokens"` + RecordedUSD *string `json:"recorded_usd"` + Share *string `json:"share_of_recorded_usd"` + CostCoverage AgentCostCoverage `json:"cost_coverage"` +} + +type AgentContributor struct { + Item string `json:"item"` + Hash string `json:"hash,omitempty"` + Label string `json:"label,omitempty"` + Before AgentValues `json:"before"` + After AgentValues `json:"after"` + Delta *string `json:"recorded_usd_delta"` +} + +type agentResolution struct { + state, label, source, prefix string + absent, empty bool + repeated, emptySuffix uint64 +} + +// ValidateAgentOptions is shared by the CLI before database dispatch and the +// offline reader. A selector never infers or falls back to a different source. +func ValidateAgentOptions(o Options) error { + if o.GroupBy != "agent" { + if o.AgentSource != "" || o.AgentTagPrefix != "" || o.RankBy != "" { + return errors.New("agent-source, agent-tag-prefix and rank-by require group-by agent") + } + return nil + } + if o.AgentSource != "agent_id" && o.AgentSource != "request-tag" { + return errors.New("group-by agent requires explicit agent-source agent_id or request-tag") + } + if o.AgentSource == "request-tag" { + if strings.TrimSpace(o.AgentTagPrefix) == "" || len(o.AgentTagPrefix) > MaxFieldBytes || !utf8.ValidString(o.AgentTagPrefix) { + return errors.New("request-tag requires a nonempty UTF-8 agent-tag-prefix of at most 8192 bytes") + } + } else if o.AgentTagPrefix != "" { + return errors.New("agent-tag-prefix requires agent-source request-tag") + } + if o.RankBy != "" && o.RankBy != "recorded-spend-change" { + return errors.New("rank-by supports recorded-spend-change for agent grouping") + } + if o.ScopeAgent != "" && o.AgentSource == "request-tag" { + return errors.New("scope-agent selects agent_id; do not combine it with request-tag attribution") + } + return nil +} + +func resolveAgent(row Row, o Options) (*agentResolution, error) { + a := &agentResolution{state: "unknown", source: o.AgentSource, prefix: o.AgentTagPrefix} + field := "agent_id" + if o.AgentSource == "request-tag" { + field = "request_tags" + } + raw, present := row.Values[field] + a.absent = !present + if !present { + return a, nil + } + if len(raw) > MaxFieldBytes || !utf8.ValidString(raw) { + return nil, errors.New("invalid agent source field; use UTF-8 within the 8192-byte field limit") + } + if strings.TrimSpace(raw) == "" || field == "request_tags" && strings.TrimSpace(raw) == "null" { + a.empty = true + return a, nil + } + if field == "agent_id" { + a.state, a.label = "known", raw + return a, nil + } + tags, err := ParseRequestTags(raw) + if err != nil { + return nil, err + } + // Deduplicate matching suffixes; retain no unselected tag values. + labels := map[string]bool{} + for _, tag := range tags { + label, match := strings.CutPrefix(tag, o.AgentTagPrefix) + if !match { + continue + } + if strings.TrimSpace(label) == "" { + a.emptySuffix++ + continue + } + if labels[label] { + a.repeated++ + } + labels[label] = true + } + if len(labels) == 1 { + a.state = "known" + for label := range labels { + a.label = label + } + } else if len(labels) > 1 { + a.state = "ambiguous" + } + return a, nil +} + +// ParseRequestTags validates bounded scalar-array text for SQL and observation +// adapters before attribution or duplicate normalization can discard evidence. +func ParseRequestTags(raw string) ([]string, error) { + if len(raw) > MaxFieldBytes { + return nil, errors.New("request_tags exceeds the 8192-byte field limit") + } + if strings.TrimSpace(raw) == "null" { + return nil, nil + } + var elements []json.RawMessage + if len(raw) > MaxFieldBytes || preflightSpendJSON([]byte(raw), 0) != nil || json.Unmarshal([]byte(raw), &elements) != nil { + return nil, errors.New("invalid request_tags; use a JSON array of strings within the 8192-byte field limit") + } + tags := make([]string, 0, len(elements)) + for _, element := range elements { + var tag string + if len(element) == 0 || element[0] != '"' || json.Unmarshal(element, &tag) != nil { + return nil, errors.New("invalid request_tags; every array element must be a string") + } + tags = append(tags, tag) + } + return tags, nil +} + +type agentPair [2]*aggregate + +func newAgentPair(before, after Period) agentPair { + return agentPair{newAggregate(before, false), newAggregate(after, false)} +} + +type agentEntry struct { + digest [32]byte + label string + pair agentPair + order int +} + +type agentAccumulator struct { + opts Options + before, after Period + known, unknown, ambiguous agentPair + coverage [2]AgentCoverage + entries map[[32]byte]*agentEntry +} + +func newAgentAccumulator(o Options, before, after Period) *agentAccumulator { + if o.GroupBy != "agent" { + return nil + } + return &agentAccumulator{opts: o, before: before, after: after, known: newAgentPair(before, after), unknown: newAgentPair(before, after), ambiguous: newAgentPair(before, after), entries: map[[32]byte]*agentEntry{}} +} + +func (a *agentAccumulator) add(row Row, side int, digest [32]byte, obs accounting.Observation, q Quality) error { + if a == nil { + return nil + } + r, c := row.agent, &a.coverage[side] + if r.absent { + c.ColumnAbsentRows++ + } + if r.empty { + c.EmptyOrNullRows++ + } + if r.repeated > 0 { + c.RepeatedAgentTagRows++ + c.RepeatedAgentTags += r.repeated + } + if r.emptySuffix > 0 { + c.EmptySuffixRows++ + c.EmptySuffixTags += r.emptySuffix + } + var p agentPair + switch r.state { + case "known": + c.KnownRows++ + p = a.known + e := a.entries[digest] + if e == nil { + if len(a.entries) == MaxAgents { + if a.opts.AgentSource == "request-tag" { + return fmt.Errorf("agent attribution exceeds 1024 agents for prefix %s; select a narrower agent-tag-prefix or export", strconv.QuoteToASCII(a.opts.AgentTagPrefix)) + } + return errors.New("agent attribution exceeds 1024 agents; select a narrower export") + } + e = &agentEntry{digest: digest, pair: newAgentPair(a.before, a.after), order: len(a.entries)} + if a.opts.ShowLabels && a.opts.AcknowledgeSensitiveLabels { + e.label = r.label + } + a.entries[digest] = e + } + if err := e.pair[side].add(row, obs, q); err != nil { + return err + } + case "ambiguous": + c.AmbiguousRows++ + p = a.ambiguous + default: + c.UnknownRows++ + if !r.absent && !r.empty && r.source == "request-tag" { + c.NoMatchingTagRows++ + } + p = a.unknown + } + return p[side].add(row, obs, q) +} + +func (a *agentAccumulator) result() *AgentReport { + if a == nil { + return nil + } + r := &AgentReport{Schema: "fleetdiff-agent-attribution/v1", Source: a.opts.AgentSource} + for i, w := range []*AgentWindow{&r.Before, &r.After} { + w.Known, w.Unknown, w.Ambiguous = a.known[i].result(), a.unknown[i].result(), a.ambiguous[i].result() + w.Coverage = a.coverage[i] + } + return r +} + +func (a *agentAccumulator) ordered() []*agentEntry { + var entries []*agentEntry + for _, e := range a.entries { + entries = append(entries, e) + } + slices.SortFunc(entries, func(a, b *agentEntry) int { return a.order - b.order }) + return entries +} + +func agentValues(w Window, total *big.Rat) AgentValues { + c := AgentCostCoverage{Missing: w.Quality.SpendMissing, Invalid: w.Quality.SpendInvalid, ZeroUnknown: w.Quality.SpendZeroUnknown} + c.Usable = w.Requests - c.Missing - c.Invalid + v := AgentValues{Requests: w.Requests, Tokens: w.Tokens, CostCoverage: c} + if c.Usable > 0 || w.Requests == 0 { + usd := w.RecordedSpend + v.RecordedUSD = &usd + n, _ := new(big.Rat).SetString(usd) + v.Share = routingShare(n, total) + } + return v +} + +func (a *agentAccumulator) aliases() map[string]string { + labels := map[string]string{} + if a != nil { + for _, e := range a.ordered() { + labels[fmt.Sprintf("%016x", binary.BigEndian.Uint64(e.digest[:8]))] = fmt.Sprintf("agent-%d", e.order+1) + } + } + return labels +} + +func (a *agentAccumulator) finish(r *Report) error { + if a == nil { + return nil + } + entries := a.ordered() + slices.SortStableFunc(entries, func(x, y *agentEntry) int { + if a.opts.RankBy == "recorded-spend-change" { + available := func(e *agentEntry) bool { + for _, p := range e.pair { + w := p.window + if w.Requests > 0 && w.Quality.SpendMissing+w.Quality.SpendInvalid == w.Requests { + return false + } + } + return true + } + xok, yok := available(x), available(y) + if xok != yok { + if xok { + return -1 + } + return 1 + } + if !xok { + return 0 + } + xd := new(big.Rat).Sub(&x.pair[1].cost, &x.pair[0].cost) + yd := new(big.Rat).Sub(&y.pair[1].cost, &y.pair[0].cost) + return yd.Cmp(xd) + } + xd, yd := int64(x.pair[1].window.Tokens)-int64(x.pair[0].window.Tokens), int64(y.pair[1].window.Tokens)-int64(y.pair[0].window.Tokens) + if xd > yd { + return -1 + } + if xd < yd { + return 1 + } + return 0 + }) + report := a.result() + report.RankBy = a.opts.RankBy + if report.RankBy == "" { + report.RankBy = "recorded-token-change" + } + report.LabelsIncluded = a.opts.ShowLabels && a.opts.AcknowledgeSensitiveLabels + report.ShareBasis = "usable recorded USD in all analyzed rows of each period, including unknown, ambiguous and undisplayed agents; missing and invalid costs remain unquantified" + totals := [2]*big.Rat{} + for i, w := range []Window{r.Before, r.After} { + totals[i], _ = new(big.Rat).SetString(w.RecordedSpend) + } + remaining := [2]*routingTotals{newRoutingTotals(), newRoutingTotals()} + report.Contributors = []AgentContributor{} + for i, e := range entries { + if i >= a.opts.Top { + for side := range remaining { + if err := remaining[side].merge(&routingTotals{a: e.pair[side]}); err != nil { + return err + } + } + continue + } + c := AgentContributor{Item: fmt.Sprintf("agent-%d", e.order+1), Label: e.label, Before: agentValues(e.pair[0].result(), totals[0]), After: agentValues(e.pair[1].result(), totals[1])} + if a.opts.ShowHashes { + c.Hash = fmt.Sprintf("%x", e.digest) + } + if c.Before.RecordedUSD != nil && c.After.RecordedUSD != nil { + delta := costText(new(big.Rat).Sub(&e.pair[1].cost, &e.pair[0].cost)) + c.Delta = &delta + } + report.Contributors = append(report.Contributors, c) + } + report.UndisplayedBefore = agentValues(remaining[0].a.result(), totals[0]) + report.UndisplayedAfter = agentValues(remaining[1].a.result(), totals[1]) + r.Agents = report + return nil +} diff --git a/internal/spend/agents_test.go b/internal/spend/agents_test.go new file mode 100644 index 0000000..139247c --- /dev/null +++ b/internal/spend/agents_test.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package spend + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" +) + +func agentOptions(t *testing.T) Options { + o := routingOptions(t) + o.RoutingAudit = false + o.GroupBy, o.AgentSource, o.AgentTagPrefix = "agent", "request-tag", "agent:" + return o +} + +func TestAgentTagResolution(t *testing.T) { + for _, tc := range []struct { + raw, state, label string + repeats, empty uint64 + }{ + {`[]`, "unknown", "", 0, 0}, + {`["other:a","Agent:a"]`, "unknown", "", 0, 0}, + {`["agent:a","agent:a","agent:"]`, "known", "a", 1, 1}, + {`["agent:a","agent:b","agent:a"]`, "ambiguous", "", 1, 0}, + {`["agent:","agent: "]`, "unknown", "", 0, 2}, + } { + r, err := resolveAgent(Row{Values: map[string]string{"request_tags": tc.raw}}, agentOptions(t)) + if err != nil || r.state != tc.state || r.label != tc.label || r.repeated != tc.repeats || r.emptySuffix != tc.empty { + t.Fatal("tag resolution mismatch", tc, err) + } + } + for _, raw := range []string{`{}`, `[null]`, `[1]`, `"agent:a"`, `["agent:a"`, `["agent:\ud800"]`, strings.Repeat("x", MaxFieldBytes+1)} { + if _, err := resolveAgent(Row{Values: map[string]string{"request_tags": raw}}, agentOptions(t)); err == nil { + t.Fatal("malformed tags accepted") + } + } +} + +func agentFixtureRows() []map[string]string { + var rows []map[string]string + for side, day := range []string{"2026-09-01", "2026-09-02"} { + for i, name := range []string{"alpha", "beta", "gamma"} { + n := 100 + if side == 1 && i == 0 { + n = 300 + } + r := routingSample(fmt.Sprintf("%d-%d", side, i), day, n, 0) + r["request_tags"] = `["agent:` + name + `","unrelated-private-tag"]` + r["agent_id"], r["spend"] = "different-id", fmt.Sprint(n/100) + rows = append(rows, r) + } + } + for i, tags := range []string{`["unrelated-private-tag"]`, `["agent:alpha","agent:beta"]`} { + r := routingSample(fmt.Sprint("extra", i), "2026-09-02", 50+i*20, 0) + r["request_tags"], r["spend"] = tags, []string{"0.50", "0.70"}[i] + rows = append(rows, r) + } + return rows +} + +func TestAgentPartitionReconciles(t *testing.T) { + rows, opts := agentFixtureRows(), agentOptions(t) + r := routingReport(t, rows, opts) + w := r.Agents.After + if r.After.Tokens != 620 || r.After.RecordedSpend != "6.2" || w.Known.Tokens != 500 || w.Known.RecordedSpend != "5" || w.Unknown.Tokens != 50 || w.Unknown.RecordedSpend != "0.5" || w.Ambiguous.Tokens != 70 || w.Ambiguous.RecordedSpend != "0.7" { + t.Fatal("exclusive partition did not reconcile") + } + if w.Coverage.KnownRows != 3 || w.Coverage.UnknownRows != 1 || w.Coverage.AmbiguousRows != 1 { + t.Fatal("coverage mismatch") + } + opts.GroupBy, opts.AgentSource, opts.AgentTagPrefix = "key", "", "" + base := routingReport(t, rows, opts) + r.Before.Quality.MissingIdentity, r.After.Quality.MissingIdentity = 0, 0 + r.Before.AttributedTokens, r.After.AttributedTokens = base.Before.AttributedTokens, base.After.AttributedTokens + r.Before.Distinct, r.After.Distinct = base.Before.Distinct, base.After.Distinct + if !reflect.DeepEqual(r.Before, base.Before) || !reflect.DeepEqual(r.After, base.After) { + t.Fatal("grouping changed base accounting") + } + data, _ := json.Marshal(r) + if strings.Contains(string(data), "unrelated-private-tag") || strings.Contains(string(data), "alpha") { + t.Fatal("default agent labels leaked") + } +} + +func TestAgentColumnAbsenceAndSourceChoice(t *testing.T) { + rows := agentFixtureRows()[:6] + for _, row := range rows { + delete(row, "request_tags") + } + o := agentOptions(t) + r := routingReport(t, rows, o) + if r.Agents.After.Coverage.ColumnAbsentRows != 3 || r.Agents.After.Unknown.Requests != 3 { + t.Fatal("missing column was inferred") + } + o.AgentSource, o.AgentTagPrefix = "agent_id", "" + r = routingReport(t, rows, o) + if r.Agents.After.Known.Requests != 3 { + t.Fatal("explicit agent_id not selected") + } +} diff --git a/internal/spend/investigate.go b/internal/spend/investigate.go index 74fb9d2..d279b7c 100644 --- a/internal/spend/investigate.go +++ b/internal/spend/investigate.go @@ -33,11 +33,13 @@ type Options struct { ProviderCountsPath, ProviderModelMapPath string ProviderContractPath string ScopeRoute, ScopeAgent string + AgentSource, AgentTagPrefix, RankBy string RoutingAudit bool RoutingBy string Top int ShowHashes bool ShowLabels bool + AcknowledgeSensitiveLabels bool // Row adapters may share model aliases without changing group identities. ModelAliases bool Now time.Time @@ -156,6 +158,14 @@ func requestDigest(key []byte, id string) [32]byte { } func groupIdentity(row Row, group string, key []byte) ([32]byte, bool, error) { + if group == "agent" { + if row.agent == nil || row.agent.state != "known" { + return [32]byte{}, false, nil + } + a := row.agent + digest, err := identity(key, "agent", a.source, a.prefix, row.AgentNamespace, a.label) + return digest, true, err + } field := map[string]string{"key": "api_key", "team": "team_id", "user": "user", "end-user": "end_user", "session": "session_id"}[group] value := row.Values[field] if strings.TrimSpace(value) == "" { @@ -181,7 +191,7 @@ func Investigate(path string, opts Options) (Report, error) { // InvestigateRows composes an offline source with the existing investigation. // The reader must replay the same completed snapshot on every pass. func InvestigateRows(read func(func(Row) error) error, opts Options) (Report, error) { - if read == nil || opts.DailySpendPath != "" || opts.RoutingAudit || opts.RoutingBy != "" || opts.ScopeRoute != "" || opts.ScopeAgent != "" || opts.ShowLabels || opts.ProviderCountsPath != "" || opts.ProviderModelMapPath != "" || opts.ProviderContractPath != "" { + if read == nil || opts.DailySpendPath != "" || opts.RoutingAudit || opts.RoutingBy != "" || opts.ScopeRoute != "" || opts.ScopeAgent != "" || opts.ShowLabels && opts.GroupBy != "agent" || opts.ProviderCountsPath != "" || opts.ProviderModelMapPath != "" || opts.ProviderContractPath != "" { return Report{}, errors.New("row sources do not support SQL coverage, provider counts or routing options") } return investigate("", opts, func(_ string, visit func(Row) error) error { return read(visit) }) @@ -194,8 +204,11 @@ func investigate(path string, opts Options, source func(string, func(Row) error) if opts.GroupBy == "" { opts.GroupBy = "key" } - if !slices.Contains([]string{"key", "team", "user", "end-user", "session"}, opts.GroupBy) || opts.Top < 1 || opts.Top > 100 { - return Report{}, errors.New("group-by must be key, team, user, end-user or session; top must be 1-100") + if !slices.Contains([]string{"key", "team", "user", "end-user", "session", "agent"}, opts.GroupBy) || opts.Top < 1 || opts.Top > 100 { + return Report{}, errors.New("group-by must be key, team, user, end-user, session or agent; top must be 1-100") + } + if err := ValidateAgentOptions(opts); err != nil { + return Report{}, err } if err := validateRoutingOptions(&opts); err != nil { return Report{}, err @@ -218,12 +231,25 @@ func investigate(path string, opts Options, source func(string, func(Row) error) opts.ScopeRoute, opts.ScopeAgent = "", "" routing := newRoutingAccumulator(opts, selection) read := Read - if routing != nil { + if routing != nil || opts.GroupBy == "agent" { read = ReadRouting } if source != nil { read = source } + if opts.GroupBy == "agent" { + base := read + read = func(path string, visit func(Row) error) error { + return base(path, func(row Row) error { + var err error + row.agent, err = resolveAgent(row, opts) + if err != nil { + return err + } + return visit(row) + }) + } + } var earliest, latest time.Time firstDigest := sha256.New() automatic := opts.BeforePeriod == "" && opts.AfterPeriod == "" @@ -259,6 +285,7 @@ func investigate(path string, opts Options, source func(string, func(Row) error) return Report{}, err } windows := [2]*aggregate{newAggregate(before, true), newAggregate(after, true)} + agents := newAgentAccumulator(opts, before, after) models := map[[32]byte][2]*aggregate{} modelOrder := map[[32]byte]int{} modelLabels := map[[32]byte]string{} @@ -345,6 +372,9 @@ func investigate(path string, opts Options, source func(string, func(Row) error) if e = w.add(row, a, q); e != nil { return e } + if e = agents.add(row, side, group, a, q); e != nil { + return e + } if present { h := binary.BigEndian.Uint64(group[:8]) w.distinct.AddHash(h) @@ -402,6 +432,7 @@ func investigate(path string, opts Options, source func(string, func(Row) error) return Report{}, errors.New("export changed while reading; use a completed immutable export") } r.Before, r.After = windows[0].result(), windows[1].result() + r.Agents = agents.result() r.RowCoverage = coverage.result() r.ProviderCoverage, err = provider.result(r.RowCoverage) if err != nil { @@ -504,7 +535,7 @@ func investigate(path string, opts Options, source func(string, func(Row) error) r.Cache.Rankings.Movers = r.Cache.Rankings.Movers[:min(opts.Top, len(r.Cache.Rankings.Movers))] groups = append(groups, r.Cache.Rankings.Movers) } - aliases := map[string]string{} + aliases := agents.aliases() for _, movers := range groups { for i := range movers { m := &movers[i] @@ -531,6 +562,9 @@ func investigate(path string, opts Options, source func(string, func(Row) error) "Recorded spend is the exact decimal sum of usable source values in USD, not an invoice. Default-zero costs have unknown origin.", "Rankings describe tracked candidates; concentration uses the net recorded increase and can exceed 100% when other contributors decreased.", } + if err := agents.finish(&r); err != nil { + return Report{}, err + } return r, nil } diff --git a/internal/spend/reader.go b/internal/spend/reader.go index 5f05eb9..45e1ef1 100644 --- a/internal/spend/reader.go +++ b/internal/spend/reader.go @@ -40,9 +40,18 @@ type Row struct { // Set only by observation adapters, never by SQL/CSV status projections. // Severity does not establish provider completion or billing finality. ObservationLevel string + // Set by the Langfuse adapter only, preserving project-scoped agents. + AgentNamespace string + agent *agentResolution } func (r Row) replayProjection() any { + if r.AgentNamespace != "" { + return struct { + Values map[string]string + ObservationLevel, AgentNamespace string + }{r.Values, r.ObservationLevel, r.AgentNamespace} + } if r.ObservationLevel == "" { return r.Values } @@ -80,7 +89,7 @@ func allowedColumn(name string) bool { case "request_id", "call_type", "api_key", "user", "team_id", "organization_id", "end_user", "model", "model_id", "model_group", "custom_llm_provider", "prompt_tokens", "completion_tokens", "total_tokens", "spend", "status", "session_id", "startTime", "endTime", "cache_read_input_tokens", - "cache_write_input_tokens", "reasoning_output_tokens", "agent_id", "rejection_kind": + "cache_write_input_tokens", "reasoning_output_tokens", "agent_id", "rejection_kind", "request_tags": return true } return false @@ -95,7 +104,7 @@ func routingReaderColumn(name string) bool { } func retainSpendEmpty(name string, routing bool) bool { - if name == "rejection_kind" { + if name == "rejection_kind" || name == "request_tags" { return true } return routing && (routingReaderColumn(name) || name == "agent_id" || name == "model" || name == "model_id") diff --git a/internal/spend/report.go b/internal/spend/report.go index ce14f7d..dff11a7 100644 --- a/internal/spend/report.go +++ b/internal/spend/report.go @@ -49,6 +49,7 @@ type IncreaseShare struct { } type Report struct { + Agents *AgentReport `json:"agents,omitempty"` AnalysisGap string `json:"analysis_unavailable,omitempty"` RowCoverage *RowCoverage `json:"row_coverage,omitempty"` Cache compare.CacheView `json:"cache"` diff --git a/internal/spend/routing.go b/internal/spend/routing.go index 03f413f..ca0dca0 100644 --- a/internal/spend/routing.go +++ b/internal/spend/routing.go @@ -459,7 +459,7 @@ func newRoutingAccumulator(opts Options, selection *scopeSelection) *routingAccu a.totals = newRoutingPair() a.routes, a.models = map[routingID]int{}, map[routingID]int{} a.modelLabels = map[routingID]string{} - a.showLabels = opts.ShowLabels + a.showLabels = opts.ShowLabels && opts.GroupBy != "agent" if a.showLabels { a.routeLabels, a.agentLabels = map[routingID]string{}, map[routingID]string{} } diff --git a/internal/spend/routing_labels_test.go b/internal/spend/routing_labels_test.go index a5d327d..9edaaba 100644 --- a/internal/spend/routing_labels_test.go +++ b/internal/spend/routing_labels_test.go @@ -56,6 +56,7 @@ func TestRoutingLabelsOptInAndAccountingUnchanged(t *testing.T) { baseline := routingReport(t, rows, o) assertRoutingPrivate(t, baseline) o.ShowLabels = true + o.AcknowledgeSensitiveLabels = true shown := routingReport(t, rows, o) c := shown.RoutingAudit.Cohorts[0] if !shown.RoutingAudit.ConfigurationLabelsIncluded || c.Route.Label != "PRIVATE_ROUTE" || c.Agent == nil || c.Agent.Label != "PRIVATE_AGENT" { @@ -78,6 +79,7 @@ func TestRoutingLabelsOptInAndAccountingUnchanged(t *testing.T) { func TestRoutingUnknownAndAmbiguousLabelsRemainAliased(t *testing.T) { o := routingOptions(t) o.ShowLabels, o.RoutingBy = true, "route-agent" + o.AcknowledgeSensitiveLabels = true rows := []map[string]string{routingSample("a", "2026-09-01", 90, 10), routingSample("b", "2026-09-02", 90, 10)} rows[0]["model"], rows[1]["model"] = "gpt-6.1-sol", "gpt-6.1-sol" delete(rows[0], "model_id") diff --git a/internal/spend/selection.go b/internal/spend/selection.go index 97d1888..937e055 100644 --- a/internal/spend/selection.go +++ b/internal/spend/selection.go @@ -16,8 +16,11 @@ type scopeSelection struct { } func validateRoutingOptions(opts *Options) error { - if opts.ShowLabels && !opts.RoutingAudit { - return errors.New("show-labels requires routing-audit") + if opts.ShowLabels != opts.AcknowledgeSensitiveLabels { + return errors.New("show-labels and acknowledge-sensitive-labels must be supplied together") + } + if opts.ShowLabels && !opts.RoutingAudit && opts.GroupBy != "agent" { + return errors.New("show-labels requires agent grouping or routing-audit; user, session, end-user and key rankings stay alias-only") } if opts.RoutingBy != "" && !opts.RoutingAudit { return errors.New("routing-by requires routing-audit") From 55cd14aa87333265dbe8ec157b714c845b9f51a2 Mon Sep 17 00:00:00 2001 From: Vijay Date: Wed, 28 Oct 2026 21:04:20 -0400 Subject: [PATCH 2/7] test: cover agent attribution in restricted network smoke Use both disclosure flags in the routing smoke command and exercise agent ID, tag partitions and selected label disclosure with and without networking. Code authors: Vijay and Codex. Signed-off-by: Vijay --- scripts/restricted-smoke.sh | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/scripts/restricted-smoke.sh b/scripts/restricted-smoke.sh index c4a79a4..01ed207 100644 --- a/scripts/restricted-smoke.sh +++ b/scripts/restricted-smoke.sh @@ -143,7 +143,8 @@ check_report routing-audit '.routing_audit.model_rows.before.requests == 3 and check_case routing-labels 0 investigate --litellm-spend /examples/routing/synthetic.csv \ --before-period 2026-10-01 --after-period 2026-10-02 --routing-audit \ - --routing-by route-agent --show-labels --hash-secret-env FLEETDIFF_SMOKE_SECRET --format json + --routing-by route-agent --show-labels --acknowledge-sensitive-labels \ + --hash-secret-env FLEETDIFF_SMOKE_SECRET --format json check_report routing-labels '.routing_audit.configuration_labels_included == true and .routing_audit.cohorts[0].route.label == "coding-default" and .routing_audit.cohorts[0].agent.label == "agent-a" and @@ -162,4 +163,31 @@ check_case what-if-routing 0 what-if --litellm-spend /examples/routing/synthetic check_report what-if-routing '.covered_observations == 2 and .excluded_observations.missing_split == 1 and .excluded_observations.unsupported_call_type == 1 and .moves[0].scoped == true' -printf 'PASS restricted offline binary smoke: 14 cases (%s)\n' "$platform" +check_case agent-id 0 investigate --litellm-spend /examples/litellm-spend/agents.csv \ + --before-period 2026-10-07 --after-period 2026-10-08 --group-by agent \ + --agent-source agent_id --rank-by recorded-spend-change \ + --hash-secret-env FLEETDIFF_SMOKE_SECRET --format json +check_report agent-id '.agents.label_source == "agent_id" and + .agents.contributors[0].recorded_usd_delta == "2" and + .agents.contributors[0].after.recorded_usd == "3" and + (.agents.contributors[0] | has("label") | not)' + +check_case agent-tags 0 investigate --litellm-spend /examples/litellm-spend/agent-tags.csv \ + --before-period 2026-10-07 --after-period 2026-10-08 --group-by agent \ + --agent-source request-tag --agent-tag-prefix 'agent:' --rank-by recorded-spend-change \ + --hash-secret-env FLEETDIFF_SMOKE_SECRET --format json +check_report agent-tags '.after.recorded_tokens == 620 and .after.recorded_spend_usd == "6.2" and + .agents.after.known.recorded_tokens == 500 and + .agents.after.unknown.recorded_tokens == 50 and + .agents.after.ambiguous.recorded_tokens == 70 and + .agents.after.coverage.repeated_agent_tag_rows == 1 and + .agents.after.coverage.empty_suffix_rows == 1' + +check_case agent-labels 0 investigate --litellm-spend /examples/litellm-spend/agent-tags.csv \ + --before-period 2026-10-07 --after-period 2026-10-08 --group-by agent \ + --agent-source request-tag --agent-tag-prefix 'agent:' --rank-by recorded-spend-change \ + --show-labels --acknowledge-sensitive-labels \ + --hash-secret-env FLEETDIFF_SMOKE_SECRET --format json +check_report agent-labels '.agents.sensitive_labels_included == true and + .agents.contributors[0].label == "alpha"' +printf 'PASS restricted offline binary smoke: 17 cases (%s)\n' "$platform" From 55701b76961c651d2c486bd83bd5f66ed0097790 Mon Sep 17 00:00:00 2001 From: Vijay Date: Wed, 28 Oct 2026 21:21:11 -0400 Subject: [PATCH 3/7] ci: allow expanded pinned SQL acceptance to finish Keep all assertions and cases. Allow fifteen minutes for the hosted SQL steps including pinned image pulls; retain existing job-wide limits. Clarify that the new tag transport and two-flag disclosure target the next minor, not the released v0.13.1 binary. No trigger or publication condition changes. Code authors: Vijay and Codex. Signed-off-by: Vijay --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- docs/NATIVE_EXPORT.md | 2 ++ docs/ROUTING.md | 2 ++ examples/litellm-spend/README.md | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec529e4..f00999c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: FLEETDIFF_BIN="$PWD/bin/fleetdiff" python3 -m unittest discover -s examples/litellm-spend -p 'test_*.py' -v - name: Pinned LiteLLM writer to read-only SQL export if: runner.os == 'Linux' - timeout-minutes: 10 + timeout-minutes: 15 run: | docker pull ghcr.io/berriai/litellm@sha256:625981c83410a3ea68eb0697590a57ec1d764d634514d54fa5db0591077ee839 docker pull postgres@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e6fe9ad..c708b93 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -122,7 +122,7 @@ jobs: run: go test ./internal/cli -run '^TestReleaseBinary' -count=1 -v - name: Pinned SQL export through the packaged importer if: matrix.os == 'linux' && matrix.arch == 'amd64' - timeout-minutes: 10 + timeout-minutes: 15 env: FLEETDIFF_BIN: ${{ github.workspace }}/unpacked/fleetdiff FLEETDIFF_LINUX_BIN: ${{ github.workspace }}/unpacked/fleetdiff diff --git a/docs/NATIVE_EXPORT.md b/docs/NATIVE_EXPORT.md index a157f4e..6c72695 100644 --- a/docs/NATIVE_EXPORT.md +++ b/docs/NATIVE_EXPORT.md @@ -220,6 +220,8 @@ totals or identity hashing. Database sorting can also affect export time. ## Limits And Failures - `--request-tags` includes every request tag on a row, not only agent tags. + This option targets the next minor release; use the matching source checkout + until it is released. It is opt-in; default exports omit the column. The manual recipe uses `-v request_tags=true` for the same JSON-array-text projection. diff --git a/docs/ROUTING.md b/docs/ROUTING.md index 2814238..31ac708 100644 --- a/docs/ROUTING.md +++ b/docs/ROUTING.md @@ -22,6 +22,8 @@ fleetdiff investigate --litellm-spend ./spend.csv \ Add `--routing-by route-agent` to separate the known agents within each route. Add `--show-labels --acknowledge-sensitive-labels` for a local report with the recorded route and agent names. +The two-flag form targets the next minor release. Released v0.13.1 uses +`--show-labels` alone for routing labels; agent contributor labels are unreleased. The usual `--group-by` still controls the existing contributor rankings; it does not change the routing table. `--format json` includes the same evidence as an additive `routing_audit` object. Default reports omit this object and section. diff --git a/examples/litellm-spend/README.md b/examples/litellm-spend/README.md index 4b174cf..d3a7858 100644 --- a/examples/litellm-spend/README.md +++ b/examples/litellm-spend/README.md @@ -216,7 +216,7 @@ default to zero. No missing-path `NULL` should be interpreted as zero cache use. ### Agent Identity And Rejection Evidence -For tag-based attribution, opt in with `-v request_tags=true` in the same psql +For next-minor tag-based attribution, opt in with `-v request_tags=true` in the same psql export, or `--request-tags` in native export. Both add the nullable `request_tags` column as JSON-array text, without expanding rows. The read-only column grant is documented in [Administrator Setup](../../docs/NATIVE_EXPORT.md#administrator-setup). From 247eacbd3e14bf4f59feef53aa21857fb382a485 Mon Sep 17 00:00:00 2001 From: Vijay Date: Wed, 28 Oct 2026 21:25:42 -0400 Subject: [PATCH 4/7] ci: preserve the published calibration workflow binding Restore ci.yml exactly to the published manifest. Main CI already passed its existing SQL limit; only packaged verification retains the bounded fifteen-minute allowance. Leave all calibration evidence and assertions unchanged. Code authors: Vijay and Codex. Signed-off-by: Vijay --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f00999c..ec529e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: FLEETDIFF_BIN="$PWD/bin/fleetdiff" python3 -m unittest discover -s examples/litellm-spend -p 'test_*.py' -v - name: Pinned LiteLLM writer to read-only SQL export if: runner.os == 'Linux' - timeout-minutes: 15 + timeout-minutes: 10 run: | docker pull ghcr.io/berriai/litellm@sha256:625981c83410a3ea68eb0697590a57ec1d764d634514d54fa5db0591077ee839 docker pull postgres@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 From af475aa8420c349d4b2b07247f93894a87f4da30 Mon Sep 17 00:00:00 2001 From: Vijay Date: Thu, 29 Oct 2026 18:09:59 -0400 Subject: [PATCH 5/7] fix: address agent attribution review findings Signed-off-by: Vijay --- CHANGELOG.md | 7 +- docs/AGENT_ATTRIBUTION.md | 10 +- examples/litellm-spend/TESTING.md | 31 ++++-- .../coverage_runtime_sources.tar.gz | Bin 25726 -> 0 bytes .../coverage_runtime_sources.tar.gz.license | 6 -- examples/litellm-spend/test_native_export.py | 36 +++++++ examples/litellm-spend/test_row_coverage.py | 16 +-- internal/cli/agents.go | 40 +++++++- internal/cli/agents_test.go | 96 ++++++++++++++++++ internal/cli/spend.go | 15 +++ internal/spend/agent_spend_test.go | 26 +++++ internal/spend/agents.go | 37 +++++-- internal/spend/agents_test.go | 25 ++++- 13 files changed, 303 insertions(+), 42 deletions(-) delete mode 100644 examples/litellm-spend/coverage_runtime_sources.tar.gz delete mode 100644 examples/litellm-spend/coverage_runtime_sources.tar.gz.license diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e9b036..24b50ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,11 @@ Notable user-visible changes are recorded here, newest first. - Add opt-in request-tag export through the shared psql/native projection, and an agent table with exact recorded USD, coverage, period shares and `--rank-by recorded-spend-change`. Default exports and reports are unchanged. -- Require `--acknowledge-sensitive-labels` together with `--show-labels` for - raw agent or routing labels. User/session/end-user/key rankings remain aliased. +- **Breaking change:** routing `--show-labels` now also requires + `--acknowledge-sensitive-labels`. Replace `--routing-audit --show-labels` with + `--routing-audit --show-labels --acknowledge-sensitive-labels` on existing + `fleetdiff investigate` commands. Raw agent labels require the same two flags. + User/session/end-user/key rankings remain aliased. See the [agent attribution contract](docs/AGENT_ATTRIBUTION.md). ## [0.13.1] - 2026-10-27 diff --git a/docs/AGENT_ATTRIBUTION.md b/docs/AGENT_ATTRIBUTION.md index 31feb2a..380e48f 100644 --- a/docs/AGENT_ATTRIBUTION.md +++ b/docs/AGENT_ATTRIBUTION.md @@ -44,7 +44,8 @@ observations. No request/observation join is inferred between sources. The default agent table orders by signed recorded-token change. `--rank-by recorded-spend-change` orders by signed change in usable recorded USD, descending, with first appearance breaking ties. An unavailable cost delta sorts -after available deltas. Before/after USD, delta, cost coverage and each period's +after available deltas and is listed separately, never hidden by `--top`. +Before/after USD, delta, cost coverage and each period's share appear beside tokens. A period with no usable cost rows is unavailable, not a recorded zero. Known agents outside `--top` are summed separately. @@ -66,8 +67,11 @@ arbitrary metadata extraction or automatic agent discovery is performed. The agent table is exact and bounded to 1,024 known agents across both periods, including project-scoped agents. Overflow fails and asks for a narrower prefix -or export; the error quotes the selected prefix. Malformed/non-string tag arrays -or tag data exceeding the existing 8 KiB field bound fail, without raw row data. +or export; the error quotes the selected prefix. Malformed JSON, non-array tag +data or data exceeding the existing 8 KiB field bound fail, without raw row data. +LiteLLM arrays containing any non-string element count as unknown identity with +separate `non_string_tag_rows` coverage, even if they also contain an agent tag; +no label is salvaged. Langfuse tag arrays retain their string-only validation. An old export without tags remains readable and reports column-absent unknown identity. The optional `agents` JSON section is versioned independently as `fleetdiff-agent-attribution/v1`; collector summaries and the base accounting diff --git a/examples/litellm-spend/TESTING.md b/examples/litellm-spend/TESTING.md index 027009e..7e5c5b2 100644 --- a/examples/litellm-spend/TESTING.md +++ b/examples/litellm-spend/TESTING.md @@ -4,12 +4,31 @@ # LiteLLM Spend Export Checks The captured row-coverage CSVs and `coverage_runtime_expected.json` retain their -original observations and provenance. `coverage_runtime_sources.tar.gz` retains -the source bytes bound by that provenance; tests verify each original digest -without treating later source edits as a new historical run. Current native -export and psql recipe compatibility are exercised separately by the pinned -database checks below. The archive contains source only and is never unpacked -to the filesystem or executed by tests. +original observations and provenance. The captured export and runtime harness +source hashes refer to commit +`73df8c41de98e55758291d5282314d00347e4a27`, not this evolving checkout. Inspect +that commit's `examples/litellm-spend/` source to reproduce those historical +digests; a later source edit is not a new historical run. No archived source +blob is needed. Current native export and psql recipe compatibility are +exercised separately by the pinned database checks below. + +## Agent Tag Shape + +LiteLLM v1.104.0 can retain non-string request-tag elements. Its +[`LiteLLM_SpendLogs` schema](https://github.com/BerriAI/litellm/blob/v1.104.0/schema.prisma#L667) +declares `request_tags` as nullable JSON, not a string array. Its +[spend writer](https://github.com/BerriAI/litellm/blob/v1.104.0/litellm/proxy/spend_tracking/spend_tracking_utils.py#L576) +serializes `metadata.tags` when it is a list without validating its elements; +the standard-logging-payload override is likewise JSON-serialized. The payload +type stores the result as a JSON string. Thus a mixed array is a possible +stored source value, not proof that the export itself is corrupt. + +Agent attribution keeps the entire such LiteLLM row in unknown identity with +`non_string_tag_rows` coverage; it does not salvage the string elements. Tests +exercise null, numeric, boolean, object and nested-array elements, mixed valid +labels, exact token/USD preservation and text/JSON privacy across CSV/JSON/JSONL. +Malformed JSON, invalid Unicode and oversized fields still fail. The Langfuse +adapter keeps its existing string-only tag validation and GENERATION boundary. ## Documentation Checks diff --git a/examples/litellm-spend/coverage_runtime_sources.tar.gz b/examples/litellm-spend/coverage_runtime_sources.tar.gz deleted file mode 100644 index f9446aa95a2c679d9f8e12d70ea7e984ac8df165..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25726 zcmV(zK<2+6iwFP!000001MR(gU)xC1D7=63DS9>8wVl|;CXfqR&n^Kc!&!oP44E5n z^b@uW+Srm;l9Mnj?`MDO(%tIT#WtDDobxuGf5P2C&c z8~gE}zW%2JpJ&gW(O>xF|H8}dKXrGuo_Dt3zYdn`ZtrxT{^|NR=u6?VNK!9`CgUhd zmz7)fK0mnsCZC-xF^Xn0KTLbipN=OlM%&%7|MG>uv-5oC#n#L2cxUUy*3<6xxYK#M z{oLR7ww`bNpYHP?pRf1vKYO$J)K4~l!!~d~Jb%7Zz5n5RZvSsRgYrASVW0Hp|MC5= z*MAzM{&YHPC38O(i>kS zGjBQ-$!!Qf(qJU~&+{lw6ETUS8PpZtXyngRe=MTOWEzCNkhq3_+Z!A6TWBnNDq63_ z5^NN$R_u?Wcr1qRkKKdA;hOa}2I2IUx(S_(K0(*9C#s7$GGL2f0#Y%L0ZQnFBPbgC zV$S3A$4wDN^!D?um`2x08)so7m>~oxc|G^y#8?uOtdLXiDZ@ ziZ6tGaRNV8q2y+f2Gfm=;b3qkdi1L9!f2+x>o(f4pG4D-e!bC#HevlX_6P5e&-%kt zC|Y;CdEmzWzZZUzCQeg0$pYp*NhXV_^?2d|v~l*Rv9Yl+_9tQ-T*L1=cCN9zA>b4H zFo&TUM?1OkwsxLT0qBW0cCT(zKdCnw?Hm8ItgV32dzcFo+@~PDp|Q6kAU*Za|AffU zbDW0o5@;nHPkk+?42SDwEV3ZC?aH(_AM%EUhMLmfY@yM5LMOTR*e}skZ(Y&{( zkvC52vW(ai4pj6O({xi6Y-2H}!2?VE&oGn*LGltPkZOSE4x^46N1qbsQuKP_C6x<8 z*dNs3-N;M)a|i!*E_Vg~6_Y6DFF|O4!Y2Uao&ZdOI7yo#fzgBrMgoLV(G+GOh(ec& zIG4?`cU;f8G&@b@&mBOUc?6r@4aTy|T)6BFfV}b25gZjQwW(UV5gYmx7E!imb;JU90J@zV%W+mHw97rP41dRYZ%9xFslJ^&m|zh0e_S- zk<#<+OqnIX;boiv9U$sNhgWCu!q3*o$<9i6cW14g6U<&A;iI1|g@S7A^6aw@>qhxF zZv5%ok8!T~fu^!>yuK00YU42@ zH4T6%0dz(7DFE(j8b$Lf54gn8o54xrji6!}Wt^2DCM@6a(KH%;aDgtTaWu6$ZokQQ z9OgAa?v9btc^}TAF><(L0W1(qDdM0=0KaEmQ?heNF0g~l2zr3$G44pBIa1HNe_^u6Sy`EYzC~G*$$KW zU2%TL1b)||_R!Lo(eSTJx$VzVuFB|+qX=J-SP>p%C;qPJcADjVeuOcr&Y<#aI8?~N8+*63?#@shPYX?32{PpcCznuUg6R<`?M#VKoHaISOr6?L=T zy(spsTkKu8vUlAt?_KBqK2Jn6&x)N|%##$@rr@K2Z;s5PUr}Hzjt>Z$m7iXQ$ zF8;qOi>+*q2qQ2?X|6pdKlU7*bS1}#^Q5@Qa*{X=fMgP_at3%}24M)4`6JSQI50_0 z(8>s$zZQ!4Qy&ktL~+u%tk!)IvHZi&r$rs+sh*HHm#+9y7^emIy?Q0GcPA3zBnhJ_qKF z*#sOl$kj3bnn&}+RK7^40w4NIn}n{3S&$^qZ7Ebk*F$j?0Rxgi=V5$6jJ;c!YdRVe znnSE9+lP~R5?n82AK>tv_%R;G*!7tggcBQ^{wDIwFhAygDY7#6_cIk&A@2@nSYl>Z%lr;bGd_ zYKSMoxd^o?g;Zm8Dey_gF(>e?-G_qIhld^>-+Y)8>~qN^T(&=Y(*^2eGFej!h&FeC z_0O4+U{RbODAt#y4q|kAnJPyM7IvFjSkVBfI^*p+)uo7HSC`#R7qD;ZRN}nDT?IiK zNlQ~aR@yJC25B9zLsVF|ra6F|&OlEooVHE5FRht1E7FO4shFq29K$B91JrYa;9j=r zdxA_n4CA&zZ9ySyBkLx&ThcT9U}-6!<5vR!&oRj5)GAF2iRIW(CXhpU;DV=Ptf8iC z*V=Qk|3OGCoKxvbvt#^ZV-0Y|ROG?sJ6^d!Mr-;w78Pq*p_|f6VH~EHJRn*Ayqqbx zEMFj20@2Bf)Um2F_Q}jWs$M?cVQTpbJ(Z2wZ!BETBb!~#e1X)NN~FwO_p^`wZVSJ4`C|QgC zV7){_gFfM}HdivXoaw(|FX z+nt@~f87854xg`c|M!hJ2w-y&!>lwyEz}5UHYWnUjWi}BV7#ymZ%Pa>aDY+ji<>A( z+Z*Q^?PRsf4Sdnt{OHA-fQg%P);lHto^Bvx*W1~Cy0tOB!adrPRW^{6s2^@jS3?`o zoOsG!Yb2z~#WmP;D&=Aidr|oRn03?I`#+AZzU=mY)&Aew*?IOnzyE>X-2P+#{|=w8 zcK?5>MQg3zwY0#Y7sG*#A~OnXtxQ_ugPIkufNde1dP^?Sk()(?%-^Qd4LYhF+obP{ z)h+sKFY&)SJ3ARNU+^8>5ywq&c7vN9m>hh0N>vm{{)1QBGe7m9LC?CYm*1mg^{xHl z3WkL$E&b$He@#^f^aC6ffkciVQ2_!j&$FR1jp7C&OU56*x9W!KrJj_sE9UcvG3&! z#o6G^t}<+MZL1t8Qk zx^`h-Ca^2&i9ek*#bac^4Hdu~0B-2=`r!HiEbC9z`An^7>yS|zQ;F{-RCd6PoH+TI(Tt7gGIs1Z|8cRwzi|z zEC>VV-o(BlcH8JTx>0slrBy|)EzH?y~7j#mSj=}D`hLG9%-Y= zT~hW!{&h!D?7LJ-Kav;O$fv41ZJ^`OSQ#9esWxu|Wg_58F;W7_epLgwiJ}h)e@9gp z*+&5&W!Z#atOsR~Mj-z~BkHfq2G+!H%SX)GGTSV<vRUr%3MO=c%iOI08TPMr;if*4{ZAHi)WWwBX!?u273oG4 z_8>IYMxi@MA|RAAPs)~6T(e{h%UA^l*#?mq#`+8N9oxhnr@yF+=S*nivPw!#e+j}# z!~<4SiVPWJ35v+J5}^M0@g3?scLf~uUJ5%_%`qEv@uiwF?HUK!d&TLR@bv;gL%Bty zvKH!_g-sw_(!%bgMKc^;)x3qKD+^1MUDh<|SnS=x`(!i?)E0;rq0787UwYprZIJ9f zBHQH1s(5U)T5Q{Z-)%h#KW9Y>6+}fge$x_u2?r0w*cwAPsDV&7g}N`=kAi+G%;%?x;7-5<#x0s!53NJ*H3uXt| z;&^IzkTr60UX@BqS~FgO?0)02?CPa!Sk@^g1>`~e)V*4af!wH`NRxTnRtu0YN`OW9 z7{owCNebB?eBU4Lz3sbi_70EU5Bu)X;O*PPtpL6i>JapO!85eT`ITlEdOJ)a~OQya* zuXoxz_WLs2#r<##c$-Fx(M?XU`bO-*-b2a?YoqU<q*dJE`{SY8ETGnD-Psv38W&x!1Q>d9gpuh15b{@G(NSvSz zhbjfbj=Q@>2HKq`^e>-sY`a-fs%w@qmDqVpbW>3tr_B2Q0wdJrIAp9%kCSa^wyppT9J%!gv+t6uCc~#Xa5m~WD z<~&N645HiAyYi<^dD$2W*b`(R*F>KM_@aCORI~K*9=1u{VY@mkdEdyP{5D)cDs;piWeIcX*dcXhq>p9z)WL zOX)u)MrEo`d$EjFVHt<1g6<-zF^oc}g~k{6=o5w)@rXxs`vr29xfwDT_6_K{l%?1+ zIE-4BXoVZIm84Oup4=BGmAd7#ptcwDF<;Z+%O1m$2ENKF40Xk$>0<1=3#lch1{#X8 z=5~!d_^iG2vAcP^F=#NqwhG%>A5_^CFj)qB$*R!%M(#Y(eKq>2y2C)10cZ8GdV zn&&W=&ZVYI$e{+;qe|xiMJo|J+g* zg;c6VBx}hm(2g8~h$iRJL_cbX@s+WR$haY0kRau84S7!*{r+?DTTyZWI9dx>t^l_1Z&+_Q3L65 z2-Qo_PQElyt9n!9f*ahy>9L!SyHh7L-y}Q4#*zR$&?)J{sANJqB z!>83+%K#jTEw=!N0*A~*(`7rm6}LHN2|5W=;18rubhCr(^c=)IR3ejNPZ;hPZ3K3@ zm|YNlzhJ0;vgbPf*TQm?lO6lut<^wzd&z^r$)c7q`k2}IQdDS$?a+X zsK0+kmgENMMb|*nE5EV3dlg)x)^P7sm@jg5N`e+xRZHIIAmri}ShaKSpyHe1;2nob z8MPg0I^mKt`g@ov@{U6##}D82hkY8O__im~$OZ8vuJ6{E-CU%j+U_pO;P87kPk(8O z+E4Ily?fU>IH)lp+dDpB0Dm*;6VtJ}_S@m${fT(}69ItVgW*AcsD6w6!O;<#pga4{ zMnjwp#Oc|=;Qg8S;qdG`QGYWSzS}z!`={SG#drNZteD~z$Hv9A*k!h5lzb!x?f)J+ z`rq|aLH~xpKt!Wc@>14$<4;s@dN_tQMCL8|J~nM z%)I1-=sbUpEaSOw9)078)-)AQ#ijVqfAo{Peko6$wgQbwfCr% z0r~Vs6wy)~_kX~!pFq(E!=KPHXn5wLGjIR+fc~DJyme3e!|xCG`G0Znbho{$q*?M6#3Sr}gF|e&0HXJ( zzJGACclKRRjK%^^FXvGm9+LqM{pvUkNBs4#V*Y8|z?t|nS^wg!wLWrWuWvWU{>RM_ zBT{XB``2#OR2*_ZG_E6+j65kQ6j|fz;`s1lEc8F=drLiDS~CALoY+b5(Z>)W6a$3} zI0a5Gij;U$PAY5uLSebu0Fzn=Kt5DeAv$H06BMibxRv-*4k?FH1m3xMDFzLgj^R)b zyo4f}c~JfaxNn}2*$3#GXw3wW0G>46R|lpB6d1zW3wUsaz?!dlmo2JWk<`T?0ZP6fbSd7Pflf{@Q{? z`fF<=*La`WQ~}#?&1fg+6*L=DI8to%e-7WV^!)!-QeXC2e*Sm2x8cms#eaXk)BWT8 z|2;lmQT`*US5D$yiNyEC5SaOQ{q}53N~Wu55so>+qTFB~y*NOL5YCtja+wbNBwlk?u0D^VGM6x zKpDi9p^RAubOr=jAl=BS4d3iPefspJ#Eho!k`YcJ8odLO+<4PT3;C4xhLqsDVpHtz zo%T`KJ%%&Dly1fUCu;xuVtluK--3U)H0{i1y_e$x1R{pbH&u!%t9QEh{T z-%Q&5I?yU)Z;-Dm_z0gq1epxPV43669ebhDwZs+@GdUAt{0(`zUyVC6#Lt^R`JS$ zRnx1?1m(C%i=>9l^q&9V+#ay$aSI*NIGSWjIv=rE!$EDVqoYT(pMv+2sUbQ$N7KbD zOh~QyPR3DGCj~hh9lt+1Qf7$1_;IAyOV2w@q}2*w7`ih1AkokuAIayBlxDFh>gvWx zeDm$MqITzv#{rWEwsLSeCSa5AaOvJC2;mf#&Z_q{b0+l%Ca|Kym$t0TeIC?0+v@zv z=JPJQYI2+G&xZ&4s@`NJnT1iE(d)s;SF0)PA-aVv~ zbk5))Q4Qnrkk_BT(%PzLWiL1MBIog3Ax9~q16f659~85*_drZXepCE=5fRoa=^zmp zWFCdn7$b&}9!EOr{pF}ZH1&k{i*)Ez?qG`SQe<9z9M_*V2nKSkH4rcEJzqng@EFGcGZEEL7pUMM5I*s(8&F-g!bgfLwGp-KkILo+}kqM1wHUebEV zfMP{AXo>=ngsvDN4VtV*X=*DuB^g*f1EL$I_M>dsl=tt)ZoAvrZg-Tcjyh?HBJoiR z#7bH+GHD7zaqzlr0Yz$s@qJ<47a_($yoSlb#7;O)LNJ-MDVi`v3?o?4h-pkd4V5JnVvmo}GvQ0Kdski2I;O$0)S<*HL_XUqiTB#jFb$P0((| z-k7caU~b_R&7zqex3j(V((3zzdQUD94Ce$5GKSFEBkU_#^L{y8V^(kva0~2mRsxjG z2wk!xT(bdb1$$pB*xRUyZ`)(p6{n2Ng1bS6|332BsJ@VxE^0Z`8d|5C+6N^A_@BOJ z=&?!Unyg^|kFDK1@3Y9d@g-|lF$j{m;7kBEt(K^8lHxTjbYHf@ajAeBCt`bRt0{sQ zO%;*K;9bik2M$Gr5Jx;kTE(u5>;ljh`$VmohT(G^GuQ&mjG!(mF+lY|ac*jLx8a z3h;8a&SdW_ul&qxQp~!4TeqqU5c{Wv_&4_LM4%-{7GAatRfBJn}tKogp zKX7Tjx58`@ksu=|lpTeijDX7oPJ85qq}iJyw}mwX%L+-mOyi{Z*60(kfTT65#|>Wg z`mQ-T8g5KBJ6V0HT4GbIT=55)0cisuTS=eSFJ1&PAydwjQB~U?ynA4$N z759HB?uyXE^U9TXfnOH;7e7ny|GHaSo$Z~1{m=71>_2~(&sU58JCI3yrS^)Yw@o8a z99pGIpKD^m=$4%@Oj1R>C)jv%)bF1i9KLzuo}Qoy&%t2-pZ#I4E3y|yhsW=KbYCAH z_cpz&1f=}b2kZ?OpEoB{-%n94C;J5>5pZjU>334ZYP{^A8xYY#uCNUIsdj5FN;iRM zecaf1kDQ&hC1GOCiR928T?(tHnZF7`yd?p4aYmVf#+ePQ@Y0PZ`BNZ}x@6X<}UJ%j)zQ5c1(H#zcKw#aD3pkAbK`Y05NA8=$BWgt^*HcMp`3Lb_ zI!CR{M5{7|^`!Na{Jk+Iyl)cg*pz9AxrBPdzxchZy9i-{|6!vy`QS;4sYq6xX`pn< zgPpP;0m>x4Oh2aV@t8QNbP=TIrD$nD;ab3{Qf3KBO^K9Jka&cMO-kZgrw|)W@%ZtF zPq|imF z=m?c`E5?4G`X#%;arkxCqxYO=osJ5sE2kOV!$uV5Es&u%Wo0M1p~oY8`D@xN`W1*@ zsrXhnkGf43jM}&=q7x${1#&9yGWvsGIc8`ANwMmR zb5N1^Ih7&>W66R(QE>mlwsYvnQvf)?lQN~&Y?ey-t77Um3Oy){=x=(wdRB~RS7xxI z&Q62&7j!f;fXbRitF2d>S(^B<;2=3TIJq1gcvj6q7KndKWi7B`+i)l@6^0AV#pG0> z5jrpdcnhJU-N?UI-2J+YC)G5A-kj>M^v=v<4Q60*5lzfBda%!zQ~vmbAJj2|vU`qx@0Yr2@& zDwY(V6e2YRlZw5Yiy1A;ch}+5BZzn5FmBVTPI;QvtCk#_PSn76!A1QoR0KyoBnzd?@NXOM>MA=wIl;!9p#Q;t-?>ZPE`$X?KztMI+pgM1U z2{%YsD&s&=QPSpRlCPWiA+?Bj-|W>VK%pGs zN{O(fzQ+2poTC&yr1pS@^1=Qz-xS`~W1nJza$K)mH4Y3`cm4qojp$@$A4(_;hzmo~ zy*=+-vc7QW&!Yp1W`m*C&M&LtQgJkGAr+gb4(Av+c*xFB?O?1QM*mkS?KSho3UXHl zEKNvQEq{Y)eEA9(R_IC9u&V2-JANI(hWf;k2_}awSEO0JA73G#$5q;{%Q_IZZWr># zVVp<+*drIbz>o zupd^$VYb+^kofY7NNJ#GgWh=XB#dDDu~Jo~oGlU45j$t+&Vi2_DVF8Ay@GO$qm1F^ zg!025oXgVnL=>G-#%mtdJF>`jE*nkV)$GoPfaULfoW_Ptq4TkQzY*`WS+`*yE29}R z!x#Wr_EL81q3G!M)ZF?JUu%Vb;+5sNyxQ}aVHaL{sG@9XK$Vp zXps3#VdAN?gCCCj2WY{OVi-Yl*iZ4OeHSx(eMs7Vl zftXUEPQiy0ij{|X`j~*2nR33djgogc?8zzFRlK6OHW=yM&u-sd2iJ!vWFf}DmzB#W zm#CcO&$IJ&Ysl0UqA0L~TaQ!-0(`hAsm24uzD^+0Bxrfq>6> zclXk0mlCL&%`~i+GC+8Ztb*1OO;3SsJ1&?BHC|nW#&~gveCXC$dylGF6$?Cyd(MSU z>VQ|x*`}s(HA?&FTwJs-Ol@$<_N3y#v5-ejJ&U}aOP0mj9lTbg5Yl{hP&!S#S8d}z z+jR=zmvzLxZP3j+6gz{|_rw7{ry$ryE4ikL9bisC-R~7)#+=|Lp5#I4!)u(Ksre&J z>1vIEK7oM(uT(xl4O6O6WyE|PaiJ+5H^pR@%6z@FLcKTWY1dS)Ch8}rhbvFVlGzt< za14gFh^M{x!y}9=PS?{YG0ImQ5k}r0V^kP8FN$CR?PITpX=UfZ*n*6y=yPeTpb@&W z#c(kgU~+Sj#GAyjVj!Mf`bl5lA6$X&2JiY9oHd)uLI<3qeRuEZ2%+ths^w?=nkNT# znn)OQbzThU#2lB3wQS5I%Gh^Cc2#*xdS1eQ5fZBO zM^=s{3^rmSLy<{rwNO5CnmT@NtCdC$E!m|xYCxgtQPYJlvXp>-$ey#YutSJv>|Ovv z_*YU{1`vk2RKl?;NjzOzO_pg%5zq#}wmr+cpMj#%3Q!Ms9cK39%#y&zf5Ir7cG_Bd zbF4p4ib-k#IZWc%gfeI=M~BH!g=%$Lz$f10HV1HFD-0QyEzv|zYMU9GUNOuxxk#8 z&Z9?;yw!aLEd-Oav~9b6wRo!8!lnLsc0;b{_2c?Etu;NwKbMRt=mm~`$mb}SWJDcy zrn;ssY@){)&K}7$Aq(fn)4^c*Cx|_`&1Edu68YP>@0RnV#iEwp+>f&%OEZQ`;{dqv zM;|Z^32;mD6yP6C#c&$5lo=Fx*CQ|R>#De9%X=1&UXTx!4Uayi3F$euG=ORIZ5-Z@ z=-+-EmyA5ac)a(){Oe@RV_7%?4fC!;5QKwKQX;F$t{YdED5-WxqbUcE9)Y2{pUMHJ z5J)oYxV5wC`i|@8qn~rbtdTOt+vMcc#U_|RRnVm2o+;7F#Rwx;x(@o%H9E{-}&8@vIhZVyuISFjb3f-y8;;NJ{3Z-HQ z8CN)Lm7Fj+Q+j5~=Gw5|YLj7D!ZLo{T7>e@`enOLHXX%mW)?5z#%&XE)Uf5o!c*>? z{?dh$uIzeFey4>y7k?HlU#=V4!q9^w`QR(8S2jzqawFeZtf@aq$qOqV>C2oT5^=af zkv&D;)L{cWgE=+XzRz0us_gaXmUWiii~orFooT11Sq!G(7?9sMKsnkYQpA(^t) zQS!mH#_FZy_kq*Gk>0Htc~&taC9`7kcIJdUvnB*TJR2x2uHD1&?q7m@4{e|{)8{x9 zC_q%g9T8Z9V^=5dO*m#le@ZtWSD97r$#C%Q z=wreLX$?UX*oD&U!&33B`YAaIW9W2@OWiJMc~w?Hx{YMtEaD&1C^IU_!$(^NUL(Q+ z{Ham0S+?v+Auy9u;4!IOM)WLt_gM2_GLKFcJ(cBMWka^zL7P*VUdNcBY)_c`gO(H* z-q0sB%8wW-r*s6Sc0f0lb;2@a!w*+B`O7?KxfavJDwJwAC!<=q%}(=cUE!H(67zU5 zMkdF_S72~jI1QXHdDK`hS>$XAK6y+%4LQZJ=Z##+GQCE{I<31jC8xjBD!b9)k8|=? zM&(5|sJAISaW)HD#`(1?|HSBduK1P;x3>!(cGJg0xY| zo2Uq0nL(Lk0L&1nPWf?!NjGz$St(maE*xv4n)hEphRct#s_WO{P!uB?N8`oFAa_;L z5N!&X;QY(FR_K{MUzWdwp~k+?HVyf+U_chu0=df7mqSRwroWQ=A%RWnjg+ZjP7kVd z?%Zv2A2!a7hA*vkRe_(dt5T^+b{EBDTv8*nv>bk<*PU|ekbbwt%PplXTChpC(yYqA z<+DmEHOf=R)5(q}6{$)mk@o-M>w`5quRdgH!kJ1JS7`oH=cVaE4+?!%dSE#c=E<#= zm$ZZ*WeqCxoibfUE}vQ}fw#O1%k%G)SgW4fLc(QJF5)@=;I}1NbWSL45%yxdR&g4a(dU+-COI*9=Go4+x{qp5YtW7;yw|=)%?(^8~n#0JU%(|yX zHdjTNZS7NHIhAL=mHw?dVS+fxQz4DsMs;R;3w>3e1UNX#GyE&zSSU?>TSeFjS1UG% z`Y7H_4z?$|i?_Cf(0RRWB1Sh}cunR)DQRZxoiakw$hHV>8u+|7Fp35)ldB?y6z!Bw zv~?1Dv}+S3ZqCfi#HKBL+ul-HhNZvyN^6(*CkJTlvNWzt_Cc$NEU-&2i{NHFtm(wH ztxl&?lTmwqZo%klD^JO#z`6>Uw;nKx{A0wSDZODfg4XnEnFF)vZtXtZ-rafDe*WTR zF@{kVrKhaY^NLCoEU?$FY28Qvbbs%pf6#O=HX}$O7Z)LF4lgcZ_;zuTUcd*>z7kk6 zY+;YBjB>FHd$pPhaM*kQ*Q~vZi%!RXd2#XbneXLnSRTsAE<4H4=)*LJIS5ASuxJt^ zcPH~4T7haBd&%^=0Poz%7Sg$t%={<1O@hXXrjb;$r}T*d#yrO)!BcN`HTL+ln%aV8Av(L!+V@(7L>(Vt6yI zux!%NiiNdsc`FxNi@IdyC6i?+{WDN>v0pfp%nm5kpp@p8_hLy!HITInJL`2Qn;_H9 z=gzTKc2~f^_x460{W$G}XWQGj6P`Uo`zmLf%&f@TDz__&ObmIB<3A;AOSOsvoxvUH z@m-<}UwgpI*J5S+-?97pd0H4xBUeZ=v?D=yr~E$;u?RGdPz%WEYoEYZB4+ zk0|TzFK9T)D))zO|NUYA0Kf08x%mN$N_ISUkFw4}V*tK|zn#@HI;H7sB{~EE)-x}! zCXophf3E>`_JIm&t3h65d}gCRoG#;ljD~R!VAEX6G7D^G&C4sn{GRxt71wPy)|z$W z!e;P&e|UQMbKen<#SXA|zX9s$WErTVHfRcAoYYxUZQ!S4-wm!q3^JX^JXx>=`PzBa z_lh}?WnmhCXFAt2p_1QiaKN1GIeMX;ZH%wT62EZJZIJPze*7N5Y+ww+PrzODjC(P@ z58IGe5xv&POihy&WM_g6iEL~aDLLlDpkE+F4l|6&Y`y>{CPr$)mLH&!f?e_FTim*CoIx zFatwZHaX@5aXVQ|gf^}kR0@(Yj4ee^0oY2E=t-uIkUdw(lbaI4gwg{LrhxTiop$HEJ8_ffYl7J|%B*aoVi`Vd&wbv$W zs460hpik(uzDR8Hg`Y~LgPM@F6+A^XgBnK5=v0LYyz<=0?Uo0ymQ;QPds!#;QZDH3 ziiNGv_uvNwD)Z<>kqV6s@5(tW<4unHyDw2BEICunlhpX0pWBCvBGr?pIm^0J6h>f* zih9hdh@Kar6jJ28UxXx^*sE|6YU#u&@x`~-Cl?}g$%3spr*~h#i?9AvaqIWheKdDu zv5N*GWqtf%I5>W*rKKYmst3-8f_udFfl zVs$Fvl|`*>b++N(VjaE>-xbX1F9p%=bh*w@ef-bk!4Jpoo1?wAPSdPpE4L_qsy3u#2dBl1E+^}RGAlK^3O|(sgdjeQIA_+Y zIowA%MZV%I*(eNURuE1G-vKOnX?WkTk)3>gLwzw- z2$W6x6W@HJNAYI3_x2t9s5li??Pm?=-Qnr!;qlwD9lZu*9rN2>BMDS%N&a31tdg@3 zC5(iynGIpG6`As_qMbCIW6W~hMbsf~W`0Yn=Bt1)D^LLbD_XsA@MYr~g!Iw!azt9G z(@}|(0}ucWGzw~vHBG<`Vck+SX~zJ=&^psU6>dcG{H7Yj7u=8-4cgZvV=ug(awhV8 zbL!O>G?xeET_G7JH!y~2s=Kk2TQOw*`s;eoG}~Bqi<6tAthW{f>R0H0Dw>X0xn$^8 ztko{zR!iM5O60iAm!{f9!f1(=m5h=VTU6ZuYEL$488wf69k*Qi%;a^J+KPXbvmn05#I>T43plrWI*`ol)dk<3qDVqCn$;V^j6K(STa_Wk{(;Evh>nH zO&@OQrIL&GYBd>LN<9&WrxZ@ex^pU*1L`%_sqPPs-oHDxGK`)5bkY|_elg4-`yP)S ztCW>=>;kinUDWdSQ0unwfEIG+-P*&bKWkw8%=J68q9rS9EZ63kPK8!VN}^`#8iRi= zZ9-@CqdI7eRDfM9a#aL6?Bo57s-kh_Y*6G@TrT7k7Se`MSr37#28tb_ux0y$~U{y?IQ3IOJ{Vdw% z6!g*RvxKWFW0LgTl<82u)*#d*+gt;O#UJl640Gh@Z!?~*uhj`p_C9<5v{zRC<2 z(voaHTPs?$g(vM zW$9EBOG)&WzVW$*Uw3!$Uo))Y!)CxN-r(3k>^`HI6b=PS?u#tH@_hmGCo)}Ijww>w zC?p3dt596b?M0H1jFZ7htXzD9%@qvAzR*IQRzn?7J!?7XpmH(X!2V$GsDHZO*Xe6v zW$x~4yOcrms)4_<1ZUC}ktF@3n8CJo=Zr9a9~(UcA%^teQ`gzePOR5^2k!gRy|;ZQ zYsLa3<8@YA&fE6cYugPkL531%B)dxwH3BaYE7tddVh^@55D&Lt=8#;#5@STO3OnWx zVe~0B*t4~Gp3y?Nb4*Ef~S_B(yU~=wf6c`(So?}A@sto!rFh+UO#5~+H72&%@h)pIolp|G6DJq@YzwTI-``^5Rg_bx+ zA@4P2;ZBjKml7HRR=fv)=X4?txfXK1k8%^ovJ@R=pnc-Dv2mv<^TyPinhe(ffAEJH zqk;Qsaf8Z$P6MbrH5povHcg9?vQOvUxPHC6i?N8C;sqQsom!)4YhZnl_+h)04>EJ$ z;^Refr%WZEpS zXT$f$`^(Q^yTak|X@7VIf6fNWdQrcVtDZAgki)oBYkJS)QBI+VL;+AJU|jBW`qL+( z3kUhN1!sRaI6W2rYjAj6I(g}C)svk6dsF=+~h#$TRz*=b2HWh0aF$eTBTIz=M6p|~+o zYygDQJ-DJd2C9YxiE{=3fkHb$zb*dh`*Wga6gq+1R9E9Ser#BT(dcPA=JOy{-Cb~Q z%zW;1-+35Jy}So zm$n~>k8T<`gGpJ2%9a{r;dIPfRmwa>9L>M}xUp(~QjVhHZEqUaO?+<{18}CO<;OFL zX`Mb}5f$)&BgMNw7bnxknl=KnS!G7Aj$sa(MyZDi9O(s)ke22_YdB8H&4}6JGERYq zlVz2SFy@k}qrh;w^@)mMW)0yyu~;&2lUY*ih!|Xxu9smtvCb zFWD3;>5TeRs@~R$Ag84zvIsDeC}q^+>C?_b6?A22%f_kN zM@EH}t^qN|MpUVERCHKs!BEr$Ax!pk>Ru~5!zPP$K>N9WoBIxP@Ufa zXXu>thiD2&2J=8lrqMN;-g@JYL4uYX6wwItTDoi_PS!s6jaP=BT=~7KTr^LkQCx+JPk4IpS>R*p9)k>012_WdbD@^_Wj;l;9{opYh(|;AE0*>@UMHPOHu-j z7BN{B$V5f;TB%nSoQr?^R@9=fBx=TBxEzR1rXqo;$UakA({ZglvCweu@U$=bKkoO* zwM=9|D9(m^$ESNVS?+273?9Bct*w($h>{QjHIs|_gnsgrqDztgO644~nZxxiYWl}i z0**a#)c?MJ1ibYA`{Ci)Po=!>$>8X4|0hvgyHxz{z=)l%rR zYbOSGrU?L!T5Wofz30s0&pJQ}GT$*y%~(uojma6MykOIz(i61k=QGh6SgTTfahl<2 zDG4Td4ZECrsf(NBW02m0IHNEr7%PRX*UlSQ_G)*=qx8nuGShFTDILy~|9U9}^UH)2y4N2eYyo@A5++t##Ku?=}9-NXWw zyaKqo>4i6D`wDHu`>yrUm5niEXc1)Ur}P?##^@yC7enutzyu0Y8~~( zZ2f^aZ6zhoK(Dt#Yo;q#pV<4fPJDbB@LKldvY~PS$ROIKpfRb8_bHRJsulOYK0Lqt z)G+(HXeHvGhet;gn%@r0kNr`ukrjzPm-r-p0fGa&UYbeJ}EJ5jDK`*MzMA$!p8`bw9&=lbxJOy~nSK&m@V$d|4!rPCmWKSO%QJ z2;-y;i-b-ju$Z-Foud9oGsGQ9AucyrYD|1 z?R2^?9xm3hJix{PtG%}IlB%rwg9Coj2EoFXjDh@+Rb)-;Z2XiT>RM>eSg&#)1iQ3- z9F&RArZFzi=I&JN@176T)|RGa;izZ z5(H{u>3kOzG{B7n$&K1g4H+!E0Km2lsIZq`X%w*5^}IfpX6%`_qA zLrvqSk~>?L?)VGk;qv{nro?`5c)FIzu0GV!b<{M)9jle_tIsO&=bnAS0kd7stgSe> z3_!eYT@TwL5^=rtcW$`?0eobjT zrTRt&2<48d?BuHOLoI!Zd4M)3d|KHmY|l!5>ur1;2QmmXgG(*6v7P%^t=3Meq}InZl%Y6)-c$60Ha=PbRFK@ zuJJ4QbDec`x%uChvg*3-m)!-KAL!{Csl(K8mTFMcM;#7O%euCf;;pi-I6M%G(EA8e zNdD5xVOHl%TT|D=?BPm=mVG+Q~Ix1lhDyXO%3~aUp`} zMQq7MR6K6gm4Gzuk`c^=Y+eQC3Q}>+7N8(Zk@woBj1fnHMriWe#S?nThR#eFx;vfX z?*4)u{g7R}F`7b2@tqfk(3cbD&hao!>e_A_f9`uaVXEHr$ku7EWW!Yl?J8V9z<_0F zHJ#t$1)Z10C0Z#X55I?vgLXvjo?`_NHebCaKerr%=zkvh^Hl5+XjqdzN*{>mB!-kH zThrhjMPbnDiz?u8gZ-0zMV1t!x?mi*=#$C}X?mlhb!rKxhV~3~hNL~E@UF@mVRt!T z-bOGHF5L&YE?K?0?#v59*L4`6UKe}{8~smzeEz3?WR}Eaa|#0HbUJGhns2hC9er~1 z`54>t+XuJNfzPvN+w>PcxxYKlo;~mUsk^iFyt4)Wb)LcVt*1}7w*J)l4aQseEby8G znvA0;T~=<@`~2Ykn|y#9Iyv~Ubrg*JF!5W5I%ChS*qfvOU~8-0+4x56W0VRE)E33b zuJ}Iqmv>8UA^80B1{|D&dD;ScHV?v(qG?2*Z$;}{F^!@*>TcvN=5gF2+5;sNBO|SIlpr1L0HAdM%b=sAzrM7^uh-8L)i}3J-#| z3;fZ^`!kVvc-~In%mnsD8ORGCyBVSZc_cg$`kyj3sJ(FNITPOl)jmex_6ON>QHV%6pvU8wXBE-y=&^_GiV=BiXN;fFiA6 z{wfY}LOHQaHqLm*_LJnqPC-~)WoBv$;MGiFZqaVyyY*ms&e^hefw>V8>yl0Nj-M?8P! zq9Iapb2!*LJo@P?O(zE&!pWlwb9EXsgo|#DC@NL zvaIb`u_2)nhJeH@uU>3`cgzAK(M_CP$~UeGN|mc~*+8qDyG${MhBmt}rL#Jw$D*HB zUaSlN-|}4M6qIDBL%O;z4Lfmv4`a)lH&Lu`Cc||WUYg1U&jp$I8k1J3?4n9B$Sz{g z0tjpAIG9XyO6h7dL_E(vc^G+-oDQM7bR%q55PBCu^twOM!K zD~H5=J|>IGMB7z_?kMZq^19;U^Ycmn=#4|11$57&C9X64CO-Mm9sF}iSVkqQt0Q*q z#dt5A`G|%B)o}|p3_c3N#b*Hx$`KTWyTNI{M40n1Mz$^Y)s=?}jz{Er!O&l+fCK`w z5umR+3S)v!RVyoee)*kNi@uXhC|yniz;QE)N4+}+ecv2od_%6dsS&h^ouNw|#vrAU zlpe5j?)e7OV`=JQyeJRMhKkz2I4$`7?~NxYONYU!E3FuhtmYEiJUNpJB2^ zACC3$nD9S=fjn<_;D4pAeq1HG&Xm3xKIwO}KnBA#O3l467+J}wy)nAIm$~^p5zn%F zrcXC$xQb-X-ty`Kpwf_2ZUnx2a5l00W_J_KSe?uKExk9sT?uG@?FI=IUB`+%*MwoA zYFm*qmDO;F%u*w{P=uW#*}j5uA4+vvg&F;uo^QNtl{CF`W8C-E@BG6ILCbeEqrQeGXgoF5;B?*Jth+6 zz$G2ox@I^AsoC0jwkBl}Uyf2Y=0TtBdCdt{n225?}(CJ9l85R zL*MP45ciY;Q$^w5J`mU8PkgDq4EpKWVAz*X508;@%PM-V*w{#h#CF*%1ZlQ!n3yu! zwt*W)d%C);s&yl8Krnj`%^VHhzCApCOMWpq*4VgT{6@S1#21S>OX>2$qN!ZuaDPuG zM0)p+4vF6Q;WY*zX#>wAZr}li+kl>kgV(Zh+t}}h$Y8&GQ^kaFn0wfLB};`x@}{;YRWvg&lzAxX>*jMR|w)iS}} z!^}F_P#TT%&gHptni7hZ+YI8nb7!~+L(JWIS?G`dnbN*xZn9(C#mTax+c>$|;f7B? z*(rKgFypM)D3z3;ux5$_vsQ?PK-;RwDV6V)A_HXX)ePK{U25#gJoRE$eUkP2a;?HR zeU~MhJy#=>?x-&REJT~`ltr7CD~=6~mC9noVFWU&pe}TWTzX$l^6DF1zM;0v7y-}N-xb}^I%>UC=>W&mu#|=EL~Vdfx7BVxgA6f zwYbfe)O1S`JhSbWdyyg-EZwTQQQWJAHP&Fxo$r9_*(ZAd2ln(avzhN;Rh>(^$|c^c z)SlN1Z|k#j=ODA;kW~tb1C|UZUw@ zjO3VGA;scCn`Dt$YD&!&GndA>Dua?^FbULgW{6~|I5YLKvcN&rs@+Aj&6Q!%Gvi#V z_$p)EOv!B;{}x-xTL3?(k#dXQUq6(bDjJ(IHLhQEL?vyN59?LI;fzgqa*9a3OC&bf z;;wD|^PluT6a3F<77$X}L}xOUz`X!O)of|`Ib`6UAChe;Mm@4d;GFE8eb?JuB=IIC zBt#}-mwwV0_@^MC9PPUx7xwa!ho!V-VWL!60#PZKYDDK#smhB|ObuhVY%oQ&#mq}S zh)$>Tigc^OdGrr5QIvlAa(43&iZq925lI3oikK%PW)is{)9u)X)N0{DfF(;3T>UKN z;>ddc(tKZlMmD{q9cA-i%wml zm~}5y03>X%lskXW6j7>rdv>0c&d!n}!%{k!y@XVjL$WTCpPKTPsYTjOF<-6lcCm?1V6vuE+3)*WzcdZa*7q*k%Dfe5{Mti%YayF(*j;(MJ-H%Fic=S~RZaEOU&d@Wjiz^J|Oj${8uEKF!M-^(%SA{8ZSgQ5rgi;KFO7wV{u9 z{}NO-P*A?golDv#4^_vg^-hqpscQ@^bIi=PJXW*!xiPIvaGEBsnZqebsvkcd2cxu& z{N@M$HerN>!66&nxL*R4OyYdZ35?JI*U3mOkIx0-z&$Q*)AIgT*THRk*~k=}>}NEO z{f|MkNGR(BO%zhEX%wWKK?jMK((AKJjan8E0MMojsE)yy{Y$AcjV*!FdQAS>W}d@o z;3X{vMU=?eK~wJvjWoUZniy*G!^`F5a}bGK0t1V3j^a2+0K?rRo3uD*{nE@_U+hO- zIazlkFiI@^E82>1@+z80(Afr%Vgwrju+45{bL@ZIoQyZ0I>k1!cPoKn8fc8^?f&A& z(ZgHllr_O+*+crl5Vu#kxU%bYVW!P)6jzp;V#kJ1;uy6R#l&(O6|kjdRt*=z!nn}Rlp zcof=Mm6e%#ST|LrRn$gT2S7=Y3Z4!>8GYu8xINmERx!6xb#u0yFm7i=`eDpn<%pq1AR zk`1fG1?vKsrq^H-4qihaz3VVa(qNPT5yQ(6GFuxf^&f_0BJ(yNeDi?Rp=re0I1=E6 zOci4!tzk?23QZEfZAn4b)F^>O9{R{jWT>Rl7Avt>70Rkrpl7}h2BIn$t7d*FYOafx z1(Z~+uSUv3)%=)NgUHr)4LPH*Y^3*FOIj=+SJwI$5%!|=vYe$TLA>%2nA#rV>>r@4 zu(8)Z*{p-DbnxE9Cq?4jDh0G~mg)v870wjsX^Q69z#Kv;DeQo(gIE8fR5|RkoU#mz zVgpdmV9Q#rQ`Kll4?$QK;|;U<$PCFYB+L!mA87z9#378Hnfu8NDxPvwdQE5 zjK=dLKOgOYWBn~cN^+dk$)7Sqoux_gezTtEEJj^FL@n@mMb}GRKpVumQ}zEl#v<>; zzj!dJz8TprtHxJ3^u6GdW;7;gQBTyD_;TeOo`DGf;+2D0vWb>@n&n|OI87R5yjVp+ zr4UZ~aniu_I??k^N3ku5%(+Sg#f?7X)2?dIY$Tn*j1trBO21<#H!OIOdr}pekfLm< zh%LK14ZHGLMVm(C_L1Pe5@95aorr3#(M{FWDy<i4l^B&3Y$BEU?ZeD#u>q%1$y)QdjfKt~?SYZAw9Z^JicI#0Ae5e+nUOW8UMy4i zRWvMfJS8_Cx+z?qBT&@|Y5GFuA2&y)VG8 za(q-D9gJDeI<(3$2r~L(gve+prEp0xBlJT_Pw)eKCu^`rn71T&yn&^RpV?1j_EzCL zeu}ZQmb6fc?f6k4j(deT3Z=z4(>hG(2mf|gRX&Fwm!hSqHvgsr)4wIM%H=92u5R@f zQJ_3oTvq1Wo_N72p1DJrt=3XB#2V%Mtz_1-1a;WlRwr*0XMT#Vrt0%RM;e1sQ&Ibv zcveL2R1{9GT4|Xo50|Wqd?L_XGN2;-oqC%v&7>|Yw_=SCj zW#M-C6zf(sV^)%}dn#y3)*mE1x+`4C9it;dNe&rZOfC~ma^;#C+>e~v;)$XeJm%Oh z)tv{oRO|t#btL{}yqL|C`W;Rmst!t|4weJp1@jlgDv(=nqYJbMT@K67CPiP7h!H6Lc68B zfK+6Zq+$YapP3aPheKXL=WI`=FOmGQL&1>61ifyf7fUYIVzl04BBI+M8BluyX66VWCWmfC>KD74&8w849t8M zM#l`w1)b!-fW{@{I!`;$w~0C{ViQ4-Ldhqlg@ri!*mm~)oXX;Nf%gW=$)pU?DpG$>7~^me#+ zd?wzXV*Y}`vEax9>Q1nbFSM$@9Nf}0EDwfO=O}6qkM35^9hP)1AHS{G!$OjliXe)) zgJM~fWv3(GX(jj%J9z^Za6qBeO4(bv#nf1Wl*YV>U zRk||S1CqFr??MhlwYNqc_v)6@%E1_JnLCCrS%A1I0tbZ}<_;`xlI9l*L21@3W@OQ_ zc81MnpHiiqv2LST5$Z0d8m;tP*7tozCJoDbqu{r(d?Fbhv%P#ZWfp&Un7Cp?mOnK& zl!X|#9k|OS+co^T%a@RgNbBm=_-jC{Mqb&F>s@<6xCT(8&y_%{Y@}6VlwqIsBRi|2+&6_-PdZpmoAYDsly>6Uz zXaZoG)0PVrUmV~OkM(joprc2sewWKL>a7Y-`Gic&=(H;`|x9hs;JJUorE% zT~?nOZ{a!Issd#?_$~wW>(wBCq-6Nx^T+3p&mW&ZK7V}v`2234{~xFp0Db_F0Ra7) BihBS6 diff --git a/examples/litellm-spend/coverage_runtime_sources.tar.gz.license b/examples/litellm-spend/coverage_runtime_sources.tar.gz.license deleted file mode 100644 index 35b3eb2..0000000 --- a/examples/litellm-spend/coverage_runtime_sources.tar.gz.license +++ /dev/null @@ -1,6 +0,0 @@ -SPDX-FileCopyrightText: 2026 Vijay and Codex -SPDX-License-Identifier: Apache-2.0 - -Historical source for the unchanged captured row-coverage fixture, retained from -commit 73df8c41de98e55758291d5282314d00347e4a27 with git archive. Source hashes -match coverage_runtime_expected.json. No new observations or private records. diff --git a/examples/litellm-spend/test_native_export.py b/examples/litellm-spend/test_native_export.py index 71fbc16..a6590e3 100644 --- a/examples/litellm-spend/test_native_export.py +++ b/examples/litellm-spend/test_native_export.py @@ -357,6 +357,42 @@ def test_request_tags_exact_bytes_and_optional_grant(self): self.export() # Default projection still works without the optional grant. self.export(tags=True, success=False) + def test_non_string_request_tags_preserve_database_file_parity(self): + psql(self.db, data=b'''UPDATE public."LiteLLM_SpendLogs" SET + request_tags='["agent:PRIVATE_INVALID_LABEL", 1, null, false, {}, []]'::jsonb;''') + apply_documented_grants(self.db) + oracle = spend.export_database(self.db, tags=True) + for fmt in ("csv", "jsonl"): + actual = self.export(fmt=fmt, tags=True) + assert_bytes(actual["spend." + fmt], oracle[fmt], "mixed request tags") + offline = self.parent + "/mixed-agent-report.csv" + private_file(self.db, offline, oracle["csv"]) + flags = ["--before-period", "2026-10-07", "--after-period", "2026-10-08", + "--group-by", "agent", "--agent-source", "request-tag", + "--agent-tag-prefix", "agent:", "--rank-by", "recorded-spend-change", + "--show-labels", "--acknowledge-sensitive-labels", "--format", "json", + "--hash-secret-env", "AGENT_TEST_SECRET"] + prefix = ["docker", "exec", self.db, "env", "-i", + "PATH=/usr/local/bin:/usr/bin:/bin", "GOMAXPROCS=1", + "FLEETDIFF_LITELLM_DATABASE_URL=" + URL, + "AGENT_TEST_SECRET=synthetic-agent-test-secret-at-least-32-bytes", + "/native/fleetdiff", "investigate"] + local = run([*prefix, "--litellm-spend", offline, *flags]) + native = run([*prefix, "--litellm-db", *flags]) + self.check_result(local, True) + self.check_result(native, True) + assert_bytes(native.stdout, local.stdout, "mixed-tag database/file report") + require(b"PRIVATE_INVALID_LABEL" not in native.stdout, + "non-string row salvaged or disclosed a label") + report = json.loads(native.stdout) + for side in ("before", "after"): + bucket = report["agents"][side] + require(bucket["coverage"]["non_string_tag_rows"] == report[side]["logged_model_requests"], + "non-string coverage dropped analyzed rows") + require(bucket["unknown"]["recorded_tokens"] == report[side]["recorded_tokens"] + and bucket["unknown"]["recorded_spend_usd"] == report[side]["recorded_spend_usd"], + "non-string identity changed accounting") + def test_half_open_bounds_empty_windows_and_repeatability(self): for start, end in ((START, "2026-10-08T00:00:00Z"), ("2026-10-08T00:00:00Z", END), diff --git a/examples/litellm-spend/test_row_coverage.py b/examples/litellm-spend/test_row_coverage.py index af4b632..a1b8bba 100644 --- a/examples/litellm-spend/test_row_coverage.py +++ b/examples/litellm-spend/test_row_coverage.py @@ -2,12 +2,10 @@ # Code authors: Vijay and Codex """Pinned dropped-row fixture through the built local CLI; no runtime required.""" import csv -import hashlib import json import os from pathlib import Path import tempfile -import tarfile import unittest import coverage_runtime_fixture @@ -69,16 +67,10 @@ class RowCoverageTests(unittest.TestCase): def test_recorded_fixture(self): coverage_runtime_fixture.check() expected = json.loads((ROOT / "coverage_runtime_expected.json").read_text()) - # Captured provenance describes the original run, not evolving source. - # Current SQL/native parity is exercised by test_native_export.py. - archive = ROOT / "coverage_runtime_sources.tar.gz" - self.assertEqual(coverage_runtime_fixture.digest(archive), - "8c52a2b7c032e34a4232c08c806d6a09eaeb61e463770eb25592b0f1de14695c") - with tarfile.open(archive, "r:gz") as sources: - for group in ("export_source_sha256", "runtime_harness_sha256"): - for name, digest in expected["provenance"][group].items(): - with sources.extractfile("examples/litellm-spend/" + name) as source: - self.assertEqual(hashlib.sha256(source.read()).hexdigest(), digest) + # Historical source provenance is pinned in TESTING.md. Current native + # export/psql parity is exercised separately by test_native_export.py. + self.assertIn("export_source_sha256", expected["provenance"]) + self.assertIn("runtime_harness_sha256", expected["provenance"]) @unittest.skipUnless(BIN, "set FLEETDIFF_BIN for built CLI tests") def test_collision_fixture_and_safe_output(self): diff --git a/internal/cli/agents.go b/internal/cli/agents.go index 8d31ba8..79a30a8 100644 --- a/internal/cli/agents.go +++ b/internal/cli/agents.go @@ -10,6 +10,8 @@ import ( "strconv" "text/tabwriter" + "github.com/llm-measurement/fleetdiff/internal/accounting" + "github.com/llm-measurement/fleetdiff/internal/compare" "github.com/llm-measurement/fleetdiff/internal/spend" ) @@ -47,6 +49,9 @@ func renderAgents(out io.Writer, a *spend.AgentReport) { c := side.w.Coverage fmt.Fprintf(out, " %s: %d known, %d unknown, %d ambiguous rows; %d column-absent, %d empty/null, %d without a matching tag.\n", side.name, c.KnownRows, c.UnknownRows, c.AmbiguousRows, c.ColumnAbsentRows, c.EmptyOrNullRows, c.NoMatchingTagRows) fmt.Fprintf(out, " Repeated agent tags: %d in %d rows; empty suffixes: %d in %d rows.\n", c.RepeatedAgentTags, c.RepeatedAgentTagRows, c.EmptySuffixTags, c.EmptySuffixRows) + if c.NonStringTagRows > 0 { + fmt.Fprintf(out, " Non-string tag elements: %d rows counted as unknown identity.\n", c.NonStringTagRows) + } usd := func(w spend.Window) string { if w.Requests > 0 && w.Quality.SpendMissing+w.Quality.SpendInvalid == w.Requests { return "unavailable" @@ -59,7 +64,7 @@ func renderAgents(out io.Writer, a *spend.AgentReport) { fmt.Fprintln(out, " Cost coverage columns: usable/missing/invalid/zero-origin-unknown rows.") w := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0) fmt.Fprintln(w, " Agent\tTokens before\tTokens after\tUSD before\tUSD after\tUSD change\tCost coverage before\tCost coverage after\tShare before\tShare after") - for _, c := range a.Contributors { + writeContributor := func(c spend.AgentContributor) { label := c.Item if c.Label != "" { label += " " + strconv.QuoteToASCII(c.Label) @@ -69,6 +74,39 @@ func renderAgents(out io.Writer, a *spend.AgentReport) { } fmt.Fprintf(w, " %s\t%d\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", label, c.Before.Tokens, c.After.Tokens, agentUSD(c.Before.RecordedUSD), agentUSD(c.After.RecordedUSD), agentUSD(c.Delta), agentCostCoverage(c.Before.CostCoverage), agentCostCoverage(c.After.CostCoverage), agentShare(c.Before.Share), agentShare(c.After.Share)) } + for _, c := range a.Contributors { + writeContributor(c) + } _ = w.Flush() + if len(a.UnavailableCosts) > 0 { + fmt.Fprintln(out, " USD change unavailable (not ranked; shown regardless of --top):") + for _, c := range a.UnavailableCosts { + writeContributor(c) + } + _ = w.Flush() + for _, c := range a.UnavailableCosts { + delta := int64(c.After.Tokens) - int64(c.Before.Tokens) + fmt.Fprintf(out, " %s: token change %s; recorded USD change unavailable.\n", c.Item, signedInterval(compare.Interval{Lower: delta, Upper: delta})) + } + } + for _, bucket := range []struct { + name string + before, after spend.Window + }{{"Unknown", a.Before.Unknown, a.After.Unknown}, {"Ambiguous", a.Before.Ambiguous, a.After.Ambiguous}} { + cost := func(v spend.Window) *big.Rat { + if v.Requests > 0 && v.Quality.SpendMissing+v.Quality.SpendInvalid == v.Requests { + return nil + } + n, _ := new(big.Rat).SetString(v.RecordedSpend) + return n + } + x, y := cost(bucket.before), cost(bucket.after) + delta := "unavailable" + if x != nil && y != nil { + delta = accounting.CostText(new(big.Rat).Sub(y, x)) + } + tokens := int64(bucket.after.Tokens) - int64(bucket.before.Tokens) + fmt.Fprintf(out, " %s (not ranked): %d -> %d rows; token change %s; recorded USD change %s.\n", bucket.name, bucket.before.Requests, bucket.after.Requests, signedInterval(compare.Interval{Lower: tokens, Upper: tokens}), delta) + } fmt.Fprintf(out, " Known agents not displayed: %d -> %d rows; %d -> %d tokens; %s -> %s recorded USD.\n", a.UndisplayedBefore.Requests, a.UndisplayedAfter.Requests, a.UndisplayedBefore.Tokens, a.UndisplayedAfter.Tokens, agentUSD(a.UndisplayedBefore.RecordedUSD), agentUSD(a.UndisplayedAfter.RecordedUSD)) } diff --git a/internal/cli/agents_test.go b/internal/cli/agents_test.go index 2cb67bf..af87393 100644 --- a/internal/cli/agents_test.go +++ b/internal/cli/agents_test.go @@ -91,6 +91,102 @@ func TestAgentCLIFormatsAndPrivacy(t *testing.T) { } else if !strings.Contains(out, "USD change") || !strings.Contains(out, "Share after") || disclose && !strings.Contains(out, "review before sharing") { t.Fatal("required columns or warning missing") } + if format == "text" { + for _, want := range []string{ + "agent-1 accounts for 62.5% of the net recorded increase (+200 of +320 tokens)", + "Unknown (not ranked): 0 -> 1 rows; token change +50; recorded USD change 0.5", + "Ambiguous (not ranked): 0 -> 1 rows; token change +70; recorded USD change 0.7", + } { + if !strings.Contains(out, want) { + t.Fatal("answer or unattributed change missing", want) + } + } + } + } + } + } +} + +func TestAgentUnavailableCostReproduction(t *testing.T) { + t.Setenv("AGENT_CLI_SECRET", "synthetic-agent-secret-for-tests-at-least-32-bytes") + var input bytes.Buffer + w := csv.NewWriter(&input) + _ = w.Write([]string{"request_id", "call_type", "startTime", "endTime", "status", "model", "prompt_tokens", "completion_tokens", "total_tokens", "spend", "agent_id", "request_tags"}) + for i, row := range []struct{ day, agent, tokens, cost string }{ + {"07", "alpha", "100", ""}, {"07", "beta", "100", "1"}, + {"08", "alpha", "900", "9"}, {"08", "beta", "100", "2"}, + {"08", "newcomer", "50", "0.50"}, + } { + start := "2026-10-" + row.day + "T12:00:00Z" + _ = w.Write([]string{string(rune('a' + i)), "completion", start, start, "success", "private-model", row.tokens, "0", row.tokens, row.cost, row.agent, "[]"}) + } + w.Flush() + path := filepath.Join(t.TempDir(), "unavailable.csv") + if err := os.WriteFile(path, input.Bytes(), 0600); err != nil { + t.Fatal(err) + } + args := []string{"investigate", "--litellm-spend", path, "--before-period", "2026-10-07", "--after-period", "2026-10-08", "--group-by", "agent", "--agent-source", "agent_id", "--rank-by", "recorded-spend-change", "--top", "1", "--hash-secret-env", "AGENT_CLI_SECRET"} + for _, disclose := range []bool{false, true} { + for _, format := range []string{"text", "json"} { + a := append(append([]string{}, args...), "--format", format) + if disclose { + a = append(a, "--show-labels", "--acknowledge-sensitive-labels") + } + code, out, errs := databaseRun(a, nil) + if code != 0 || errs != "" || strings.Contains(out, "alpha") != disclose { + t.Fatal("unavailable reproduction failed or leaked", code, errs) + } + if format == "json" { + var r spend.Report + if json.Unmarshal([]byte(out), &r) != nil || len(r.Agents.Contributors) != 1 || r.Agents.Contributors[0].Item != "agent-2" || len(r.Agents.UnavailableCosts) != 1 || r.Agents.UnavailableCosts[0].Item != "agent-1" || r.Agents.UnavailableCosts[0].After.Tokens-r.Agents.UnavailableCosts[0].Before.Tokens != 800 || r.Agents.UndisplayedAfter.Tokens != 50 { + t.Fatal("unavailable agent hidden in JSON") + } + } else if !strings.Contains(out, "USD change unavailable (not ranked; shown regardless of --top)") || !strings.Contains(out, "agent-1: token change +800; recorded USD change unavailable") { + t.Fatal("unavailable agent hidden in text") + } + } + } +} + +func TestAgentNonStringTagFormats(t *testing.T) { + t.Setenv("AGENT_CLI_SECRET", "synthetic-agent-secret-for-tests-at-least-32-bytes") + for _, raw := range []string{`["agent:PRIVATE_INVALID_LABEL",null]`, `[1,false,{},[]]`} { + rows := []map[string]string{} + for _, day := range []string{"07", "08"} { + rows = append(rows, map[string]string{"request_id": day, "call_type": "completion", "startTime": "2026-10-" + day + "T12:00:00Z", "endTime": "2026-10-" + day + "T12:00:01Z", "status": "success", "model": "private-model", "prompt_tokens": "100", "completion_tokens": "0", "total_tokens": "100", "spend": "1", "request_tags": raw}) + } + var csvData, jsonl bytes.Buffer + writer := csv.NewWriter(&csvData) + keys := []string{"request_id", "call_type", "startTime", "endTime", "status", "model", "prompt_tokens", "completion_tokens", "total_tokens", "spend", "request_tags"} + _ = writer.Write(keys) + for _, row := range rows { + values := []string{} + for _, k := range keys { + values = append(values, row[k]) + } + _ = writer.Write(values) + _ = json.NewEncoder(&jsonl).Encode(row) + } + writer.Flush() + array, _ := json.Marshal(rows) + for ext, data := range map[string][]byte{"csv": csvData.Bytes(), "json": array, "jsonl": jsonl.Bytes()} { + path := filepath.Join(t.TempDir(), "non-string."+ext) + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + for _, format := range []string{"json", "text"} { + code, out, errs := databaseRun(append(agentCLIArgs(path), "--format", format, "--show-labels", "--acknowledge-sensitive-labels"), nil) + if code != 0 || errs != "" || strings.Contains(out, "PRIVATE_INVALID_LABEL") { + t.Fatal("non-string tag rejected or label salvaged", code, errs) + } + if format == "json" { + var r spend.Report + if json.Unmarshal([]byte(out), &r) != nil || r.Agents.After.Coverage.NonStringTagRows != 1 || r.Agents.After.Unknown.Tokens != 100 || r.After.RecordedSpend != "1" { + t.Fatal("non-string coverage or totals wrong") + } + } else if !strings.Contains(out, "Non-string tag elements: 1 rows counted as unknown identity") { + t.Fatal("non-string coverage not explained") + } } } } diff --git a/internal/cli/spend.go b/internal/cli/spend.go index e6a31c3..48af17c 100644 --- a/internal/cli/spend.go +++ b/internal/cli/spend.go @@ -114,6 +114,21 @@ func renderSpend(out io.Writer, r spend.Report) { fmt.Fprintln(out) } if r.Agents != nil { + if c := r.Increase; c != nil && !r.Incomplete { + label, verb := fmt.Sprintf("%d leading tracked agents", c.Count), "account" + if c.Count == 1 && len(r.Rankings.Movers) > 0 { + label, verb = r.Rankings.Movers[0].Item, "accounts" + } + share := spendPercent(100 * c.Share.Lower) + if c.Delta.Lower != c.Delta.Upper { + lower, upper := c.Share.Lower*10000, c.Share.Upper*10000 + if c.Share.Lower == c.Share.Upper { + lower, upper = math.Nextafter(lower, math.Inf(-1)), math.Nextafter(upper, math.Inf(1)) + } + share = fmt.Sprintf("[%s, %s]", spendPercent(math.Floor(lower)/100), spendPercent(math.Ceil(upper)/100)) + } + fmt.Fprintf(out, " %s %s for %s of the net recorded increase (%s of +%s tokens).\n", label, verb, share, signedInterval(c.Delta), spendNumber(r.After.Tokens-r.Before.Tokens)) + } renderAgents(out, r.Agents) } for _, m := range r.Rankings.Movers { diff --git a/internal/spend/agent_spend_test.go b/internal/spend/agent_spend_test.go index d519a6d..5ee70a0 100644 --- a/internal/spend/agent_spend_test.go +++ b/internal/spend/agent_spend_test.go @@ -95,3 +95,29 @@ func TestAgentCostAggregateExceedsPerRowBound(t *testing.T) { t.Fatal("aggregate cost reapplied per-row validation") } } + +func TestAgentUnavailableCostNeverHiddenByTop(t *testing.T) { + rows := agentFixtureRows()[:6] + for i, row := range rows { + row["spend"] = []string{"", "1", "", "9", "2", ""}[i] + } + o := agentOptions(t) + o.RankBy, o.Top = "recorded-spend-change", 1 + r := routingReport(t, rows, o) + if len(r.Agents.Contributors) != 1 || r.Agents.Contributors[0].Item != "agent-2" || len(r.Agents.UnavailableCosts) != 2 || r.Agents.UndisplayedBefore.Requests != 0 || r.Agents.UndisplayedAfter.Requests != 0 { + t.Fatal("unavailable agent hidden by --top", r.Agents) + } + for _, c := range r.Agents.UnavailableCosts { + if c.Delta != nil || c.Before.RecordedUSD != nil || c.Before.CostCoverage.Missing != 1 { + t.Fatal("unavailable cost became a zero delta", c) + } + } + // With every cost unavailable, no agents are silently folded into a remainder. + for _, row := range rows { + row["spend"] = "" + } + r = routingReport(t, rows, o) + if len(r.Agents.Contributors) != 0 || len(r.Agents.UnavailableCosts) != 3 || r.Agents.UndisplayedAfter.Requests != 0 { + t.Fatal("all-unavailable ranking dropped agents") + } +} diff --git a/internal/spend/agents.go b/internal/spend/agents.go index 6304e9f..c3f76ab 100644 --- a/internal/spend/agents.go +++ b/internal/spend/agents.go @@ -26,6 +26,7 @@ type AgentCoverage struct { ColumnAbsentRows uint64 `json:"column_absent_rows"` EmptyOrNullRows uint64 `json:"empty_or_null_rows"` NoMatchingTagRows uint64 `json:"no_matching_tag_rows"` + NonStringTagRows uint64 `json:"non_string_tag_rows"` RepeatedAgentTagRows uint64 `json:"repeated_agent_tag_rows"` RepeatedAgentTags uint64 `json:"repeated_agent_tags"` EmptySuffixRows uint64 `json:"empty_suffix_rows"` @@ -47,6 +48,7 @@ type AgentReport struct { RankBy string `json:"rank_by"` LabelsIncluded bool `json:"sensitive_labels_included"` Contributors []AgentContributor `json:"contributors"` + UnavailableCosts []AgentContributor `json:"unavailable_cost_contributors,omitempty"` UndisplayedBefore AgentValues `json:"known_undisplayed_before"` UndisplayedAfter AgentValues `json:"known_undisplayed_after"` ShareBasis string `json:"spend_share_basis"` @@ -78,7 +80,7 @@ type AgentContributor struct { type agentResolution struct { state, label, source, prefix string - absent, empty bool + absent, empty, nonString bool repeated, emptySuffix uint64 } @@ -134,6 +136,10 @@ func resolveAgent(row Row, o Options) (*agentResolution, error) { } tags, err := ParseRequestTags(raw) if err != nil { + if errors.Is(err, ErrNonStringRequestTags) && row.AgentNamespace == "" { + a.nonString = true + return a, nil + } return nil, err } // Deduplicate matching suffixes; retain no unselected tag values. @@ -163,6 +169,8 @@ func resolveAgent(row Row, o Options) (*agentResolution, error) { return a, nil } +var ErrNonStringRequestTags = errors.New("invalid request_tags; every array element must be a string") + // ParseRequestTags validates bounded scalar-array text for SQL and observation // adapters before attribution or duplicate normalization can discard evidence. func ParseRequestTags(raw string) ([]string, error) { @@ -180,7 +188,7 @@ func ParseRequestTags(raw string) ([]string, error) { for _, element := range elements { var tag string if len(element) == 0 || element[0] != '"' || json.Unmarshal(element, &tag) != nil { - return nil, errors.New("invalid request_tags; every array element must be a string") + return nil, ErrNonStringRequestTags } tags = append(tags, tag) } @@ -226,6 +234,9 @@ func (a *agentAccumulator) add(row Row, side int, digest [32]byte, obs accountin if r.empty { c.EmptyOrNullRows++ } + if r.nonString { + c.NonStringTagRows++ + } if r.repeated > 0 { c.RepeatedAgentTagRows++ c.RepeatedAgentTags += r.repeated @@ -261,7 +272,7 @@ func (a *agentAccumulator) add(row Row, side int, digest [32]byte, obs accountin p = a.ambiguous default: c.UnknownRows++ - if !r.absent && !r.empty && r.source == "request-tag" { + if !r.absent && !r.empty && !r.nonString && r.source == "request-tag" { c.NoMatchingTagRows++ } p = a.unknown @@ -366,14 +377,6 @@ func (a *agentAccumulator) finish(r *Report) error { remaining := [2]*routingTotals{newRoutingTotals(), newRoutingTotals()} report.Contributors = []AgentContributor{} for i, e := range entries { - if i >= a.opts.Top { - for side := range remaining { - if err := remaining[side].merge(&routingTotals{a: e.pair[side]}); err != nil { - return err - } - } - continue - } c := AgentContributor{Item: fmt.Sprintf("agent-%d", e.order+1), Label: e.label, Before: agentValues(e.pair[0].result(), totals[0]), After: agentValues(e.pair[1].result(), totals[1])} if a.opts.ShowHashes { c.Hash = fmt.Sprintf("%x", e.digest) @@ -382,6 +385,18 @@ func (a *agentAccumulator) finish(r *Report) error { delta := costText(new(big.Rat).Sub(&e.pair[1].cost, &e.pair[0].cost)) c.Delta = &delta } + if a.opts.RankBy == "recorded-spend-change" && c.Delta == nil { + report.UnavailableCosts = append(report.UnavailableCosts, c) + continue + } + if i >= a.opts.Top { + for side := range remaining { + if err := remaining[side].merge(&routingTotals{a: e.pair[side]}); err != nil { + return err + } + } + continue + } report.Contributors = append(report.Contributors, c) } report.UndisplayedBefore = agentValues(remaining[0].a.result(), totals[0]) diff --git a/internal/spend/agents_test.go b/internal/spend/agents_test.go index 139247c..fdff833 100644 --- a/internal/spend/agents_test.go +++ b/internal/spend/agents_test.go @@ -28,19 +28,42 @@ func TestAgentTagResolution(t *testing.T) { {`["agent:a","agent:a","agent:"]`, "known", "a", 1, 1}, {`["agent:a","agent:b","agent:a"]`, "ambiguous", "", 1, 0}, {`["agent:","agent: "]`, "unknown", "", 0, 2}, + {`[null]`, "unknown", "", 0, 0}, + {`[1]`, "unknown", "", 0, 0}, + {`["agent:a",true,{"private":"value"}]`, "unknown", "", 0, 0}, } { r, err := resolveAgent(Row{Values: map[string]string{"request_tags": tc.raw}}, agentOptions(t)) if err != nil || r.state != tc.state || r.label != tc.label || r.repeated != tc.repeats || r.emptySuffix != tc.empty { t.Fatal("tag resolution mismatch", tc, err) } } - for _, raw := range []string{`{}`, `[null]`, `[1]`, `"agent:a"`, `["agent:a"`, `["agent:\ud800"]`, strings.Repeat("x", MaxFieldBytes+1)} { + for _, raw := range []string{`{}`, `"agent:a"`, `["agent:a"`, `["agent:\ud800"]`, `[1,"\ud800"]`, strings.Repeat("x", MaxFieldBytes+1)} { if _, err := resolveAgent(Row{Values: map[string]string{"request_tags": raw}}, agentOptions(t)); err == nil { t.Fatal("malformed tags accepted") } } } +func TestAgentNonStringTagsPreserveAccounting(t *testing.T) { + for _, raw := range []string{`[null]`, `[1]`, `[false]`, `[{"PRIVATE":"value"}]`, `[["agent:a"]]`, `["agent:alpha",1]`} { + rows := agentFixtureRows() + rows[3]["request_tags"] = raw + o := agentOptions(t) + for _, disclose := range []bool{false, true} { + o.ShowLabels, o.AcknowledgeSensitiveLabels = disclose, disclose + r := routingReport(t, rows, o) + w := r.Agents.After + if r.After.Tokens != 620 || r.After.RecordedSpend != "6.2" || w.Known.Tokens != 200 || w.Unknown.Tokens != 350 || w.Unknown.RecordedSpend != "3.5" || w.Ambiguous.Tokens != 70 || w.Coverage.NonStringTagRows != 1 || w.Coverage.UnknownRows != 2 || w.Coverage.NoMatchingTagRows != 1 { + t.Fatal("non-string tags changed accounting or coverage", raw, w) + } + b, _ := json.Marshal(r) + if strings.Contains(string(b), "PRIVATE") || strings.Contains(string(b), `"label":"alpha"`) && disclose == false { + t.Fatal("invalid tags escaped") + } + } + } +} + func agentFixtureRows() []map[string]string { var rows []map[string]string for side, day := range []string{"2026-09-01", "2026-09-02"} { From 7b56ab203dd507bd4f1b3b548fec56a2a61fa68e Mon Sep 17 00:00:00 2001 From: Vijay Date: Thu, 29 Oct 2026 18:10:21 -0400 Subject: [PATCH 6/7] feat: polish agent reports and prepare v0.14.0 Signed-off-by: Vijay --- CHANGELOG.md | 9 ++ README.md | 18 ++-- SECURITY.md | 1 + docs/AGENT_ATTRIBUTION.md | 145 +++++++++++++++------------- docs/LANGFUSE.md | 6 +- docs/OPERATIONS.md | 19 ++-- docs/RELEASING.md | 11 ++- docs/ROW_COVERAGE.md | 2 +- docs/releases/v0.14.0.md | 62 ++++++++++++ examples/litellm-spend/README.md | 4 +- examples/litellm-spend/test_docs.py | 22 ++++- internal/cli/agents.go | 63 ++++++++---- internal/cli/agents_test.go | 103 +++++++++++++++++++- internal/cli/spend.go | 9 +- internal/langfuse/agents_test.go | 13 +++ internal/spend/agent_spend_test.go | 16 +++ internal/spend/agents.go | 57 ++++++----- scripts/install.sh | 4 +- scripts/install_test.go | 2 +- 19 files changed, 431 insertions(+), 135 deletions(-) create mode 100644 docs/releases/v0.14.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 24b50ec..e1baa0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Notable user-visible changes are recorded here, newest first. ## [Unreleased] +## [0.14.0] - Unreleased + +Release preparation; no tag or publication yet. See [draft release notes](docs/releases/v0.14.0.md). + - Add explicit agent attribution from LiteLLM `agent_id` or prefixed request tags on LiteLLM and Langfuse, with exclusive known/unknown/ambiguous totals, repeated/empty-tag coverage and an exact 1,024-agent bound. @@ -16,6 +20,11 @@ Notable user-visible changes are recorded here, newest first. `fleetdiff investigate` commands. Raw agent labels require the same two flags. User/session/end-user/key rankings remain aliased. See the [agent attribution contract](docs/AGENT_ATTRIBUTION.md). +- Include escaped selected agent labels in the answer sentence, distinguish + identical Langfuse labels with project aliases, and keep raw project names private. +- Keep agent text compact: omit tag-only coverage for agent IDs, show cost + coverage columns only for incomplete displayed costs, and name unknown and + ambiguous identity explicitly. Agent JSON and default reports are unchanged. ## [0.13.1] - 2026-10-27 diff --git a/README.md b/README.md index e6c6221..2b9872e 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,17 @@ fleetdiff reads LiteLLM databases and exports or Langfuse exports to show what c + -### 1. Install v0.13.1 +### 1. Install v0.14.0 + +These v0.14.0 instructions require its published release assets. Until +publication, use [v0.13.1](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.13.1). Linux or macOS; needs curl, gh, shasum and tar. The installer checks the release before unpacking it. ```sh -bash -o pipefail -c 'curl --fail --location --proto "=https" --tlsv1.2 https://raw.githubusercontent.com/llm-measurement/fleetdiff/0d7db95c5bb3013f8c564c91fbb07b6235a2664e/scripts/install.sh | sh -s -- v0.13.1 ./fleetdiff-install' +bash -o pipefail -c 'curl --fail --location --proto "=https" --tlsv1.2 https://raw.githubusercontent.com/llm-measurement/fleetdiff/0d7db95c5bb3013f8c564c91fbb07b6235a2664e/scripts/install.sh | sh -s -- v0.14.0 ./fleetdiff-install' ./fleetdiff-install/fleetdiff --version ``` @@ -85,7 +89,7 @@ Install fleetdiff using [step 1](#1-install-v0120), then run the synthetic examp ```sh curl --fail --location --proto '=https' --tlsv1.2 \ - https://raw.githubusercontent.com/llm-measurement/fleetdiff/v0.13.1/examples/litellm-spend/synthetic.csv \ + https://raw.githubusercontent.com/llm-measurement/fleetdiff/v0.14.0/examples/litellm-spend/synthetic.csv \ --output synthetic.csv ./fleetdiff-install/fleetdiff investigate --litellm-spend synthetic.csv \ --before-period 2026-10-07 --after-period 2026-10-08 @@ -118,16 +122,16 @@ The summary-file path also shows session concentration: ## Install -The verified v0.13.1 installer selects the platform binary for your machine. +The installer verifies the v0.14.0 platform binary for your machine after publication. For public release downloads, `gh` need not be signed in. -For manual downloads, use the [v0.13.1 archives](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.13.1) +For manual downloads, use the [v0.14.0 archives](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.14.0) and [verification instructions](docs/OPERATIONS.md#install-and-verify). An administrator can verify and transfer artifacts through an approved internal mirror. With a security-patched Go 1.26 toolchain, install the same version from its module: ```sh -go install github.com/llm-measurement/fleetdiff/cmd/fleetdiff@v0.13.1 +go install github.com/llm-measurement/fleetdiff/cmd/fleetdiff@v0.14.0 ``` The executable goes to `GOBIN`, or `$(go env GOPATH)/bin` when unset. @@ -174,7 +178,7 @@ Guides: [LiteLLM spend](https://llm-measurement.github.io/litellm-spend.html) | - [Questions and contributor rankings](docs/WORKFLOWS.md#questions-it-answers). - [How summary comparison works](docs/WORKFLOWS.md#how-it-works), [comparison contract](docs/COMPARISON.md) and [investigation contract](docs/INVESTIGATION.md). - [Operations](docs/OPERATIONS.md), [resource measurements](docs/BENCHMARKS.md) and [checks](docs/WORKFLOWS.md#built-to-be-checked). -- [FAQ](docs/FAQ.md), [security](SECURITY.md), [release notes](docs/releases/v0.13.1.md) and [release procedure](docs/RELEASING.md). +- [FAQ](docs/FAQ.md), [security](SECURITY.md), [release notes](docs/releases/v0.14.0.md) and [release procedure](docs/RELEASING.md). Questions or feedback: [open an issue](https://github.com/llm-measurement/fleetdiff/issues). See [Contributing](CONTRIBUTING.md) for checks and signed, signed-off commits. diff --git a/SECURITY.md b/SECURITY.md index 4060fde..e706f00 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -3,6 +3,7 @@ ## Supported Versions Security fixes target the latest patch in the `0.13.x` release line and `main`. +Once v0.14.0 is published, fixes target the `0.14.x` release line and `main`. Use the newest patch and record its version and revision when running trials. Older patches and development snapshots do not receive routine backports. Dependencies and the Go toolchain are reviewed on an ongoing basis. diff --git a/docs/AGENT_ATTRIBUTION.md b/docs/AGENT_ATTRIBUTION.md index 380e48f..a3d11c0 100644 --- a/docs/AGENT_ATTRIBUTION.md +++ b/docs/AGENT_ATTRIBUTION.md @@ -1,7 +1,17 @@ -# Agent Attribution +# Agent Spend Attribution -Development scope for the next minor release. The released v0.13.1 binary does -not support these options. Build this checkout to run the examples: +Find which recorded agents drove a token or spend change between two periods +of LiteLLM spend logs or Langfuse exports. + +The included synthetic fixture goes from **300 to 620 tokens** and **USD 3 to +6.20**. Alpha adds **200 of the 320-token net increase (62.5%)**. After the +change, known agents account for 500 tokens/USD5, unknown identity for +50 tokens/USD 0.50, and ambiguous identity for 70 tokens/USD 0.70. + +## Run The Fixture + +These options are prepared for v0.14.0, not available in v0.13.1. Build this +checkout while release publication is pending: ```sh go build -o bin/fleetdiff ./cmd/fleetdiff @@ -11,68 +21,73 @@ bin/fleetdiff investigate --litellm-spend examples/litellm-spend/agent-tags.csv --rank-by recorded-spend-change ``` -The fixture records 620 tokens and $6.20 after the change: known agents account -for 500 tokens/$5, unknown identity for 50/$0.50, and ambiguous identity for -70/$0.70. `agents.csv` is the three-agent fixture without the last two rows. -Agent alpha's tokens increase from 100 to 300. These are synthetic observations. - -## Source Selection - -`--group-by agent` requires an explicit `--agent-source`: - -- `agent_id`: the recorded LiteLLM spend-log ID, on CSV/JSON/JSONL or database. -- `request-tag`: request tags with a required `--agent-tag-prefix`. Supported on - LiteLLM sources and Langfuse exports. The prefix is exact and case-sensitive; - suffixes retain their source spelling. No fallback to another field occurs. - -For LiteLLM database investigation, replace the file option with `--litellm-db`; -the selected tag source enables tag export automatically. Standalone native -export requires `--request-tags`; psql requires `-v request_tags=true`. -See [export limits](NATIVE_EXPORT.md#limits-and-failures) and the adjacent -[optional column grant](NATIVE_EXPORT.md#administrator-setup). - -Zero matching nonblank suffixes means unknown identity. One distinct suffix -means a known agent, even if repeated. Two or more distinct suffixes means -ambiguous identity. Repeated matching tags and empty suffixes have separate -coverage counts; empty suffixes never match an agent. Unknown and ambiguous rows -retain all their accounting in separate buckets. Each analyzed row is counted -once. Langfuse identities remain project-scoped, and its unit stays GENERATION -observations. No request/observation join is inferred between sources. - -## Ranking And Disclosure - -The default agent table orders by signed recorded-token change. -`--rank-by recorded-spend-change` orders by signed change in usable recorded USD, -descending, with first appearance breaking ties. An unavailable cost delta sorts -after available deltas and is listed separately, never hidden by `--top`. -Before/after USD, delta, cost coverage and each period's -share appear beside tokens. A period with no usable cost rows is unavailable, -not a recorded zero. Known agents outside `--top` are summed separately. - -Shares use usable recorded USD in all analyzed rows in that period, including -unknown, ambiguous and undisplayed agents. Missing/invalid costs remain -unquantified; zero-cost origin remains unknown. Shares are rounded to 12 decimal -places in JSON and two percentage places in text. Costs retain exact source -decimals. Recorded USD is source accounting, not an invoice. Unsupported -LiteLLM call types and partial Langfuse exports keep their existing boundaries. - -Default output uses keyed aliases. To include only the selected agent label, -add both `--show-labels --acknowledge-sensitive-labels`; either alone fails. -Reports identify disclosure and warn to review before sharing. Other tag values, -user/session/end-user/key identities and unrelated routing labels remain hidden. -Recorded IDs and tags can contain personal identifiers; no friendly-name lookup, -arbitrary metadata extraction or automatic agent discovery is performed. +The answer includes: + +```text +agent-1 accounts for 62.5% of the net recorded increase (+200 of +320 tokens). +``` + +The table shows before/after tokens, recorded USD, USD change and each period's +spend share. Unknown and ambiguous changes appear separately below the ranking. +The four cost-coverage columns appear only when a displayed row has missing, +invalid or zero-origin-unknown cost; otherwise one line confirms usable recorded +USD in both periods. Known agents outside `--top` remain in a separate total. +Agents whose USD change is unavailable are always listed, even with `--top`. + +## Choose The Source + +- LiteLLM recorded IDs: `--group-by agent --agent-source agent_id`. +- LiteLLM or Langfuse tags: `--group-by agent --agent-source request-tag + --agent-tag-prefix 'agent:'`. + +The default ranking is signed recorded-token change. Add +`--rank-by recorded-spend-change` for signed recorded-USD change, descending; +first appearance breaks ties. + +For a LiteLLM database, replace the file option with `--litellm-db`. Tag +attribution enables the required tag projection automatically. Standalone +native export needs `--request-tags`; psql needs `-v request_tags=true`. +Exporting tags retains all tag values, not just the selected prefix. Review +the [export limits](NATIVE_EXPORT.md#limits-and-failures) and +[optional column grant](NATIVE_EXPORT.md#administrator-setup) before sharing an +export. These transport options do not disclose unrelated tags in reports. + +## Include Selected Labels + +Add both `--show-labels --acknowledge-sensitive-labels`; either alone fails. +The selected label is escaped in the answer and table, for example: + +```text +agent-1 "alpha" accounts for 62.5% of the net recorded increase (+200 of +320 tokens). +``` + +Langfuse identities stay project-scoped. Identical labels are distinguished as +`agent-1 "alpha" (project-1)` and `agent-2 "alpha" (project-2)`, never with raw +project names. Project aliases describe this report's encounter order, not a +cross-report project ID. Unrelated tags and user/session/end-user/key identities +remain hidden; the text warns before sharing disclosed labels. ## Limits -The agent table is exact and bounded to 1,024 known agents across both periods, -including project-scoped agents. Overflow fails and asks for a narrower prefix -or export; the error quotes the selected prefix. Malformed JSON, non-array tag -data or data exceeding the existing 8 KiB field bound fail, without raw row data. -LiteLLM arrays containing any non-string element count as unknown identity with -separate `non_string_tag_rows` coverage, even if they also contain an agent tag; -no label is salvaged. Langfuse tag arrays retain their string-only validation. -An old export without tags remains readable and reports column-absent unknown -identity. The optional `agents` JSON section is versioned independently as -`fleetdiff-agent-attribution/v1`; collector summaries and the base accounting -fingerprint are unchanged. Default export projections remain unchanged. +- The selector is explicit: no field fallback, agent discovery, friendly-name + lookup or request/observation join. Langfuse counts GENERATION observations. +- Prefixes are exact and case-sensitive. Zero distinct nonblank suffixes means + unknown, one means known, two or more means ambiguous. Repeated identical tags + are deduplicated; repetitions and empty suffixes have separate coverage. + Each row contributes once, and all identity buckets retain their accounting. +- LiteLLM arrays with any non-string element remain unknown with + `non_string_tag_rows` coverage; no string label is salvaged. Langfuse retains + string-only arrays. Missing source columns remain column-absent unknown. +- Malformed JSON, non-array tags, invalid UTF-8 and source fields over 8 KiB fail + without dumping rows. More than 1,024 project-scoped known agents fails and + asks for a narrower prefix or export; identities are not silently dropped. +- Shares use usable recorded USD from all analyzed rows, including unknown, + ambiguous and undisplayed agents. Missing/invalid costs stay unquantified; + zero-cost origin stays unknown. A period with no usable cost is unavailable, + not zero. Costs retain source decimals; JSON shares use 12 decimal places and + text percentages use two. Recorded USD is source accounting, not an invoice. +- The optional JSON section remains `fleetdiff-agent-attribution/v1`. Its + fields, base accounting fingerprint, default export projection and reports + without agent mode are unchanged. Project display aliases are text-only. + Unsupported LiteLLM call types and partial Langfuse exports retain their + existing boundaries. diff --git a/docs/LANGFUSE.md b/docs/LANGFUSE.md index b8859ef..6980429 100644 --- a/docs/LANGFUSE.md +++ b/docs/LANGFUSE.md @@ -21,11 +21,11 @@ It reads local files only, not Langfuse services or provider APIs. ## Try the synthetic example -Install using the [README](../README.md#1-install-v0131), then get the example -files (the installer does not include a source checkout): +After v0.14.0 is published, install using the [README](../README.md#1-install-v0140), +then get the example files (the installer does not include a source checkout): ```sh -git clone --depth 1 --branch v0.13.1 https://github.com/llm-measurement/fleetdiff +git clone --depth 1 --branch v0.14.0 https://github.com/llm-measurement/fleetdiff ``` From the directory containing `fleetdiff-install` and this checkout, compare diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 0ae6fbb..86012e8 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -10,8 +10,9 @@ private local exports before analysis. The initial supported targets are Linux and macOS on AMD64 and ARM64. Native Windows binaries are not provided. Download a binary from the -[release page](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.13.1). -Go is not needed to run it. The commands below use `curl` for downloading and +[release page](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.14.0). +These v0.14.0 download commands require its published release assets; use the +v0.13.1 release until publication. Go is not needed to run it. The commands below use `curl` for downloading and the [GitHub CLI](https://cli.github.com/) for provenance verification. The commands select Linux or macOS and AMD64 or ARM64 from the current shell's @@ -22,7 +23,7 @@ subshell stops on any download or verification failure: ```sh ( set -eu -version=v0.13.1 +version=v0.14.0 case "$(uname -s)" in Linux) os=linux ;; Darwin) os=darwin ;; @@ -56,7 +57,7 @@ From a reviewed repository checkout, the same download and verification steps are available as a script (no Go compiler needed): ```sh -sh scripts/install.sh v0.13.1 ./fleetdiff-install +sh scripts/install.sh v0.14.0 ./fleetdiff-install ./fleetdiff-install/fleetdiff --version ``` @@ -67,14 +68,14 @@ curl --fail --location --proto '=https' --tlsv1.2 \ https://raw.githubusercontent.com/llm-measurement/fleetdiff/0d7db95c5bb3013f8c564c91fbb07b6235a2664e/scripts/install.sh \ --output fleetdiff-install.sh less fleetdiff-install.sh -sh fleetdiff-install.sh v0.13.1 ./fleetdiff-install +sh fleetdiff-install.sh v0.14.0 ./fleetdiff-install ./fleetdiff-install/fleetdiff --version ``` -The immutable script pin is a reviewed installer; the explicit `v0.13.1` +The immutable script pin is a reviewed installer; the explicit `v0.14.0` argument selects the release independently of that script's older default. -The v0.13.1 binary includes [native LiteLLM export](NATIVE_EXPORT.md), +The v0.14.0 binary includes [native LiteLLM export](NATIVE_EXPORT.md), [model-mix what-if repricing](WHAT_IF.md), the [LiteLLM spend importer](../examples/litellm-spend/README.md), optional [daily row coverage](ROW_COVERAGE.md), [Langfuse local exports](LANGFUSE.md) @@ -124,7 +125,7 @@ Install a published version with a currently patched Go 1.26 or later toolchain: go install github.com/llm-measurement/fleetdiff/cmd/fleetdiff@latest ``` -Use `@v0.13.1` instead of `@latest` to pin that release. The binary is installed +Use `@v0.14.0` instead of `@latest` to pin that release. The binary is installed under `GOBIN`, or `$(go env GOPATH)/bin` when unset; put that directory on `PATH`. This path uses Go module verification rather than the archive attestation above, and does not include unreleased checkout features. @@ -145,7 +146,7 @@ Check `--version` on the binary you actually run: | Build | Version | Revision | |---|---|---| | Unstamped checkout build above | Go's embedded module/VCS version, or `dev` when unavailable | `unknown` | -| Tagged `go install ...@v0.13.1` | `v0.13.1`, from Go's module metadata | `unknown` | +| Tagged `go install ...@v0.14.0` | `v0.14.0`, from Go's module metadata | `unknown` | | Verified release archive | Stamped release version | Stamped source revision | A checkout may report a pseudo-version and `+dirty`, depending on the Go toolchain diff --git a/docs/RELEASING.md b/docs/RELEASING.md index d9780ca..d7cec26 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -97,7 +97,7 @@ verify that every advertised version is downloadable. Do not replace immutable t or artifacts to correct a failed publication; fix forward with a new version if any release content must change. -## After Publishing v0.13.1 +## After Publishing v0.14.0 From an anonymous clone in a fresh directory outside any synced workspace: @@ -106,7 +106,7 @@ From an anonymous clone in a fresh directory outside any synced workspace: [OPERATIONS](OPERATIONS.md#install-and-verify). Confirm the signed tag and recorded build revision match the approved commit. 2. Run the README's verified installer into a new directory. Confirm the native - executable reports `v0.13.1`, then run the packaged-binary tests above. + executable reports `v0.14.0`, then run the packaged-binary tests above. 3. Set `FLEETDIFF_BIN` to that freshly installed executable and run the exact [pinned database test](../examples/litellm-spend/TESTING.md#pinned-database-integration). It exercises the real pinned writer, a SELECT-only SQL export and the released @@ -154,6 +154,13 @@ From an anonymous clone in a fresh directory outside any synced workspace: Exercise the README's database investigation, private cleanup and retained exports. Record the exact executable and distinguish native-driver tests from the older writer/psql tests; skipped database checks do not pass this gate. +13. Run the agent-tag fixture from AGENT_ATTRIBUTION with the installed binary. + Check the 620-token/USD6.20 totals, alpha's +200 of +320-token answer, + unknown/ambiguous buckets, unavailable-cost visibility and two-flag escaped + label disclosure. Check project aliases on a two-project Langfuse export. + Retain the 18 default text/JSON byte comparisons against v0.13.1 separately + from these opt-in agent checks. Publish site pins and guide text only after + these released-artifact gates pass; do not treat source tests as that proof. The release workflow tests every native platform before publication. The local outsider run proves the post-publication download and database path on its own diff --git a/docs/ROW_COVERAGE.md b/docs/ROW_COVERAGE.md index 671c92d..4ba8bbf 100644 --- a/docs/ROW_COVERAGE.md +++ b/docs/ROW_COVERAGE.md @@ -8,7 +8,7 @@ fixture reproduces a real response-ID collision through pinned LiteLLM's proxy and database writers. The next UTC day matches: 1 row and 1 daily request. The check leaves token and spend totals exactly as recorded in the export. -Install [fleetdiff v0.9.0](../README.md#2-install-v090). From the repository +Install [fleetdiff v0.14.0](../README.md#1-install-v0140) after publication. From the repository root, select the verified binary for the commands below: ```sh diff --git a/docs/releases/v0.14.0.md b/docs/releases/v0.14.0.md new file mode 100644 index 0000000..ae362fd --- /dev/null +++ b/docs/releases/v0.14.0.md @@ -0,0 +1,62 @@ +# fleetdiff v0.14.0 + +Draft release preparation. Not tagged or published; merge, tagging and release +each require maintainer approval. + +Compare two periods of LiteLLM spend logs or Langfuse exports to see which +recorded agents drove token and spend changes. Choose an explicit agent ID or +request-tag source; the report does not guess agent identities. + +## Agent Attribution + +- Use `--group-by agent --agent-source agent_id` for LiteLLM's recorded agent IDs. +- Use `--agent-source request-tag --agent-tag-prefix 'agent:'` for LiteLLM or + Langfuse tags. Identical repeated tags resolve to one agent; distinct matching + labels remain ambiguous, and unknown/ambiguous rows retain their accounting. +- Rank signed token changes, or add `--rank-by recorded-spend-change` for exact + recorded-USD changes. USD-unavailable agents stay visible even with `--top`. +- See the answer sentence, before/after tokens and USD, period shares and + unattributed changes. Cost-coverage columns appear only for incomplete + displayed costs. Recorded USD is not a provider invoice. +- Raw selected labels require both disclosure flags and are escaped in text. + Identical Langfuse labels remain project-scoped and use project aliases; + raw project names are not displayed. +- Keep invalid LiteLLM tag-element rows as unknown identity with separate + coverage. Langfuse retains string-only validation. Exact agent counting is + bounded to 1,024 identities; no silent truncation occurs. + +The synthetic fixture records 300 -> 620 tokens and USD 3 -> 6.20. Alpha adds +200 of the 320-token net increase (62.5%). See the +[agent guide](../AGENT_ATTRIBUTION.md) for commands and limits. + +## Breaking Routing Label Disclosure + +Routing `--show-labels` now also requires `--acknowledge-sensitive-labels`. +Replace an existing routing command such as: + +```sh +fleetdiff investigate --litellm-spend spend.csv --routing-audit --show-labels +``` + +with: + +```sh +fleetdiff investigate --litellm-spend spend.csv --routing-audit \ + --show-labels --acknowledge-sensitive-labels +``` + +The same two flags disclose selected agent labels. User, session, end-user and +key rankings remain aliased. Default exports and reports remain byte-identical +to v0.13.1; the agent JSON section retains its independently versioned contract. +No new runtime dependency is introduced. + +## Publication Gates + +Follow [RELEASING](../RELEASING.md). Finalize these draft notes and the changelog +date only after release approval. Published archives, checksums, SBOM/provenance, +anonymous installer tests, packaged-binary and pinned database checks, and site +version/provenance verification cannot pass from an unpublished local checkout. +Site re-pinning and the new agent guide remain unpublished until the released +artifact checks pass. Do not create a tag merely to test packaging. + +Code authors: Vijay and Codex diff --git a/examples/litellm-spend/README.md b/examples/litellm-spend/README.md index d3a7858..f38d79b 100644 --- a/examples/litellm-spend/README.md +++ b/examples/litellm-spend/README.md @@ -7,7 +7,7 @@ increase, and two keys account for all of it.** The report separates request count from request size where the usage records support that comparison. -Install [fleetdiff v0.13.1](../../README.md#1-install-v0131), then run this fixture +Install [fleetdiff v0.14.0](../../README.md#1-install-v0140) after publication, then run this fixture from the repository root with the verified binary: ```sh @@ -357,7 +357,7 @@ the call types and asks for a supported request-level export. [session-budget.csv](session-budget.csv) is a newly authored, deterministic Apache-2.0 fixture with four sessions and no production data. The companion [license notice](session-budget.csv.license) keeps the CSV directly importable. -Use fleetdiff v0.13.1 for the session-budget command: +Use fleetdiff v0.14.0 after publication for the session-budget command: ```sh fleetdiff investigate --litellm-spend examples/litellm-spend/session-budget.csv \ diff --git a/examples/litellm-spend/test_docs.py b/examples/litellm-spend/test_docs.py index 9ecfc1c..f837cb6 100644 --- a/examples/litellm-spend/test_docs.py +++ b/examples/litellm-spend/test_docs.py @@ -67,7 +67,7 @@ def test_readme_shape_and_previous_anchors(self): def test_native_release_promotion(self): readme = (ROOT / "README.md").read_text() - self.assertLess(readme.index("### 1. Install v0.13.1"), + self.assertLess(readme.index("### 1. Install v0.14.0"), readme.index("investigate --litellm-db")) self.assertIn("Manual Alternative: Export With psql", readme) self.assertNotIn("go build", readme) @@ -86,6 +86,23 @@ def test_native_release_promotion(self): self.assertNotRegex(content.lower(), r"unreleased v0\.12|native candidate|candidate (?:build|recipe|checkout)") + def test_v014_preparation_and_agent_guide(self): + readme = (ROOT / "README.md").read_text() + installer = (ROOT / "scripts/install.sh").read_text() + self.assertIn("v0.14.0", installer) + self.assertNotIn("v0.13.1", installer) + self.assertIn("publication", readme) + self.assertIn("v0.14.0", (ROOT / "docs/ROW_COVERAGE.md").read_text()) + guide = (ROOT / "docs/AGENT_ATTRIBUTION.md").read_text() + self.assertLess(guide.index("300 to 620 tokens"), guide.index("## Run")) + for phrase in ("200 of the 320-token", "62.5%", "project-1", "project-2", + "--show-labels --acknowledge-sensitive-labels"): + self.assertIn(phrase, guide) + notes = (ROOT / "docs/releases/v0.14.0.md").read_text() + self.assertIn("## Breaking Routing Label Disclosure", notes) + self.assertIn("--show-labels --acknowledge-sensitive-labels", notes) + self.assertIn("Not tagged or published", notes) + def test_langfuse_guide_release_pins(self): readme = (ROOT / "README.md").read_text() versions = re.findall(r"^### 1\. Install (v\d+\.\d+\.\d+)$", readme, re.M) @@ -112,7 +129,8 @@ def test_document_links(self): "SECURITY.md", "CHANGELOG.md", "docs/NATIVE_EXPORT.md", "docs/ROUTING.md", "docs/INSPECT.md", "docs/COMPARISON.md", "docs/releases/v0.11.0.md", "docs/releases/v0.12.0.md", "docs/releases/v0.13.1.md", - "docs/LANGFUSE.md", "examples/litellm-spend/TESTING.md")) + "docs/LANGFUSE.md", "docs/AGENT_ATTRIBUTION.md", "docs/releases/v0.14.0.md", + "examples/litellm-spend/TESTING.md")) for path in paths: for link in re.findall(r"\]\(([^)\s]+)\)", path.read_text()): parsed = urlsplit(link) diff --git a/internal/cli/agents.go b/internal/cli/agents.go index 79a30a8..65d9c7e 100644 --- a/internal/cli/agents.go +++ b/internal/cli/agents.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "math/big" - "strconv" "text/tabwriter" "github.com/llm-measurement/fleetdiff/internal/accounting" @@ -33,8 +32,12 @@ func agentShare(v *string) string { return new(big.Rat).Mul(n, big.NewRat(100, 1)).FloatString(2) + "%" } -func agentCostCoverage(c spend.AgentCostCoverage) string { - return fmt.Sprintf("%d/%d/%d/%d", c.Usable, c.Missing, c.Invalid, c.ZeroUnknown) +func agentLabel(c spend.AgentContributor) string { + label := routingLabel(spend.RoutingIdentity{Item: c.Item, Label: c.Label, Hash: c.Hash}) + if c.Label != "" && c.ProjectAlias != "" { + label += " (" + c.ProjectAlias + ")" + } + return label } func renderAgents(out io.Writer, a *spend.AgentReport) { @@ -47,8 +50,12 @@ func renderAgents(out io.Writer, a *spend.AgentReport) { w spend.AgentWindow }{{"Before", a.Before}, {"After", a.After}} { c := side.w.Coverage - fmt.Fprintf(out, " %s: %d known, %d unknown, %d ambiguous rows; %d column-absent, %d empty/null, %d without a matching tag.\n", side.name, c.KnownRows, c.UnknownRows, c.AmbiguousRows, c.ColumnAbsentRows, c.EmptyOrNullRows, c.NoMatchingTagRows) - fmt.Fprintf(out, " Repeated agent tags: %d in %d rows; empty suffixes: %d in %d rows.\n", c.RepeatedAgentTags, c.RepeatedAgentTagRows, c.EmptySuffixTags, c.EmptySuffixRows) + if a.Source == "agent_id" { + fmt.Fprintf(out, " %s: %d known, %d unknown rows; %d column-absent, %d empty/null.\n", side.name, c.KnownRows, c.UnknownRows, c.ColumnAbsentRows, c.EmptyOrNullRows) + } else { + fmt.Fprintf(out, " %s: %d known, %d unknown, %d ambiguous rows; %d column-absent, %d empty/null, %d without a matching tag.\n", side.name, c.KnownRows, c.UnknownRows, c.AmbiguousRows, c.ColumnAbsentRows, c.EmptyOrNullRows, c.NoMatchingTagRows) + fmt.Fprintf(out, " Repeated agent tags: %d in %d rows; empty suffixes: %d in %d rows.\n", c.RepeatedAgentTags, c.RepeatedAgentTagRows, c.EmptySuffixTags, c.EmptySuffixRows) + } if c.NonStringTagRows > 0 { fmt.Fprintf(out, " Non-string tag elements: %d rows counted as unknown identity.\n", c.NonStringTagRows) } @@ -58,21 +65,39 @@ func renderAgents(out io.Writer, a *spend.AgentReport) { } return w.RecordedSpend } - fmt.Fprintf(out, " Unknown: %d tokens / %s recorded USD; ambiguous: %d tokens / %s recorded USD.\n", side.w.Unknown.Tokens, usd(side.w.Unknown), side.w.Ambiguous.Tokens, usd(side.w.Ambiguous)) + fmt.Fprintf(out, " Unknown: %d tokens / %s recorded USD", side.w.Unknown.Tokens, usd(side.w.Unknown)) + if a.Source != "agent_id" { + fmt.Fprintf(out, "; ambiguous: %d tokens / %s recorded USD", side.w.Ambiguous.Tokens, usd(side.w.Ambiguous)) + } + fmt.Fprintln(out, ".") } fmt.Fprintln(out, " Shares use all usable recorded USD in each period, including unknown, ambiguous and undisplayed agents. Missing/invalid costs remain unquantified; recorded USD is not an invoice.") - fmt.Fprintln(out, " Cost coverage columns: usable/missing/invalid/zero-origin-unknown rows.") + showCosts := false + for _, contributors := range [][]spend.AgentContributor{a.Contributors, a.UnavailableCosts} { + for _, c := range contributors { + for _, v := range []spend.AgentValues{c.Before, c.After} { + showCosts = showCosts || v.CostCoverage.Missing+v.CostCoverage.Invalid+v.CostCoverage.ZeroUnknown != 0 + } + } + } + if showCosts { + fmt.Fprintln(out, " Cost coverage columns show before -> after row counts.") + } else { + fmt.Fprintln(out, " All displayed agents have usable recorded USD in both periods.") + } w := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0) - fmt.Fprintln(w, " Agent\tTokens before\tTokens after\tUSD before\tUSD after\tUSD change\tCost coverage before\tCost coverage after\tShare before\tShare after") + fmt.Fprint(w, " Agent\tTokens before\tTokens after\tUSD before\tUSD after\tUSD change\tShare before\tShare after") + if showCosts { + fmt.Fprint(w, "\tUsable rows\tMissing rows\tInvalid rows\tZero-origin-unknown rows") + } + fmt.Fprintln(w) writeContributor := func(c spend.AgentContributor) { - label := c.Item - if c.Label != "" { - label += " " + strconv.QuoteToASCII(c.Label) + fmt.Fprintf(w, " %s\t%d\t%d\t%s\t%s\t%s\t%s\t%s", agentLabel(c), c.Before.Tokens, c.After.Tokens, agentUSD(c.Before.RecordedUSD), agentUSD(c.After.RecordedUSD), agentUSD(c.Delta), agentShare(c.Before.Share), agentShare(c.After.Share)) + if showCosts { + b, a := c.Before.CostCoverage, c.After.CostCoverage + fmt.Fprintf(w, "\t%d -> %d\t%d -> %d\t%d -> %d\t%d -> %d", b.Usable, a.Usable, b.Missing, a.Missing, b.Invalid, a.Invalid, b.ZeroUnknown, a.ZeroUnknown) } - if c.Hash != "" { - label += " (" + c.Hash + ")" - } - fmt.Fprintf(w, " %s\t%d\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", label, c.Before.Tokens, c.After.Tokens, agentUSD(c.Before.RecordedUSD), agentUSD(c.After.RecordedUSD), agentUSD(c.Delta), agentCostCoverage(c.Before.CostCoverage), agentCostCoverage(c.After.CostCoverage), agentShare(c.Before.Share), agentShare(c.After.Share)) + fmt.Fprintln(w) } for _, c := range a.Contributors { writeContributor(c) @@ -89,10 +114,14 @@ func renderAgents(out io.Writer, a *spend.AgentReport) { fmt.Fprintf(out, " %s: token change %s; recorded USD change unavailable.\n", c.Item, signedInterval(compare.Interval{Lower: delta, Upper: delta})) } } - for _, bucket := range []struct { + buckets := []struct { name string before, after spend.Window - }{{"Unknown", a.Before.Unknown, a.After.Unknown}, {"Ambiguous", a.Before.Ambiguous, a.After.Ambiguous}} { + }{{"Unknown", a.Before.Unknown, a.After.Unknown}, {"Ambiguous", a.Before.Ambiguous, a.After.Ambiguous}} + if a.Source == "agent_id" { + buckets = buckets[:1] + } + for _, bucket := range buckets { cost := func(v spend.Window) *big.Rat { if v.Requests > 0 && v.Quality.SpendMissing+v.Quality.SpendInvalid == v.Requests { return nil diff --git a/internal/cli/agents_test.go b/internal/cli/agents_test.go index af87393..dba6476 100644 --- a/internal/cli/agents_test.go +++ b/internal/cli/agents_test.go @@ -92,8 +92,12 @@ func TestAgentCLIFormatsAndPrivacy(t *testing.T) { t.Fatal("required columns or warning missing") } if format == "text" { + answer := "agent-1" + if disclose { + answer += ` "alpha"` + } for _, want := range []string{ - "agent-1 accounts for 62.5% of the net recorded increase (+200 of +320 tokens)", + answer + " accounts for 62.5% of the net recorded increase (+200 of +320 tokens)", "Unknown (not ranked): 0 -> 1 rows; token change +50; recorded USD change 0.5", "Ambiguous (not ranked): 0 -> 1 rows; token change +70; recorded USD change 0.7", } { @@ -221,3 +225,100 @@ func TestAgentTextEscapesSelectedLabel(t *testing.T) { t.Fatal("label changed terminal structure") } } + +func TestAgentIDTextPolishAndEscapedAnswer(t *testing.T) { + t.Setenv("AGENT_CLI_SECRET", "synthetic-agent-secret-for-tests-at-least-32-bytes") + label := "alpha\n\x1b[31m\u202e" + var input bytes.Buffer + w := csv.NewWriter(&input) + _ = w.Write([]string{"request_id", "call_type", "startTime", "endTime", "status", "model", "prompt_tokens", "completion_tokens", "total_tokens", "spend", "agent_id", "request_tags"}) + for i, row := range []struct{ day, agent, tokens, cost string }{ + {"07", label, "100", "1"}, {"07", "beta", "100", "1"}, + {"08", label, "300", "3"}, {"08", "beta", "100", "100"}, {"08", "", "50", "0.5"}, + } { + start := "2026-10-" + row.day + "T12:00:00Z" + _ = w.Write([]string{string(rune('a' + i)), "completion", start, start, "success", "private-model", row.tokens, "0", row.tokens, row.cost, row.agent, `["agent:x","agent:y"]`}) + } + w.Flush() + path := filepath.Join(t.TempDir(), "agents.csv") + if err := os.WriteFile(path, input.Bytes(), 0600); err != nil { + t.Fatal(err) + } + args := []string{"investigate", "--litellm-spend", path, "--before-period", "2026-10-07", "--after-period", "2026-10-08", "--group-by", "agent", "--agent-source", "agent_id", "--rank-by", "recorded-spend-change", "--top", "1", "--hash-secret-env", "AGENT_CLI_SECRET", "--show-labels", "--acknowledge-sensitive-labels"} + code, out, errs := databaseRun(args, nil) + if code != 0 || errs != "" { + t.Fatal(code, errs) + } + for _, want := range []string{`agent-1 "alpha\n\x1b[31m\u202e" accounts for`, "Agent identity unknown or ambiguous: 0 -> 1.", "All displayed agents have usable recorded USD in both periods."} { + if !strings.Contains(out, want) { + t.Fatal("agent polish missing", want, out) + } + } + for _, forbidden := range []string{"\x1b", "\u202e", "without a matching tag", "Repeated agent tags", "empty suffixes", "Ambiguous (not ranked)", "; ambiguous:", "Group identities missing", "Missing rows", "Zero-origin-unknown rows"} { + if strings.Contains(out, forbidden) { + t.Fatal("agent_id text includes inapplicable coverage or unsafe label", forbidden) + } + } + code, data, errs := databaseRun(append(args, "--format", "json"), nil) + var r spend.Report + if code != 0 || errs != "" || json.Unmarshal([]byte(data), &r) != nil { + t.Fatal("JSON failed", code, errs) + } + for _, field := range []string{`"ambiguous_rows": 0`, `"no_matching_tag_rows": 0`, `"empty_suffix_rows": 0`} { + if !strings.Contains(data, field) { + t.Fatal("JSON coverage was removed", field) + } + } + if r.Agents.Contributors[0].Item != "agent-2" { + t.Fatal("test must keep the token leader outside the displayed cost ranking") + } +} + +func TestAgentCostColumnsOnlyForDisplayedIncompleteCosts(t *testing.T) { + usd := "1" + value := spend.AgentValues{RecordedUSD: &usd, CostCoverage: spend.AgentCostCoverage{Usable: 1}} + for _, kind := range []string{"complete", "missing", "invalid", "zero-origin", "unavailable"} { + t.Run(kind, func(t *testing.T) { + c := spend.AgentContributor{Item: "agent-1", Before: value, After: value, Delta: &usd} + r := &spend.AgentReport{Source: "agent_id", Contributors: []spend.AgentContributor{c}} + r.UndisplayedAfter.CostCoverage.Missing = 99 + r.After.Unknown.Quality.SpendMissing = 10 + switch kind { + case "missing": + r.Contributors[0].Before.CostCoverage.Missing = 1 + case "invalid": + r.Contributors[0].After.CostCoverage.Invalid = 1 + case "zero-origin": + r.Contributors[0].After.CostCoverage.ZeroUnknown = 1 + case "unavailable": + c.Item, c.Delta, c.Before.RecordedUSD = "agent-2", nil, nil + c.Before.CostCoverage.Missing = 1 + r.UnavailableCosts = []spend.AgentContributor{c} + } + var out bytes.Buffer + renderAgents(&out, r) + for _, column := range []string{"Usable rows", "Missing rows", "Invalid rows", "Zero-origin-unknown rows"} { + if strings.Contains(out.String(), column) != (kind != "complete") { + t.Fatal("coverage columns did not follow displayed rows", column, out.String()) + } + } + if strings.Contains(out.String(), "All displayed agents have usable recorded USD in both periods.") != (kind == "complete") { + t.Fatal("complete-cost line mismatch") + } + }) + } +} + +func TestAgentTextProjectAliases(t *testing.T) { + var out bytes.Buffer + renderAgents(&out, &spend.AgentReport{Source: "request-tag", LabelsIncluded: true, + Contributors: []spend.AgentContributor{ + {Item: "agent-1", Label: "alpha", ProjectAlias: "project-1"}, + {Item: "agent-2", Label: "alpha", ProjectAlias: "project-2"}, + }}) + for _, want := range []string{`agent-1 "alpha" (project-1)`, `agent-2 "alpha" (project-2)`} { + if !strings.Contains(out.String(), want) { + t.Fatal("identical project-scoped labels are ambiguous", out.String()) + } + } +} diff --git a/internal/cli/spend.go b/internal/cli/spend.go index 48af17c..5eb8899 100644 --- a/internal/cli/spend.go +++ b/internal/cli/spend.go @@ -118,6 +118,9 @@ func renderSpend(out io.Writer, r spend.Report) { label, verb := fmt.Sprintf("%d leading tracked agents", c.Count), "account" if c.Count == 1 && len(r.Rankings.Movers) > 0 { label, verb = r.Rankings.Movers[0].Item, "accounts" + if leading := r.Agents.LeadingContributor; leading != nil && leading.Item == label { + label = agentLabel(*leading) + } } share := spendPercent(100 * c.Share.Lower) if c.Delta.Lower != c.Delta.Upper { @@ -220,6 +223,10 @@ func renderSpend(out io.Writer, r spend.Report) { } } a, b := r.Before.Quality, r.After.Quality + identityLabel := "Group identities missing" + if r.Agents != nil { + identityLabel = "Agent identity unknown or ambiguous" + } var levels []string for _, level := range []string{"DEFAULT", "DEBUG", "WARNING", "ERROR", "unknown"} { if a.ObservationLevels[level] != 0 || b.ObservationLevels[level] != 0 { @@ -251,7 +258,7 @@ func renderSpend(out io.Writer, r spend.Report) { {"Total field absent", a.TotalMissing, b.TotalMissing}, {"Failed records (overlap usage categories)", a.Failed, b.Failed}, {"Unknown status", a.UnknownStatus, b.UnknownStatus}, - {"Group identities missing", a.MissingIdentity, b.MissingIdentity}, + {identityLabel, a.MissingIdentity, b.MissingIdentity}, } { if v.a != 0 || v.b != 0 { problems = true diff --git a/internal/langfuse/agents_test.go b/internal/langfuse/agents_test.go index 5be37a7..10effd7 100644 --- a/internal/langfuse/agents_test.go +++ b/internal/langfuse/agents_test.go @@ -4,6 +4,7 @@ package langfuse import ( + "encoding/json" "strings" "testing" ) @@ -32,6 +33,18 @@ func TestAgentTagsProjectScopeAndCoverage(t *testing.T) { if err != nil || shown.Agents.Contributors[0].Label != "alpha" || shown.Agents.Contributors[1].Label != "alpha" || shown.Before.Tokens != r.Before.Tokens { t.Fatal("Langfuse disclosure or scope changed", err) } + if shown.Agents.Contributors[0].ProjectAlias != "project-1" || shown.Agents.Contributors[1].ProjectAlias != "project-2" { + t.Fatal("project labels lack stable aliases", shown.Agents.Contributors) + } + data, _ := json.Marshal(shown.Agents) + if strings.Contains(string(data), "project-") || strings.Contains(string(data), "ProjectAlias") || strings.Contains(string(data), "LeadingContributor") { + t.Fatal("text-only disambiguation changed JSON") + } + for _, c := range r.Agents.Contributors { + if c.ProjectAlias != "" || c.Label != "" { + t.Fatal("undisclosed project metadata escaped") + } + } o.AgentSource, o.AgentTagPrefix = "agent_id", "" if _, err := Investigate(root, "agent", "", o); err == nil { t.Fatal("Langfuse agent_id inference accepted") diff --git a/internal/spend/agent_spend_test.go b/internal/spend/agent_spend_test.go index 5ee70a0..fac4a3b 100644 --- a/internal/spend/agent_spend_test.go +++ b/internal/spend/agent_spend_test.go @@ -72,6 +72,22 @@ func TestAgentSpendMissingAndDisclosure(t *testing.T) { } } +func TestAgentTextMetadataDoesNotChangeJSON(t *testing.T) { + r := routingReport(t, agentFixtureRows(), agentOptions(t)) + before, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + r.Agents.LeadingContributor = &AgentContributor{Item: "agent-1", Label: "text-only", ProjectAlias: "project-1"} + for i := range r.Agents.Contributors { + r.Agents.Contributors[i].ProjectAlias = "project-1" + } + after, err := json.Marshal(r) + if err != nil || string(before) != string(after) { + t.Fatal("text-only metadata changed JSON", err) + } +} + func TestAgentCapNamesPrefix(t *testing.T) { var rows []map[string]string for i := 0; i <= MaxAgents; i++ { diff --git a/internal/spend/agents.go b/internal/spend/agents.go index c3f76ab..d281f81 100644 --- a/internal/spend/agents.go +++ b/internal/spend/agents.go @@ -41,17 +41,18 @@ type AgentWindow struct { } type AgentReport struct { - Schema string `json:"schema"` - Source string `json:"label_source"` - Before AgentWindow `json:"before"` - After AgentWindow `json:"after"` - RankBy string `json:"rank_by"` - LabelsIncluded bool `json:"sensitive_labels_included"` - Contributors []AgentContributor `json:"contributors"` - UnavailableCosts []AgentContributor `json:"unavailable_cost_contributors,omitempty"` - UndisplayedBefore AgentValues `json:"known_undisplayed_before"` - UndisplayedAfter AgentValues `json:"known_undisplayed_after"` - ShareBasis string `json:"spend_share_basis"` + Schema string `json:"schema"` + Source string `json:"label_source"` + Before AgentWindow `json:"before"` + After AgentWindow `json:"after"` + RankBy string `json:"rank_by"` + LabelsIncluded bool `json:"sensitive_labels_included"` + Contributors []AgentContributor `json:"contributors"` + UnavailableCosts []AgentContributor `json:"unavailable_cost_contributors,omitempty"` + UndisplayedBefore AgentValues `json:"known_undisplayed_before"` + UndisplayedAfter AgentValues `json:"known_undisplayed_after"` + ShareBasis string `json:"spend_share_basis"` + LeadingContributor *AgentContributor `json:"-"` } type AgentCostCoverage struct { @@ -70,12 +71,13 @@ type AgentValues struct { } type AgentContributor struct { - Item string `json:"item"` - Hash string `json:"hash,omitempty"` - Label string `json:"label,omitempty"` - Before AgentValues `json:"before"` - After AgentValues `json:"after"` - Delta *string `json:"recorded_usd_delta"` + Item string `json:"item"` + Hash string `json:"hash,omitempty"` + Label string `json:"label,omitempty"` + Before AgentValues `json:"before"` + After AgentValues `json:"after"` + Delta *string `json:"recorded_usd_delta"` + ProjectAlias string `json:"-"` } type agentResolution struct { @@ -202,10 +204,11 @@ func newAgentPair(before, after Period) agentPair { } type agentEntry struct { - digest [32]byte - label string - pair agentPair - order int + digest [32]byte + label string + project string + pair agentPair + order int } type agentAccumulator struct { @@ -261,6 +264,7 @@ func (a *agentAccumulator) add(row Row, side int, digest [32]byte, obs accountin e = &agentEntry{digest: digest, pair: newAgentPair(a.before, a.after), order: len(a.entries)} if a.opts.ShowLabels && a.opts.AcknowledgeSensitiveLabels { e.label = r.label + e.project = row.AgentNamespace } a.entries[digest] = e } @@ -329,6 +333,12 @@ func (a *agentAccumulator) finish(r *Report) error { return nil } entries := a.ordered() + projects := map[string]string{} + for _, e := range entries { + if e.project != "" && projects[e.project] == "" { + projects[e.project] = fmt.Sprintf("project-%d", len(projects)+1) + } + } slices.SortStableFunc(entries, func(x, y *agentEntry) int { if a.opts.RankBy == "recorded-spend-change" { available := func(e *agentEntry) bool { @@ -377,7 +387,7 @@ func (a *agentAccumulator) finish(r *Report) error { remaining := [2]*routingTotals{newRoutingTotals(), newRoutingTotals()} report.Contributors = []AgentContributor{} for i, e := range entries { - c := AgentContributor{Item: fmt.Sprintf("agent-%d", e.order+1), Label: e.label, Before: agentValues(e.pair[0].result(), totals[0]), After: agentValues(e.pair[1].result(), totals[1])} + c := AgentContributor{Item: fmt.Sprintf("agent-%d", e.order+1), Label: e.label, ProjectAlias: projects[e.project], Before: agentValues(e.pair[0].result(), totals[0]), After: agentValues(e.pair[1].result(), totals[1])} if a.opts.ShowHashes { c.Hash = fmt.Sprintf("%x", e.digest) } @@ -385,6 +395,9 @@ func (a *agentAccumulator) finish(r *Report) error { delta := costText(new(big.Rat).Sub(&e.pair[1].cost, &e.pair[0].cost)) c.Delta = &delta } + if len(r.Rankings.Movers) > 0 && c.Item == r.Rankings.Movers[0].Item { + report.LeadingContributor = &c + } if a.opts.RankBy == "recorded-spend-change" && c.Delta == nil { report.UnavailableCosts = append(report.UnavailableCosts, c) continue diff --git a/scripts/install.sh b/scripts/install.sh index 9797328..c717ff8 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -6,11 +6,11 @@ umask 077 usage() { printf '%s\n' 'Usage: sh scripts/install.sh [VERSION [NEW_DIRECTORY]]' \ - 'Defaults: v0.13.1 ./fleetdiff-install. Requires curl, gh, shasum, and tar.' + 'Defaults: v0.14.0 ./fleetdiff-install. Requires curl, gh, shasum, and tar.' } case "${1:-}" in --help|-h) usage; exit 0 ;; esac if [ "$#" -gt 2 ]; then usage >&2; exit 2; fi -version=${1-v0.13.1} +version=${1-v0.14.0} destination=${2-./fleetdiff-install} case "$version" in ''|*[!a-zA-Z0-9.-]*) printf '%s\n' 'Supply a release version such as v0.3.0.' >&2; exit 2 ;; diff --git a/scripts/install_test.go b/scripts/install_test.go index e70397a..1869d4f 100644 --- a/scripts/install_test.go +++ b/scripts/install_test.go @@ -15,7 +15,7 @@ import ( "testing" ) -const installTestVersion = "v0.13.1" +const installTestVersion = "v0.14.0" const installTestURL = "https://raw.githubusercontent.com/llm-measurement/fleetdiff/0d7db95c5bb3013f8c564c91fbb07b6235a2664e/scripts/install.sh" func TestInstallDocumentationVersionPins(t *testing.T) { From 7d3719010f9516ddbef55beadfab5b02b62d1be7 Mon Sep 17 00:00:00 2001 From: Vijay Date: Thu, 29 Oct 2026 19:14:21 -0400 Subject: [PATCH 7/7] fix: attribute agent increases to the positive mover Signed-off-by: Vijay --- internal/cli/agents_test.go | 40 ++++++++++++++++++++++++++++++ internal/cli/spend.go | 34 ++++++++++++++++++------- internal/spend/agent_spend_test.go | 37 +++++++++++++++++++++++++++ internal/spend/agents.go | 11 +++++++- 4 files changed, 112 insertions(+), 10 deletions(-) diff --git a/internal/cli/agents_test.go b/internal/cli/agents_test.go index dba6476..b992ee2 100644 --- a/internal/cli/agents_test.go +++ b/internal/cli/agents_test.go @@ -274,6 +274,46 @@ func TestAgentIDTextPolishAndEscapedAnswer(t *testing.T) { } } +func TestAgentIncreaseNamesPositiveMoverWithOffsets(t *testing.T) { + t.Setenv("AGENT_CLI_SECRET", "synthetic-agent-secret-for-tests-at-least-32-bytes") + var input bytes.Buffer + w := csv.NewWriter(&input) + _ = w.Write([]string{"request_id", "call_type", "startTime", "endTime", "status", "model", "prompt_tokens", "completion_tokens", "total_tokens", "spend", "agent_id"}) + for i, row := range []struct{ day, agent, tokens string }{ + {"07", "alpha", "1000"}, {"07", "beta", "0"}, + {"08", "alpha", "300"}, {"08", "beta", "600"}, {"08", "", "200"}, + } { + start := "2026-10-" + row.day + "T12:00:00Z" + _ = w.Write([]string{string(rune('a' + i)), "completion", start, start, "success", "private-model", row.tokens, "0", row.tokens, "0", row.agent}) + } + w.Flush() + path := filepath.Join(t.TempDir(), "agents.csv") + if err := os.WriteFile(path, input.Bytes(), 0600); err != nil { + t.Fatal(err) + } + for _, labels := range []bool{false, true} { + args := []string{"investigate", "--litellm-spend", path, "--before-period", "2026-10-07", "--after-period", "2026-10-08", "--group-by", "agent", "--agent-source", "agent_id", "--hash-secret-env", "AGENT_CLI_SECRET"} + name := "agent-2" + if labels { + args = append(args, "--show-labels", "--acknowledge-sensitive-labels") + name += ` "beta"` + } + code, out, errs := databaseRun(args, nil) + want := name + " added 600 tokens while others fell by 500 (net +100)." + if code != 0 || errs != "" || !strings.Contains(out, want) || strings.Contains(out, "600%") { + t.Fatal("wrong increase attribution", code, errs, out) + } + code, data, errs := databaseRun(append(args, "--format", "json"), nil) + var r spend.Report + if code != 0 || errs != "" || json.Unmarshal([]byte(data), &r) != nil { + t.Fatal("JSON failed", code, errs) + } + if r.Before.Tokens != 1000 || r.After.Tokens != 1100 || r.Increase.Count != 1 || r.Increase.Delta.Lower != 600 || r.Increase.Delta.Upper != 600 || r.Increase.Share.Lower != 6 || r.Rankings.Movers[0].Item != "agent-1" || r.Rankings.Movers[0].Delta.Lower != -700 { + t.Fatal("text correction changed JSON accounting or rankings", data) + } + } +} + func TestAgentCostColumnsOnlyForDisplayedIncompleteCosts(t *testing.T) { usd := "1" value := spend.AgentValues{RecordedUSD: &usd, CostCoverage: spend.AgentCostCoverage{Usable: 1}} diff --git a/internal/cli/spend.go b/internal/cli/spend.go index 5eb8899..c55b0ea 100644 --- a/internal/cli/spend.go +++ b/internal/cli/spend.go @@ -116,21 +116,37 @@ func renderSpend(out io.Writer, r spend.Report) { if r.Agents != nil { if c := r.Increase; c != nil && !r.Incomplete { label, verb := fmt.Sprintf("%d leading tracked agents", c.Count), "account" - if c.Count == 1 && len(r.Rankings.Movers) > 0 { - label, verb = r.Rankings.Movers[0].Item, "accounts" + if c.Count == 1 { + for _, m := range r.Rankings.Movers { + if m.Delta.Lower > 0 { + label, verb = m.Item, "accounts" + break + } + } if leading := r.Agents.LeadingContributor; leading != nil && leading.Item == label { label = agentLabel(*leading) } } - share := spendPercent(100 * c.Share.Lower) - if c.Delta.Lower != c.Delta.Upper { - lower, upper := c.Share.Lower*10000, c.Share.Upper*10000 - if c.Share.Lower == c.Share.Upper { - lower, upper = math.Nextafter(lower, math.Inf(-1)), math.Nextafter(upper, math.Inf(1)) + net := int64(r.After.Tokens - r.Before.Tokens) + if c.Delta.Upper > net { + if c.Delta.Lower >= net { + fell := compare.Interval{Lower: c.Delta.Lower - net, Upper: c.Delta.Upper - net} + fmt.Fprintf(out, " %s added %s tokens while others fell by %s (net +%s).\n", label, spendInterval(c.Delta), spendInterval(fell), spendNumber(net)) + } else { + changed := compare.Interval{Lower: net - c.Delta.Upper, Upper: net - c.Delta.Lower} + fmt.Fprintf(out, " %s added %s tokens while others changed by %s (net +%s).\n", label, spendInterval(c.Delta), signedInterval(changed), spendNumber(net)) } - share = fmt.Sprintf("[%s, %s]", spendPercent(math.Floor(lower)/100), spendPercent(math.Ceil(upper)/100)) + } else { + share := spendPercent(100 * c.Share.Lower) + if c.Delta.Lower != c.Delta.Upper { + lower, upper := c.Share.Lower*10000, c.Share.Upper*10000 + if c.Share.Lower == c.Share.Upper { + lower, upper = math.Nextafter(lower, math.Inf(-1)), math.Nextafter(upper, math.Inf(1)) + } + share = fmt.Sprintf("[%s, %s]", spendPercent(math.Floor(lower)/100), spendPercent(math.Ceil(upper)/100)) + } + fmt.Fprintf(out, " %s %s for %s of the net recorded increase (%s of +%s tokens).\n", label, verb, share, signedInterval(c.Delta), spendNumber(r.After.Tokens-r.Before.Tokens)) } - fmt.Fprintf(out, " %s %s for %s of the net recorded increase (%s of +%s tokens).\n", label, verb, share, signedInterval(c.Delta), spendNumber(r.After.Tokens-r.Before.Tokens)) } renderAgents(out, r.Agents) } diff --git a/internal/spend/agent_spend_test.go b/internal/spend/agent_spend_test.go index fac4a3b..9b852ba 100644 --- a/internal/spend/agent_spend_test.go +++ b/internal/spend/agent_spend_test.go @@ -88,6 +88,43 @@ func TestAgentTextMetadataDoesNotChangeJSON(t *testing.T) { } } +func TestAgentLeadingContributorIsPositiveMover(t *testing.T) { + var rows []map[string]string + for i, row := range []struct { + day, agent string + tokens int + }{ + {"2026-09-01", "alpha", 1000}, {"2026-09-01", "beta", 0}, + {"2026-09-02", "alpha", 300}, {"2026-09-02", "beta", 600}, {"2026-09-02", "", 200}, + } { + r := routingSample(fmt.Sprint(i), row.day, row.tokens, 0) + r["request_tags"] = "[]" + if row.agent != "" { + r["request_tags"] = fmt.Sprintf(`["agent:%s"]`, row.agent) + } + rows = append(rows, r) + } + o := agentOptions(t) + o.ShowLabels, o.AcknowledgeSensitiveLabels = true, true + r := routingReport(t, rows, o) + if r.Increase == nil || r.Increase.Count != 1 || r.Rankings.Movers[0].Item != "agent-1" { + t.Fatal("fixture must have one positive mover behind a larger decline") + } + if c := r.Agents.LeadingContributor; c == nil || c.Item != "agent-2" || c.Label != "beta" { + t.Fatal("leading contributor is not the positive mover", c) + } + before, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + // Restore the old text-only selection: serialized accounting must be identical. + r.Agents.LeadingContributor = &AgentContributor{Item: "agent-1", Label: "alpha"} + after, err := json.Marshal(r) + if err != nil || string(before) != string(after) { + t.Fatal("positive-mover selection changed JSON", err) + } +} + func TestAgentCapNamesPrefix(t *testing.T) { var rows []map[string]string for i := 0; i <= MaxAgents; i++ { diff --git a/internal/spend/agents.go b/internal/spend/agents.go index d281f81..8861ef1 100644 --- a/internal/spend/agents.go +++ b/internal/spend/agents.go @@ -386,6 +386,15 @@ func (a *agentAccumulator) finish(r *Report) error { } remaining := [2]*routingTotals{newRoutingTotals(), newRoutingTotals()} report.Contributors = []AgentContributor{} + leading := "" + if r.Increase != nil && r.Increase.Count == 1 { + for _, m := range r.Rankings.Movers { + if m.Delta.Lower > 0 { + leading = m.Item + break + } + } + } for i, e := range entries { c := AgentContributor{Item: fmt.Sprintf("agent-%d", e.order+1), Label: e.label, ProjectAlias: projects[e.project], Before: agentValues(e.pair[0].result(), totals[0]), After: agentValues(e.pair[1].result(), totals[1])} if a.opts.ShowHashes { @@ -395,7 +404,7 @@ func (a *agentAccumulator) finish(r *Report) error { delta := costText(new(big.Rat).Sub(&e.pair[1].cost, &e.pair[0].cost)) c.Delta = &delta } - if len(r.Rankings.Movers) > 0 && c.Item == r.Rankings.Movers[0].Item { + if c.Item == leading { report.LeadingContributor = &c } if a.opts.RankBy == "recorded-spend-change" && c.Delta == nil {