diff --git a/internal/cli/langfuse_docs_test.go b/internal/cli/langfuse_docs_test.go index 3ae9fd2..93c5f9d 100644 --- a/internal/cli/langfuse_docs_test.go +++ b/internal/cli/langfuse_docs_test.go @@ -88,6 +88,41 @@ func TestLangfusePublicExample(t *testing.T) { } } +func TestLangfusePublicExampleModelAliases(t *testing.T) { + for _, format := range []string{"text", "json"} { + t.Run(format, func(t *testing.T) { + args := langfuseExampleArgs(format) + for i := range args { + if args[i] == "trace_name" { + args[i] = "provided_model_name" + } + } + var out, diagnostic bytes.Buffer + if code := Run(args, &out, &diagnostic); code != 0 || diagnostic.Len() != 0 { + t.Fatalf("example command failed: code %d %s", code, diagnostic.String()) + } + if strings.Contains(out.String(), "synthetic-model") || strings.Contains(out.String(), "provided_model_name-1") { + t.Fatal("raw model or a second model-alias vocabulary escaped") + } + if format == "json" { + var r langfuse.Report + if err := json.Unmarshal(out.Bytes(), &r); err != nil { + t.Fatal(err) + } + if len(r.Models) != 1 || len(r.Rankings.Movers) != 1 || r.Models[0].Item != "model-1" || r.Models[0].Item != r.Rankings.Movers[0].Item { + t.Fatal("headline and contributor model aliases disagree") + } + } else { + for _, want := range []string{"What changed?\n model-1:", "Who drove the increase?\n model-1:"} { + if !strings.Contains(out.String(), want) { + t.Fatalf("model section missing %q", want) + } + } + } + }) + } +} + func TestReleaseBinaryLangfuseExample(t *testing.T) { binary := os.Getenv("FLEETDIFF_RELEASE_BINARY") if binary == "" { diff --git a/internal/langfuse/directory.go b/internal/langfuse/directory.go index ad1e152..5fe19b0 100644 --- a/internal/langfuse/directory.go +++ b/internal/langfuse/directory.go @@ -543,6 +543,7 @@ func Investigate(dir, group, tag string, opts spend.Options) (Report, error) { if opts.GroupBy == "" { opts.GroupBy = "key" } + opts.ModelAliases = group == "provided_model_name" r, err := spend.InvestigateRows(func(visit func(spend.Row) error) error { for _, row := range snap.rows { if row.Values["call_type"] != "completion" { @@ -580,10 +581,13 @@ func Investigate(dir, group, tag string, opts spend.Options) (Report, error) { "Raw user, session and custom labels are suppressed with local keyed aliases. Status is unknown; status_message is never decoded.", "Level is an observation severity enum, not provider success or billing finality. DEFAULT/DEBUG/WARNING alone do not require usage review; ERROR and missing/invalid levels do. Other usage checks still apply. Per-request splits remain unavailable while call status is unknown.", } + if opts.ModelAliases { + r.Notes = append(r.Notes, "Model labels in contributor rankings match the model totals. Bracketed group aliases distinguish project-scoped contributors when the same model occurs in multiple projects; those contributors are not merged.") + } for _, c := range []*compare.Concentration{&r.Rankings, r.Cache.Rankings} { if c != nil { for i := range c.Movers { - _, n, ok := strings.Cut(c.Movers[i].Item, "-") + n, ok := strings.CutPrefix(c.Movers[i].Item, opts.GroupBy+"-") if ok { c.Movers[i].Item = group + "-" + n } diff --git a/internal/langfuse/directory_test.go b/internal/langfuse/directory_test.go index 92c734e..e4401cb 100644 --- a/internal/langfuse/directory_test.go +++ b/internal/langfuse/directory_test.go @@ -87,7 +87,11 @@ func TestDirectoryCompositionAndGrouping(t *testing.T) { if bytes.Contains(b, []byte("PRIVATE_")) { t.Fatal("raw label leaked") } - if r.GroupBy != group || len(r.Rankings.Movers) != 1 || !strings.HasPrefix(r.Rankings.Movers[0].Item, group+"-") { + prefix := group + "-" + if group == "provided_model_name" { + prefix = "model-" + } + if r.GroupBy != group || len(r.Rankings.Movers) != 1 || !strings.HasPrefix(r.Rankings.Movers[0].Item, prefix) { t.Fatal("group not composed") } }) diff --git a/internal/langfuse/model_aliases_test.go b/internal/langfuse/model_aliases_test.go new file mode 100644 index 0000000..3cee7df --- /dev/null +++ b/internal/langfuse/model_aliases_test.go @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package langfuse + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/llm-measurement/fleetdiff/internal/attribution" + "github.com/llm-measurement/fleetdiff/internal/reprice" +) + +func TestModelAliasesAcrossRankings(t *testing.T) { + root := t.TempDir() + var body strings.Builder + for _, row := range []struct { + id, day, model string + input, cache, output int + }{ + {"a0", "2026-10-10", "PRIVATE_A", 100, 900, 100}, + {"b0", "2026-10-10", "PRIVATE_B", 100, 900, 100}, + {"c0", "2026-10-10", "PRIVATE_C", 100, 900, 100}, + {"a1", "2026-10-17", "PRIVATE_A", 100, 900, 9000}, + {"b1", "2026-10-17", "PRIVATE_B", 900, 100, 100}, + {"c1", "2026-10-17", "PRIVATE_C", 500, 500, 2000}, + } { + var value map[string]any + if err := json.Unmarshal([]byte(rowJSON(row.id, "project", row.day)), &value); err != nil { + t.Fatal(err) + } + value["provided_model_name"] = row.model + value["usage_details"] = map[string]int{"input": row.input, "input_cached_tokens": row.cache, "output": row.output, "total": row.input + row.cache + row.output} + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + body.Write(encoded) + body.WriteByte('\n') + } + addRun(t, root, "project", "models", "2026-10-10T00:00:00Z", "2026-10-18T00:00:00Z", body.String(), 6) + for _, hashes := range []bool{false, true} { + for _, top := range []int{1, 20} { + for _, fixed := range []bool{false, true} { + t.Run(fmt.Sprintf("hashes-%t/top-%d/fixed-%t", hashes, top, fixed), func(t *testing.T) { + opts := options() + opts.ShowHashes, opts.Top = hashes, top + if fixed { + t.Setenv("FLEETDIFF_TEST_MODEL_ALIASES", "local-model-alias-regression-key") + opts.SecretEnv = "FLEETDIFF_TEST_MODEL_ALIASES" + } + r, err := Investigate(root, "provided_model_name", "", opts) + if err != nil { + t.Fatal(err) + } + if r.Before.Tokens != 3300 || r.After.Tokens != 14100 || len(r.Models) != 3 || r.Cache.Rankings == nil { + t.Fatal("alias selection changed totals or cache coverage") + } + for i, model := range r.Models { + if model.Item != fmt.Sprintf("model-%d", i+1) { + t.Fatal("headline aliases changed") + } + } + if r.Rankings.Movers[0].Item != "model-1" || r.Cache.Rankings.Movers[0].Item != "model-3" { + t.Fatalf("aliases follow ranking positions instead of model identities: %s / %s", r.Rankings.Movers[0].Item, r.Cache.Rankings.Movers[0].Item) + } + encoded, err := json.Marshal(r) + if err != nil || strings.Contains(string(encoded), "PRIVATE_") { + t.Fatal("private label escaped", err) + } + }) + } + } + } +} + +func TestModelAliasesPreserveProjectScope(t *testing.T) { + root := t.TempDir() + for _, project := range []string{"project-a", "project-b"} { + addRun(t, root, project, "run", "2026-10-10T00:00:00Z", "2026-10-18T00:00:00Z", rowJSON("one", project, "2026-10-10")+rowJSON("two", project, "2026-10-17"), 2) + } + const secret = "local-model-alias-regression-key" + t.Setenv("FLEETDIFF_TEST_MODEL_ALIASES", secret) + opts := options() + opts.ShowHashes, opts.SecretEnv = true, "FLEETDIFF_TEST_MODEL_ALIASES" + r, err := Investigate(root, "provided_model_name", "", opts) + if err != nil { + t.Fatal(err) + } + if len(r.Models) != 1 || len(r.Rankings.Movers) != 2 || r.Before.Tokens != 280 || r.After.Tokens != 280 { + t.Fatal("global model totals or project-scoped contributors changed") + } + modelHash, err := attribution.Identity([]byte(secret), "litellm-spend/v1", "model", "PRIVATE_MODEL") + if err != nil || r.Models[0].Hash != fmt.Sprintf("%x", modelHash) { + t.Fatal("model hash scope changed", err) + } + seen := map[string]bool{} + for _, mover := range r.Rankings.Movers { + if !strings.HasPrefix(mover.Item, "model-1 [group-") || seen[mover.Item] || mover.Before.Lower != 140 || mover.After.Lower != 140 { + t.Fatal("separate project groups lost their model alias or disambiguation", mover) + } + seen[mover.Item] = true + } + for _, project := range []string{"project-a", "project-b"} { + hash, err := attribution.Identity([]byte(secret), "litellm-spend/v1", "key", framed("langfuse", project, "PRIVATE_MODEL")) + if err != nil { + t.Fatal(err) + } + found := false + for _, mover := range r.Rankings.Movers { + found = found || mover.Hash == fmt.Sprintf("%016x", binary.BigEndian.Uint64(hash[:8])) + } + if !found { + t.Fatal("project-scoped keyed identity changed") + } + } +} + +func TestModelLabelsPreservePublicNames(t *testing.T) { + prices, _, err := reprice.Bundled() + if err != nil || len(prices.Models) == 0 { + t.Fatal("need a bundled model", err) + } + model := prices.Models[0].ID + body := strings.ReplaceAll(rowJSON("one", "project", "2026-10-10")+rowJSON("two", "project", "2026-10-17"), "PRIVATE_MODEL", model) + root := t.TempDir() + addRun(t, root, "project", "known-model", "2026-10-10T00:00:00Z", "2026-10-18T00:00:00Z", body, 2) + r, err := Investigate(root, "provided_model_name", "", options()) + if err != nil || len(r.Models) != 1 || len(r.Rankings.Movers) != 1 || r.Models[0].Item != model || r.Rankings.Movers[0].Item != model { + t.Fatal("headline and contributor did not preserve the same public model label", err) + } + encoded, err := json.Marshal(r) + if err != nil || strings.Contains(string(encoded), "PRIVATE_") { + t.Fatal("private label escaped alongside the public model label", err) + } +} diff --git a/internal/spend/coverage.go b/internal/spend/coverage.go index 46e6c9f..f1d7ca0 100644 --- a/internal/spend/coverage.go +++ b/internal/spend/coverage.go @@ -100,10 +100,10 @@ func orderCandidates(path string, r *Report, key, expected []byte, selection *sc if routing { read = ReadRouting } - return orderCandidatesRead(path, r, key, expected, selection, read) + return orderCandidatesRead(path, r, key, expected, selection, read, nil) } -func orderCandidatesRead(path string, r *Report, key, expected []byte, selection *scopeSelection, read func(string, func(Row) error) error) error { +func orderCandidatesRead(path string, r *Report, key, expected []byte, selection *scopeSelection, read func(string, func(Row) error) error, modelAliases map[[32]byte]string) error { groups := []*compare.Concentration{&r.Rankings} if r.Cache.Rankings != nil { groups = append(groups, r.Cache.Rankings) @@ -112,6 +112,8 @@ func orderCandidatesRead(path string, r *Report, key, expected []byte, selection return nil } order := map[uint64]int{} + candidateModels := map[uint64][32]byte{} + ambiguousModels := map[uint64]bool{} for _, g := range groups { for _, m := range g.Movers { h, err := strconv.ParseUint(m.Hash, 16, 64) @@ -142,8 +144,20 @@ func orderCandidatesRead(path string, r *Report, key, expected []byte, selection return err } id := binary.BigEndian.Uint64(h[:8]) - if n, tracked := order[id]; present && tracked && n == 0 { - order[id] = row.Number + if n, tracked := order[id]; present && tracked { + if n == 0 { + order[id] = row.Number + } + if modelAliases != nil { + model, err := identity(key, "model", row.Values["model"]) + if err != nil { + return err + } + if previous, ok := candidateModels[id]; ok && previous != model { + ambiguousModels[id] = true + } + candidateModels[id] = model + } } return nil }) @@ -155,6 +169,16 @@ func orderCandidatesRead(path string, r *Report, key, expected []byte, selection } strength := func(v compare.Interval) int64 { return max(v.Upper, -v.Lower) } for _, g := range groups { + if modelAliases != nil { + for i := range g.Movers { + m := &g.Movers[i] + id, _ := strconv.ParseUint(m.Hash, 16, 64) + m.Item = "" + if model, ok := candidateModels[id]; ok && !ambiguousModels[id] { + m.Item = modelAliases[model] + } + } + } slices.SortFunc(g.Movers, func(a, b compare.Mover) int { x, y := strength(a.Delta), strength(b.Delta) if g == r.Cache.Rankings { diff --git a/internal/spend/investigate.go b/internal/spend/investigate.go index 0a9ee76..74fb9d2 100644 --- a/internal/spend/investigate.go +++ b/internal/spend/investigate.go @@ -38,7 +38,9 @@ type Options struct { Top int ShowHashes bool ShowLabels bool - Now time.Time + // Row adapters may share model aliases without changing group identities. + ModelAliases bool + Now time.Time } type aggregate struct { @@ -186,6 +188,9 @@ func InvestigateRows(read func(func(Row) error) error, opts Options) (Report, er } func investigate(path string, opts Options, source func(string, func(Row) error) error) (Report, error) { + if opts.ModelAliases && source == nil { + return Report{}, errors.New("shared model aliases require a row source") + } if opts.GroupBy == "" { opts.GroupBy = "key" } @@ -257,6 +262,8 @@ func investigate(path string, opts Options, source func(string, func(Row) error) models := map[[32]byte][2]*aggregate{} modelOrder := map[[32]byte]int{} modelLabels := map[[32]byte]string{} + modelGroups := map[[32]byte][32]byte{} + splitModels := map[[32]byte]bool{} // Sort full HMAC digests after reading instead of retaining a hash table. // This keeps duplicate detection exact at its existing 256-bit identity. var seen requestDigests @@ -362,6 +369,13 @@ func investigate(path string, opts Options, source func(string, func(Row) error) if e != nil { return e } + if opts.ModelAliases && present { + if first, ok := modelGroups[model]; ok { + splitModels[model] = splitModels[model] || first != group + } else { + modelGroups[model] = group + } + } pair, ok := models[model] if !ok { if len(models) >= MaxModels { @@ -439,6 +453,11 @@ func investigate(path string, opts Options, source func(string, func(Row) error) publicCounts[public]++ } } + var modelAliases map[[32]byte]string + if opts.ModelAliases { + modelAliases = map[[32]byte]string{} + } + splitAliases := map[string]bool{} for i, v := range ordered { m := Model{Item: fmt.Sprintf("model-%d", i+1), Before: v.pair[0].result(), After: v.pair[1].result()} if public := modelLabels[v.hash]; public != "" { @@ -448,6 +467,10 @@ func investigate(path string, opts Options, source func(string, func(Row) error) m.Item = public } } + if opts.ModelAliases { + modelAliases[v.hash] = m.Item + splitAliases[m.Item] = splitModels[v.hash] + } m.Volume = volume(m.Before, m.After) if opts.ShowHashes { m.Hash = fmt.Sprintf("%x", v.hash) @@ -469,7 +492,7 @@ func investigate(path string, opts Options, source func(string, func(Row) error) if r.Before.Cache.Read.Covered == nil || r.After.Cache.Read.Covered == nil || *r.Before.Cache.Read.Covered == 0 || *r.After.Cache.Read.Covered == 0 { r.Cache.Rankings = nil } - if err := orderCandidatesRead(path, &r, key, secondDigest.Sum(nil), selection, read); err != nil { + if err := orderCandidatesRead(path, &r, key, secondDigest.Sum(nil), selection, read, modelAliases); err != nil { return Report{}, err } if !r.Incomplete { @@ -487,6 +510,12 @@ func investigate(path string, opts Options, source func(string, func(Row) error) m := &movers[i] if aliases[m.Hash] == "" { aliases[m.Hash] = fmt.Sprintf("%s-%d", opts.GroupBy, len(aliases)+1) + if opts.ModelAliases && m.Item != "" { + aliases[m.Hash] = m.Item + if splitAliases[m.Item] { + aliases[m.Hash] += fmt.Sprintf(" [group-%d]", len(aliases)) + } + } } m.Item = aliases[m.Hash] if !opts.ShowHashes {