Langfuse: enum-only observation severity for next release

CHANGELOG.md

@@ -2,7 +2,16 @@
 
 Notable user-visible changes are recorded here, newest first.
 
-## [0.13.0] - Unreleased
+## [Unreleased]
+
+- Read only Langfuse's bounded `level` enum, show ERROR observation token
+  subsets and avoid severity-only usage-review flags for DEFAULT/DEBUG/WARNING.
+  Provider call status and billing finality remain unknown; missing/invalid
+  levels still require review. Token/cost totals and LiteLLM behavior are unchanged.
+- Extend JSON/JSONL/CSV and gzip privacy checks without reading `status_message`
+  or observation payloads. This does not change the published v0.13.0 artifacts.
+
+## [0.13.0] - 2026-10-24
 
 See the [release notes](docs/releases/v0.13.0.md).
 

docs/LANGFUSE.md

@@ -57,9 +57,11 @@ Who drove the increase?
   1 leading tracked trace_name accounts for 100% of the net recorded increase.
 ```
 
-This version doesn't read a success or status field from Langfuse observations,
-so the report marks status as unknown and lists those records for usage review.
-Recorded token and cost totals are still shown.
+The v0.13.0 binary does not read a success or status field from Langfuse
+observations: it reports unknown status and lists those records for usage review.
+Recorded token and cost totals are still shown. The fixture's `level` field is
+ignored by v0.13.0; non-error severity is not proof of provider success or complete
+billing.
 
 ## Grouping and accounting
 

examples/langfuse-spend/export/synthetic-langfuse/manifests/2026-10-07T01-00-00.json

@@ -15,7 +15,7 @@
       "format": "jsonl-raw",
       "compressed": false,
       "contentType": "application/x-ndjson; charset=utf-8",
-      "sizeBytes": 1322,
+      "sizeBytes": 1394,
       "rowCount": 4
     }
   ]

examples/langfuse-spend/export/synthetic-langfuse/manifests/2026-10-08T01-00-00.json

@@ -15,7 +15,7 @@
       "format": "jsonl-raw",
       "compressed": false,
       "contentType": "application/x-ndjson; charset=utf-8",
-      "sizeBytes": 1984,
+      "sizeBytes": 2092,
       "rowCount": 6
     }
   ]

examples/langfuse-spend/export/synthetic-langfuse/observations_v2/2026-10-07T00-00-00.jsonl

@@ -1,4 +1,4 @@
-{"id":"2026-10-07-1","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-07 00:05:00.000000","end_time":"2026-10-07 00:05:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
-{"id":"2026-10-07-2","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-07 00:10:00.000000","end_time":"2026-10-07 00:10:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
-{"id":"2026-10-07-3","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-07 00:15:00.000000","end_time":"2026-10-07 00:15:01.000000","provided_model_name":"synthetic-model","trace_name":"background","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
-{"id":"2026-10-07-4","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-07 00:20:00.000000","end_time":"2026-10-07 00:20:01.000000","provided_model_name":"synthetic-model","trace_name":"background","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
+{"level":"DEFAULT","id":"2026-10-07-1","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-07 00:05:00.000000","end_time":"2026-10-07 00:05:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
+{"level":"DEFAULT","id":"2026-10-07-2","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-07 00:10:00.000000","end_time":"2026-10-07 00:10:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
+{"level":"DEFAULT","id":"2026-10-07-3","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-07 00:15:00.000000","end_time":"2026-10-07 00:15:01.000000","provided_model_name":"synthetic-model","trace_name":"background","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
+{"level":"DEFAULT","id":"2026-10-07-4","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-07 00:20:00.000000","end_time":"2026-10-07 00:20:01.000000","provided_model_name":"synthetic-model","trace_name":"background","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}

examples/langfuse-spend/export/synthetic-langfuse/observations_v2/2026-10-08T00-00-00.jsonl

@@ -1,6 +1,6 @@
-{"id":"2026-10-08-1","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:05:00.000000","end_time":"2026-10-08 00:05:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":2250,"output":750,"total":3000},"total_cost":0.003}
-{"id":"2026-10-08-2","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:10:00.000000","end_time":"2026-10-08 00:10:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":2250,"output":750,"total":3000},"total_cost":0.003}
-{"id":"2026-10-08-3","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:15:00.000000","end_time":"2026-10-08 00:15:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":2250,"output":750,"total":3000},"total_cost":0.003}
-{"id":"2026-10-08-4","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:20:00.000000","end_time":"2026-10-08 00:20:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":2250,"output":750,"total":3000},"total_cost":0.003}
-{"id":"2026-10-08-5","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:25:00.000000","end_time":"2026-10-08 00:25:01.000000","provided_model_name":"synthetic-model","trace_name":"background","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
-{"id":"2026-10-08-6","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:30:00.000000","end_time":"2026-10-08 00:30:01.000000","provided_model_name":"synthetic-model","trace_name":"background","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
+{"level":"DEFAULT","id":"2026-10-08-1","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:05:00.000000","end_time":"2026-10-08 00:05:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":2250,"output":750,"total":3000},"total_cost":0.003}
+{"level":"DEFAULT","id":"2026-10-08-2","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:10:00.000000","end_time":"2026-10-08 00:10:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":2250,"output":750,"total":3000},"total_cost":0.003}
+{"level":"DEFAULT","id":"2026-10-08-3","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:15:00.000000","end_time":"2026-10-08 00:15:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":2250,"output":750,"total":3000},"total_cost":0.003}
+{"level":"DEFAULT","id":"2026-10-08-4","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:20:00.000000","end_time":"2026-10-08 00:20:01.000000","provided_model_name":"synthetic-model","trace_name":"interactive","release":"synthetic-example","usage_details":{"input":2250,"output":750,"total":3000},"total_cost":0.003}
+{"level":"DEFAULT","id":"2026-10-08-5","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:25:00.000000","end_time":"2026-10-08 00:25:01.000000","provided_model_name":"synthetic-model","trace_name":"background","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}
+{"level":"DEFAULT","id":"2026-10-08-6","project_id":"synthetic-langfuse","type":"GENERATION","start_time":"2026-10-08 00:30:00.000000","end_time":"2026-10-08 00:30:01.000000","provided_model_name":"synthetic-model","trace_name":"background","release":"synthetic-example","usage_details":{"input":1500,"output":500,"total":2000},"total_cost":0.002}

internal/cli/langfuse.go

@@ -46,6 +46,7 @@ func runLangfuse(dir, group, tag, format string, opts spend.Options, out, errout
 		text = strings.ReplaceAll(text, "Comparison incomplete: some selected call types are not analyzed.", "Comparison incomplete: the supplied local export does not fully cover the selected periods.")
 		text = strings.ReplaceAll(text, "gateway-recorded", "Langfuse-recorded")
 		text = strings.ReplaceAll(text, "your local LiteLLM logs", "your local Langfuse export")
+		text = strings.ReplaceAll(text, "Unknown status", "Call status unknown (Langfuse doesn't record it)")
 		if c.Incomplete {
 			text = strings.ReplaceAll(text, "Whole-period per-request split unavailable while selected call types remain unanalyzed.", "Whole-period per-request split unavailable while local export coverage is incomplete.")
 			text = strings.ReplaceAll(text, "Next: review unsupported call types in your local Langfuse export; analyze those rows before drawing a whole-export token-change conclusion.", "Next: restore missing or partial files and manifests before drawing a whole-period conclusion; observed totals above remain partial.")

internal/cli/langfuse_docs_test.go

@@ -40,7 +40,8 @@ func checkLangfuseExample(t *testing.T, format string, output []byte) {
 			r.Before.Requests != 4 || r.After.Requests != 6 || r.Before.Tokens != 8000 || r.After.Tokens != 16000 ||
 			r.Before.RecordedSpend != "0.008" || r.After.RecordedSpend != "0.016" ||
 			r.Before.Quality.UnknownStatus != 4 || r.After.Quality.UnknownStatus != 6 ||
-			r.Before.Quality.Unclear != 4 || r.After.Quality.Unclear != 6 ||
+			r.Before.Quality.Unclear != 0 || r.After.Quality.Unclear != 0 ||
+			r.Before.Quality.ObservationLevels["DEFAULT"] != 4 || r.After.Quality.ObservationLevels["DEFAULT"] != 6 ||
 			r.Increase == nil || r.Increase.Count != 1 || r.Increase.Share.Lower != 1 {
 			t.Fatal("public example totals, contributor or manifest coverage differ")
 		}
@@ -49,11 +50,15 @@ func checkLangfuseExample(t *testing.T, format string, output []byte) {
 			"trace_name-1: 4,000 -> 12,000 tokens; change +8,000.",
 			"trace_name-2: 4,000 -> 4,000 tokens; change +0.",
 			"1 leading tracked trace_name accounts for 100% of the net recorded increase.",
-			"Unknown status: 4 -> 6."} {
+			"Call status unknown (Langfuse doesn't record it): 4 -> 6.",
+			"Observation levels: DEFAULT 4 -> 6."} {
 			if !strings.Contains(string(output), want) {
 				t.Fatalf("example missing %q", want)
 			}
 		}
+		if strings.Contains(string(output), "Records needing usage review") || strings.Contains(string(output), "Tokens recorded on ERROR observations") {
+			t.Fatal("clean DEFAULT example contains review or ERROR-token noise")
+		}
 	}
 }
 
@@ -62,9 +67,12 @@ func TestLangfuseGuideStatusContract(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	want := "This version doesn't read a success or status field from Langfuse observations, so the report marks status as unknown and lists those records for usage review. Recorded token and cost totals are still shown."
+	want := "non-error severity is not proof of provider success or complete billing."
 	if !strings.Contains(strings.Join(strings.Fields(string(doc)), " "), want) {
-		t.Fatal("Langfuse status limitation must describe this version, not only the example")
+		t.Fatal("Langfuse severity must not imply provider success or billing completeness")
+	}
+	if strings.Contains(string(doc), "## Observation severity") || strings.Contains(string(doc), "(unreleased)") {
+		t.Fatal("deferred next-release section must not ship on main yet")
 	}
 }
 

internal/cli/langfuse_level_test.go

@@ -0,0 +1,163 @@
+// SPDX-License-Identifier: Apache-2.0
+// Code authors: Vijay and Codex
+
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/llm-measurement/fleetdiff/internal/langfuse"
+	"github.com/llm-measurement/fleetdiff/internal/spend"
+)
+
+func TestLangfuseLevelCLIReviewPrivacyAndSubsets(t *testing.T) {
+	testLangfuseLevel(t, "")
+}
+
+func TestReleaseBinaryLangfuseLevel(t *testing.T) {
+	binary := os.Getenv("FLEETDIFF_RELEASE_BINARY")
+	if binary == "" {
+		t.Skip("set FLEETDIFF_RELEASE_BINARY to test the packaged binary")
+	}
+	binary, err := filepath.Abs(binary)
+	if err != nil {
+		t.Fatal(err)
+	}
+	testLangfuseLevel(t, binary)
+}
+
+func testLangfuseLevel(t *testing.T, binary string) {
+	t.Helper()
+	for _, mode := range []string{"known-non-error", "error-and-invalid"} {
+		t.Run(mode, func(t *testing.T) {
+			root := t.TempDir()
+			if err := os.CopyFS(root, os.DirFS("../../examples/langfuse-spend/export")); err != nil {
+				t.Fatal(err)
+			}
+			files, err := filepath.Glob(filepath.Join(root, "synthetic-langfuse/observations_v2/*.jsonl"))
+			if err != nil || len(files) != 2 {
+				t.Fatal("synthetic files unavailable")
+			}
+			for _, file := range files {
+				data, err := os.ReadFile(file)
+				if err != nil {
+					t.Fatal(err)
+				}
+				var body strings.Builder
+				for i, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
+					line = strings.Replace(line, `"level":"DEFAULT",`, "", 1)
+					level := []string{"DEFAULT", "DEBUG", "WARNING"}[i%3]
+					if mode == "error-and-invalid" && i == 0 {
+						level = "ERROR"
+					} else if mode == "error-and-invalid" && i == 1 {
+						level = "PRIVATE_LEVEL_SENTINEL"
+					}
+					body.WriteString(`{"level":"` + level + `","input":"PRIVATE_INPUT_SENTINEL","output":"PRIVATE_OUTPUT_SENTINEL","metadata":"PRIVATE_METADATA_SENTINEL","tool_calls":"PRIVATE_TOOLS_SENTINEL","status_message":"PRIVATE_STATUS_SENTINEL",` + line[1:] + "\n")
+				}
+				if err := os.WriteFile(file, []byte(body.String()), 0600); err != nil {
+					t.Fatal(err)
+				}
+			}
+			manifests, err := filepath.Glob(filepath.Join(root, "synthetic-langfuse/manifests/*.json"))
+			if err != nil || len(manifests) != 2 {
+				t.Fatal("synthetic manifests unavailable")
+			}
+			for _, path := range manifests {
+				data, err := os.ReadFile(path)
+				if err != nil {
+					t.Fatal(err)
+				}
+				var m map[string]any
+				if err := json.Unmarshal(data, &m); err != nil {
+					t.Fatal(err)
+				}
+				for _, f := range m["files"].([]any) {
+					file := f.(map[string]any)
+					info, err := os.Stat(filepath.Join(root, "synthetic-langfuse/observations_v2", filepath.Base(file["key"].(string))))
+					if err != nil {
+						t.Fatal(err)
+					}
+					file["sizeBytes"] = info.Size()
+				}
+				data, _ = json.Marshal(m)
+				if err := os.WriteFile(path, data, 0600); err != nil {
+					t.Fatal(err)
+				}
+			}
+			for _, format := range []string{"text", "json"} {
+				args := langfuseExampleArgs(format)
+				args[2] = root
+				var out, diagnostic bytes.Buffer
+				code := 0
+				if binary == "" {
+					code = Run(args, &out, &diagnostic)
+				} else {
+					ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+					cmd := exec.CommandContext(ctx, binary, args...)
+					cmd.Stdout, cmd.Stderr = &out, &diagnostic
+					err := cmd.Run()
+					cancel()
+					if err != nil {
+						t.Fatal("packaged severity command failed", err)
+					}
+				}
+				if code != 0 || diagnostic.Len() != 0 {
+					t.Fatal("severity command failed", code, diagnostic.String())
+				}
+				if strings.Contains(out.String()+diagnostic.String(), "PRIVATE_") {
+					t.Fatal("severity or payload sentinel leaked")
+				}
+				if format == "text" {
+					completeness := strings.Index(out.String(), "How complete is the export?")
+					levels := strings.Index(out.String(), "  Observation levels:")
+					if completeness < 0 || levels < completeness || strings.Count(out.String(), "Observation levels:") != 1 || strings.Contains(out.String(), "Observation level DEFAULT:") || strings.Contains(out.String(), "unknown 0 -> 0") || strings.Contains(out.String(), "ERROR 0 -> 0") {
+						t.Fatal("level summary is not compact, nonzero-only and in completeness section")
+					}
+					if !strings.Contains(out.String(), "provider call status is unknown") || mode == "error-and-invalid" && !strings.Contains(out.String(), "Tokens recorded on ERROR observations:") {
+						t.Fatal("severity/status or error-subset wording absent")
+					}
+					if mode == "known-non-error" && strings.Contains(out.String(), "Records needing usage review") {
+						t.Fatal("non-error severity caused usage-review noise")
+					}
+					if mode == "known-non-error" && strings.Contains(out.String(), "Tokens recorded on ERROR observations:") {
+						t.Fatal("zero ERROR-token line should be omitted")
+					}
+					if !strings.Contains(out.String(), "Call status unknown (Langfuse doesn't record it): 4 -> 6.") {
+						t.Fatal("short call-status wording absent")
+					}
+					continue
+				}
+				var r langfuse.Report
+				if err := json.Unmarshal(out.Bytes(), &r); err != nil {
+					t.Fatal(err)
+				}
+				if r.Before.Tokens != 8000 || r.After.Tokens != 16000 || r.Before.RecordedSpend != "0.008" || r.After.RecordedSpend != "0.016" || r.Before.Quality.UnknownStatus != 4 || r.After.Quality.UnknownStatus != 6 || r.Before.Quality.Failed != 0 || r.After.Quality.Failed != 0 || r.Volume != nil {
+					t.Fatal("level changed recorded totals or claimed provider status")
+				}
+				if mode == "known-non-error" {
+					if r.Before.Quality.Unclear != 0 || r.After.Quality.Unclear != 0 || r.Before.ErrorObservationTokens != nil {
+						t.Fatal("non-error severity caused review/error subset")
+					}
+				} else if r.Before.Quality.Unclear != 2 || r.After.Quality.Unclear != 2 || r.Before.Quality.ErrorObservations != 1 || r.After.Quality.ObservationLevels["unknown"] != 1 || r.Before.ErrorObservationTokens == nil || r.Before.ErrorObservationTokens.Total != 2000 || r.After.ErrorObservationTokens == nil || r.After.ErrorObservationTokens.Total != 3000 {
+					t.Fatal("error or invalid severity review/subset differs")
+				}
+			}
+		})
+	}
+}
+
+func TestZeroErrorTokenLineOmitted(t *testing.T) {
+	var out bytes.Buffer
+	renderSpend(&out, spend.Report{Before: spend.Window{ErrorObservationTokens: &spend.FailureTokens{Finality: "unknown"}}})
+	if strings.Contains(out.String(), "Tokens recorded on ERROR observations:") {
+		t.Fatal("non-nil zero token subset produced an ERROR-token line")
+	}
+}