diff --git a/cmd/snowglobe/main.go b/cmd/snowglobe/main.go index 040428e..8e716cb 100644 --- a/cmd/snowglobe/main.go +++ b/cmd/snowglobe/main.go @@ -111,6 +111,9 @@ func newProvider(ctx context.Context, serviceName, endpoint, instanceID, hostNam attribute.String("service.instance.id", instanceID), attribute.String("host.name", hostName), )), + // Sample everything, as before, but declare it (ot=th:0) so backends + // can tell a complete stream from a sampled one. + sdktrace.WithSampler(declaredFullSampler{}), } // Metrics are derived from the spans this provider produces rather than // hand-instrumented at the ~200 scenario call sites. That keeps traces and diff --git a/cmd/snowglobe/sampler.go b/cmd/snowglobe/sampler.go new file mode 100644 index 0000000..4c25efb --- /dev/null +++ b/cmd/snowglobe/sampler.go @@ -0,0 +1,46 @@ +package main + +import ( + "strings" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +// declaredFullSampler records and samples every span, exactly as the default +// sampler did, and says so on the wire: each span's tracestate carries the +// OpenTelemetry vendor entry ot=th:0 (sampling threshold 0, i.e. 100%). +// A backend reading the stream can then tell a complete trace stream from a +// probabilistically sampled one, so a span that never arrives reads as +// missing rather than as unsampled. +type declaredFullSampler struct{} + +func (declaredFullSampler) ShouldSample(p sdktrace.SamplingParameters) sdktrace.SamplingResult { + ts := trace.SpanContextFromContext(p.ParentContext).TraceState() + // Insert places the member first, as W3C requires for a modified entry. + // It only fails on an invalid key or value; if it ever does, keep the + // parent's tracestate rather than dropping it. + if updated, err := ts.Insert("ot", withTh0(ts.Get("ot"))); err == nil { + ts = updated + } + return sdktrace.SamplingResult{ + Decision: sdktrace.RecordAndSample, + Tracestate: ts, + } +} + +func (declaredFullSampler) Description() string { return "DeclaredFull{ot=th:0}" } + +// withTh0 returns the ot tracestate value with its th sub-key set to 0 and +// placed first. The value is ;-separated k:v sub-keys; any existing th is +// replaced and every other sub-key (such as rv) is kept in order. +func withTh0(v string) string { + parts := []string{"th:0"} + for _, kv := range strings.Split(v, ";") { + if kv == "" || kv == "th" || strings.HasPrefix(kv, "th:") { + continue + } + parts = append(parts, kv) + } + return strings.Join(parts, ";") +} diff --git a/cmd/snowglobe/sampler_test.go b/cmd/snowglobe/sampler_test.go new file mode 100644 index 0000000..4040f74 --- /dev/null +++ b/cmd/snowglobe/sampler_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "testing" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +func newDeclaredFullProvider() (*sdktrace.TracerProvider, *tracetest.SpanRecorder) { + rec := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(declaredFullSampler{}), + sdktrace.WithSpanProcessor(rec), + ) + return tp, rec +} + +func TestDeclaredFullRootSpanCarriesTh0(t *testing.T) { + tp, _ := newDeclaredFullProvider() + t.Cleanup(func() { + if err := tp.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown: %v", err) + } + }) + + _, span := tp.Tracer("t").Start(context.Background(), "root") + span.End() + + if got := span.SpanContext().TraceState().Get("ot"); got != "th:0" { + t.Fatalf("root ot = %q, want th:0", got) + } +} + +func TestDeclaredFullChildSpanCarriesTh0(t *testing.T) { + tp, _ := newDeclaredFullProvider() + t.Cleanup(func() { + if err := tp.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown: %v", err) + } + }) + // Snowglobe runs one provider per service instance, so a downstream hop + // starts its span from a different provider with the caller's context. + other, _ := newDeclaredFullProvider() + t.Cleanup(func() { + if err := other.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown: %v", err) + } + }) + + ctx, root := tp.Tracer("t").Start(context.Background(), "root") + _, child := tp.Tracer("t").Start(ctx, "child") + _, remote := other.Tracer("other").Start(ctx, "downstream") + child.End() + remote.End() + root.End() + + for name, s := range map[string]trace.Span{"same-provider child": child, "other-provider child": remote} { + if got := s.SpanContext().TraceState().Get("ot"); got != "th:0" { + t.Fatalf("%s ot = %q, want th:0", name, got) + } + if s.SpanContext().TraceID() != root.SpanContext().TraceID() { + t.Fatalf("%s left the parent's trace", name) + } + } +} + +func TestDeclaredFullKeepsParentTracestateWithOtFirst(t *testing.T) { + tp, _ := newDeclaredFullProvider() + t.Cleanup(func() { + if err := tp.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown: %v", err) + } + }) + + ts, err := trace.ParseTraceState("foo=bar") + if err != nil { + t.Fatal(err) + } + parent := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: trace.TraceID{1}, + SpanID: trace.SpanID{1}, + TraceFlags: trace.FlagsSampled, + TraceState: ts, + Remote: true, + }) + ctx := trace.ContextWithRemoteSpanContext(context.Background(), parent) + _, span := tp.Tracer("t").Start(ctx, "child") + span.End() + + if got := span.SpanContext().TraceState().String(); got != "ot=th:0,foo=bar" { + t.Fatalf("tracestate = %q, want ot=th:0,foo=bar", got) + } +} + +func TestWithTh0(t *testing.T) { + cases := []struct{ in, want string }{ + {"rv:abc123", "th:0;rv:abc123"}, + {"th:8", "th:0"}, + {"", "th:0"}, + {"rv:abc123;th:8", "th:0;rv:abc123"}, + } + for _, c := range cases { + if got := withTh0(c.in); got != c.want { + t.Errorf("withTh0(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestDeclaredFullRecordsAndSamplesEverySpan(t *testing.T) { + tp, rec := newDeclaredFullProvider() + t.Cleanup(func() { + if err := tp.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown: %v", err) + } + }) + + const n = 25 + tr := tp.Tracer("t") + ctx, root := tr.Start(context.Background(), "root") + for i := 0; i < n-1; i++ { + _, s := tr.Start(ctx, "child") + if !s.IsRecording() || !s.SpanContext().IsSampled() { + t.Fatalf("span %d not recorded and sampled", i) + } + s.End() + } + root.End() + + if got := len(rec.Ended()); got != n { + t.Fatalf("recorded %d spans, want %d", got, n) + } + for _, s := range rec.Ended() { + if !s.SpanContext().IsSampled() { + t.Fatalf("span %q not sampled", s.Name()) + } + } +}