feat(tracing): declare 100% sampling (ot=th:0) on every span

cmd/snowglobe/main.go

@@ -111,6 +111,9 @@ func newProvider(ctx context.Context, serviceName, endpoint, instanceID, hostNam
 			attribute.String("service.instance.id", instanceID),
 			attribute.String("host.name", hostName),
 		)),
+		// Sample everything, as before, but declare it (ot=th:0) so backends
+		// can tell a complete stream from a sampled one.
+		sdktrace.WithSampler(declaredFullSampler{}),
 	}
 	// Metrics are derived from the spans this provider produces rather than
 	// hand-instrumented at the ~200 scenario call sites. That keeps traces and

cmd/snowglobe/sampler.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"strings"
+
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// declaredFullSampler records and samples every span, exactly as the default
+// sampler did, and says so on the wire: each span's tracestate carries the
+// OpenTelemetry vendor entry ot=th:0 (sampling threshold 0, i.e. 100%).
+// A backend reading the stream can then tell a complete trace stream from a
+// probabilistically sampled one, so a span that never arrives reads as
+// missing rather than as unsampled.
+type declaredFullSampler struct{}
+
+func (declaredFullSampler) ShouldSample(p sdktrace.SamplingParameters) sdktrace.SamplingResult {
+	ts := trace.SpanContextFromContext(p.ParentContext).TraceState()
+	// Insert places the member first, as W3C requires for a modified entry.
+	// It only fails on an invalid key or value; if it ever does, keep the
+	// parent's tracestate rather than dropping it.
+	if updated, err := ts.Insert("ot", withTh0(ts.Get("ot"))); err == nil {
+		ts = updated
+	}
+	return sdktrace.SamplingResult{
+		Decision:   sdktrace.RecordAndSample,
+		Tracestate: ts,
+	}
+}
+
+func (declaredFullSampler) Description() string { return "DeclaredFull{ot=th:0}" }
+
+// withTh0 returns the ot tracestate value with its th sub-key set to 0 and
+// placed first. The value is ;-separated k:v sub-keys; any existing th is
+// replaced and every other sub-key (such as rv) is kept in order.
+func withTh0(v string) string {
+	parts := []string{"th:0"}
+	for _, kv := range strings.Split(v, ";") {
+		if kv == "" || kv == "th" || strings.HasPrefix(kv, "th:") {
+			continue
+		}
+		parts = append(parts, kv)
+	}
+	return strings.Join(parts, ";")
+}

cmd/snowglobe/sampler_test.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func newDeclaredFullProvider() (*sdktrace.TracerProvider, *tracetest.SpanRecorder) {
+	rec := tracetest.NewSpanRecorder()
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(declaredFullSampler{}),
+		sdktrace.WithSpanProcessor(rec),
+	)
+	return tp, rec
+}
+
+func TestDeclaredFullRootSpanCarriesTh0(t *testing.T) {
+	tp, _ := newDeclaredFullProvider()
+	t.Cleanup(func() {
+		if err := tp.Shutdown(context.Background()); err != nil {
+			t.Errorf("shutdown: %v", err)
+		}
+	})
+
+	_, span := tp.Tracer("t").Start(context.Background(), "root")
+	span.End()
+
+	if got := span.SpanContext().TraceState().Get("ot"); got != "th:0" {
+		t.Fatalf("root ot = %q, want th:0", got)
+	}
+}
+
+func TestDeclaredFullChildSpanCarriesTh0(t *testing.T) {
+	tp, _ := newDeclaredFullProvider()
+	t.Cleanup(func() {
+		if err := tp.Shutdown(context.Background()); err != nil {
+			t.Errorf("shutdown: %v", err)
+		}
+	})
+	// Snowglobe runs one provider per service instance, so a downstream hop
+	// starts its span from a different provider with the caller's context.
+	other, _ := newDeclaredFullProvider()
+	t.Cleanup(func() {
+		if err := other.Shutdown(context.Background()); err != nil {
+			t.Errorf("shutdown: %v", err)
+		}
+	})
+
+	ctx, root := tp.Tracer("t").Start(context.Background(), "root")
+	_, child := tp.Tracer("t").Start(ctx, "child")
+	_, remote := other.Tracer("other").Start(ctx, "downstream")
+	child.End()
+	remote.End()
+	root.End()
+
+	for name, s := range map[string]trace.Span{"same-provider child": child, "other-provider child": remote} {
+		if got := s.SpanContext().TraceState().Get("ot"); got != "th:0" {
+			t.Fatalf("%s ot = %q, want th:0", name, got)
+		}
+		if s.SpanContext().TraceID() != root.SpanContext().TraceID() {
+			t.Fatalf("%s left the parent's trace", name)
+		}
+	}
+}
+
+func TestDeclaredFullKeepsParentTracestateWithOtFirst(t *testing.T) {
+	tp, _ := newDeclaredFullProvider()
+	t.Cleanup(func() {
+		if err := tp.Shutdown(context.Background()); err != nil {
+			t.Errorf("shutdown: %v", err)
+		}
+	})
+
+	ts, err := trace.ParseTraceState("foo=bar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	parent := trace.NewSpanContext(trace.SpanContextConfig{
+		TraceID:    trace.TraceID{1},
+		SpanID:     trace.SpanID{1},
+		TraceFlags: trace.FlagsSampled,
+		TraceState: ts,
+		Remote:     true,
+	})
+	ctx := trace.ContextWithRemoteSpanContext(context.Background(), parent)
+	_, span := tp.Tracer("t").Start(ctx, "child")
+	span.End()
+
+	if got := span.SpanContext().TraceState().String(); got != "ot=th:0,foo=bar" {
+		t.Fatalf("tracestate = %q, want ot=th:0,foo=bar", got)
+	}
+}
+
+func TestWithTh0(t *testing.T) {
+	cases := []struct{ in, want string }{
+		{"rv:abc123", "th:0;rv:abc123"},
+		{"th:8", "th:0"},
+		{"", "th:0"},
+		{"rv:abc123;th:8", "th:0;rv:abc123"},
+	}
+	for _, c := range cases {
+		if got := withTh0(c.in); got != c.want {
+			t.Errorf("withTh0(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+func TestDeclaredFullRecordsAndSamplesEverySpan(t *testing.T) {
+	tp, rec := newDeclaredFullProvider()
+	t.Cleanup(func() {
+		if err := tp.Shutdown(context.Background()); err != nil {
+			t.Errorf("shutdown: %v", err)
+		}
+	})
+
+	const n = 25
+	tr := tp.Tracer("t")
+	ctx, root := tr.Start(context.Background(), "root")
+	for i := 0; i < n-1; i++ {
+		_, s := tr.Start(ctx, "child")
+		if !s.IsRecording() || !s.SpanContext().IsSampled() {
+			t.Fatalf("span %d not recorded and sampled", i)
+		}
+		s.End()
+	}
+	root.End()
+
+	if got := len(rec.Ended()); got != n {
+		t.Fatalf("recorded %d spans, want %d", got, n)
+	}
+	for _, s := range rec.Ended() {
+		if !s.SpanContext().IsSampled() {
+			t.Fatalf("span %q not sampled", s.Name())
+		}
+	}
+}