feat: add OpenTelemetry tracing support

cmd/app/grpc.go

@@ -35,6 +35,8 @@
 	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+
 	"github.com/sigstore/fulcio/pkg/ca"
 	"github.com/sigstore/fulcio/pkg/config"
 	gw "github.com/sigstore/fulcio/pkg/generated/protobuf"
@@ -161,6 +163,7 @@
 	logger, opts := log.SetupGRPCLogging()
 
 	serverOpts := []grpc.ServerOption{
+		grpc.StatsHandler(otelgrpc.NewServerHandler()),
 		grpc.UnaryInterceptor(
 			grpcmw.ChainUnaryServer(
 				grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandlerContext(panicRecoveryHandler)), // recovers from per-transaction panics elegantly, so put it first
@@ -274,8 +277,10 @@
 func createLegacyGRPCServer(cfg *config.FulcioConfig, unixDomainSocket string, v2Server gw.CAServer) (*grpcServer, error) {
 	logger, opts := log.SetupGRPCLogging()
 
-	myServer := grpc.NewServer(grpc.UnaryInterceptor(
+	myServer := grpc.NewServer(
+		grpc.StatsHandler(otelgrpc.NewServerHandler()),
+		grpc.UnaryInterceptor(
 		grpcmw.ChainUnaryServer(
 			grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandlerContext(panicRecoveryHandler)), // recovers from per-transaction panics elegantly, so put it first
 			middleware.UnaryRequestID(middleware.UseXRequestIDMetadataOption(true), middleware.XRequestMetadataLimitOption(128)),
 			grpc_zap.UnaryServerInterceptor(logger, opts...),

cmd/app/http.go

@@ -33,6 +33,7 @@
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/cors"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	gw "github.com/sigstore/fulcio/pkg/generated/protobuf"
 	legacy_gw "github.com/sigstore/fulcio/pkg/generated/protobuf/legacy"
 	"github.com/sigstore/fulcio/pkg/log"
@@ -101,6 +102,7 @@
 	// enable CORS
 	// cors.Default() configures to accept requests for all domains
 	handler = cors.Default().Handler(handler)
+	handler = otelhttp.NewHandler(handler, "fulcio-http")
 
 	api := http.Server{
 		Addr:    serverEndpoint,

cmd/app/serve.go

@@ -47,6 +47,7 @@
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sigstore/model-validation-operator/pkg/tracing"
 	certauth "github.com/sigstore/fulcio/pkg/ca"
 	"github.com/sigstore/fulcio/pkg/ca/ephemeralca"
 	"github.com/sigstore/fulcio/pkg/ca/fileca"
@@ -65,6 +66,8 @@
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.uber.org/zap"
 	"goa.design/goa/v3/grpc/middleware"
 	"google.golang.org/api/option"
@@ -135,6 +138,11 @@
 		v1.PublicKeyDetails_PKIX_ED25519,
 	}), "the list of allowed client signing algorithms")
 
+	cmd.Flags().Bool("tracing-enabled", false, "enable OpenTelemetry tracing")
+	cmd.Flags().String("tracing-endpoint", "", "OTLP gRPC collector endpoint (default: localhost:4317 via OTEL_EXPORTER_OTLP_ENDPOINT)")
+	cmd.Flags().Bool("tracing-insecure", true, "use insecure gRPC connection to the collector")
+	cmd.Flags().Bool("tracing-stdout", false, "export traces to stdout instead of OTLP (for debugging)")
+
 	// convert "http-host" flag to "host" and "http-port" flag to be "port"
 	cmd.Flags().SetNormalizeFunc(func(_ *pflag.FlagSet, name string) pflag.NormalizedName {
 		switch name {
@@ -232,6 +240,28 @@
 	// Setup the logger to dev/prod
 	log.ConfigureLogger(viper.GetString("log_type"))
 
+	if viper.GetBool("tracing-enabled") {
+		opts := []tracing.Option{
+			tracing.WithServiceName("fulcio"),
+			tracing.WithInsecure(viper.GetBool("tracing-insecure")),
+		}
+		if ep := viper.GetString("tracing-endpoint"); ep != "" {
+			opts = append(opts, tracing.WithEndpoint(ep))
+		}
+		if viper.GetBool("tracing-stdout") {
+			opts = append(opts, tracing.WithStdoutExporter())
+		}
+		shutdownTracing, err := tracing.SetupTracing(ctx, opts...)
+		if err != nil {
+			log.Logger.Fatalf("failed to initialize tracing: %v", err)
+		}
+		defer func() {
+			if err := shutdownTracing(context.Background()); err != nil {
+				log.Logger.Errorf("error shutting down tracing: %v", err)
+			}
+		}()
+	}
+
 	algorithmStrings := viper.GetStringSlice("client-signing-algorithms")
 	var algorithmConfig []v1.PublicKeyDetails
 	for _, s := range algorithmStrings {
@@ -342,11 +372,12 @@
 			}
 			httpClient = &http.Client{
 				Timeout:   30 * time.Second,
-				Transport: transport,
+				Transport: otelhttp.NewTransport(transport),
 			}
 		} else {
 			httpClient = &http.Client{
-				Timeout: 30 * time.Second,
+				Timeout:   30 * time.Second,
+				Transport: otelhttp.NewTransport(http.DefaultTransport),
 			}
 		}
 		ctClient, err = ctclient.New(logURL, httpClient, opts)
@@ -468,6 +499,7 @@
 
 	d := duplex.New(
 		port,
+		grpc.StatsHandler(otelgrpc.NewServerHandler()),
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
 		grpc.KeepaliveParams(keepalive.ServerParameters{
 			MaxConnectionIdle: viper.GetDuration("idle-connection-timeout"),

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
 	github.com/coreos/go-oidc/v3 v3.17.0
 	github.com/fsnotify/fsnotify v1.9.0
-	github.com/go-jose/go-jose/v4 v4.1.3
+	github.com/go-jose/go-jose/v4 v4.1.4
 	github.com/google/certificate-transparency-go v1.3.3
 	github.com/google/go-cmp v0.7.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
@@ -23,6 +23,7 @@ require (
 	github.com/prometheus/client_model v0.6.2
 	github.com/prometheus/common v0.67.5
 	github.com/rs/cors v1.11.1
+	github.com/sigstore/model-validation-operator/pkg/tracing v0.0.0-20260613122948-a36fd5feae2c
 	github.com/sigstore/protobuf-specs v0.5.0
 	github.com/sigstore/sigstore v1.10.5
 	github.com/sigstore/sigstore/pkg/signature/kms/aws v1.10.5
@@ -37,13 +38,15 @@ require (
 	github.com/tink-crypto/tink-go-awskms/v2 v2.1.0
 	github.com/tink-crypto/tink-go-gcpkms/v2 v2.2.0
 	github.com/tink-crypto/tink-go/v2 v2.6.0
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0
 	go.step.sm/crypto v0.77.1
 	go.uber.org/zap v1.27.1
 	go.yaml.in/yaml/v3 v3.0.4
 	goa.design/goa/v3 v3.23.4
 	google.golang.org/api v0.273.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7
-	google.golang.org/grpc v1.79.3
+	google.golang.org/grpc v1.81.1
 	google.golang.org/protobuf v1.36.11
 	sigs.k8s.io/release-utils v0.12.3
 )
@@ -145,26 +148,25 @@ require (
 	github.com/thales-e-security/pool v0.0.2 // indirect
 	github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
-	go.opentelemetry.io/otel v1.42.0 // indirect
+	go.opentelemetry.io/otel v1.44.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
-	go.opentelemetry.io/otel/metric v1.42.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.42.0 // indirect
-	go.opentelemetry.io/otel/trace v1.42.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.44.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
+	go.opentelemetry.io/otel/trace v1.44.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
-	golang.org/x/crypto v0.49.0 // indirect
-	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/crypto v0.51.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/term v0.41.0 // indirect
-	golang.org/x/text v0.35.0 // indirect
+	golang.org/x/sys v0.45.0 // indirect
+	golang.org/x/term v0.43.0 // indirect
+	golang.org/x/text v0.37.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
 	google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )