Push metrics over OTLP/HTTP instead of Prometheus /metrics (MARSOHS-1593)

README.md

@@ -183,6 +183,40 @@ How a run's outcome is recorded:
 - any other exit status (2 = panic/timeout, signal): the run fails with the
   exit code and output.
 
+## Metrics
+
+Both `tester serve` and `tester run` push metrics over OTLP/HTTP; there is no
+`/metrics` endpoint to scrape. Export is configured entirely through the
+standard OpenTelemetry environment variables and is off when no endpoint is
+set:
+
+```sh
+OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-nyc3.digitalocean.com:443
+OTEL_EXPORTER_OTLP_HEADERS="authorization=Bearer $DO_API_TOKEN"
+OTEL_EXPORTER_OTLP_COMPRESSION=gzip
+OTEL_SERVICE_NAME=tester            # tester-runner for workers
+OTEL_METRIC_EXPORT_INTERVAL=60000   # ms, optional
+```
+
+`service.instance.id` defaults to the hostname; override with
+`OTEL_RESOURCE_ATTRIBUTES=service.instance.id=...`.
+
+| metric | kind | attributes | source |
+| --- | --- | --- | --- |
+| `tester.test.duration` (s) | histogram | `name`, `state` | server |
+| `tester.test.last_run` (unix s) | gauge | `name`, `state` | server |
+| `tester.run.queue_wait` (s) | histogram | `package` | server |
+| `tester.run.duration` (s) | histogram | `package`, `result` | server |
+| `tester.run.claims` | counter | `result` = claimed / empty / error | server |
+| `tester.run.scheduled` | counter | `package` | server |
+| `tester.run.reset` | counter | `package`, `result` = reset / failed | server |
+| `tester.runs.pending` | gauge | `package`, `state` = queued / running | server |
+| `tester.runner.polls` | counter | `result` = claimed / empty / error | runner |
+| `tester.runner.run.duration` (s) | histogram | `package`, `result` | runner |
+
+`tester.test.duration` keeps the bucket boundaries of the Prometheus
+`tester_tb_run_duration_s` histogram it replaces.
+
 ## Development
 
 ```sh

cmd/tester/run.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/digitalocean/tester/runner"
+	"github.com/digitalocean/tester/telemetry"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -18,6 +19,16 @@ var runCmd = &cobra.Command{
 	Short: "start a test runner",
 	Args:  cobra.ExactArgs(0),
 	Run: func(cmd *cobra.Command, args []string) {
+		telemetryShutdown, err := telemetry.Setup(context.Background(), "tester-runner")
+		if err != nil {
+			log.Fatalf("failed to configure telemetry: %s", err)
+		}
+		defer func() {
+			if err := telemetryShutdown(context.Background()); err != nil {
+				log.Printf("failed to flush telemetry: %s", err)
+			}
+		}()
+
 		opts := []runner.Option{runner.WithTesterAddr(viper.GetString("run-tester-addr"))}
 		if apiKey := viper.GetString("run-api-key"); apiKey != "" {
 			opts = append(opts, runner.WithAPIKey(apiKey))

cmd/tester/serve.go

@@ -20,8 +20,8 @@ import (
 	"github.com/digitalocean/tester/http/okta"
 	"github.com/digitalocean/tester/scheduler"
 	"github.com/digitalocean/tester/slack"
+	"github.com/digitalocean/tester/telemetry"
 	"github.com/jackc/pgx/v5/pgxpool"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"golang.org/x/sync/errgroup"
@@ -59,6 +59,16 @@ var serveCmd = &cobra.Command{
 			pkgBin.Close()
 		}
 
+		telemetryShutdown, err := telemetry.Setup(context.Background(), "tester")
+		if err != nil {
+			log.Fatalf("failed to configure telemetry: %s", err)
+		}
+		defer func() {
+			if err := telemetryShutdown(context.Background()); err != nil {
+				log.Printf("failed to flush telemetry: %s", err)
+			}
+		}()
+
 		l, err := net.Listen("tcp", viper.GetString("serve-addr"))
 		if err != nil {
 			log.Fatalf("failed to listen on %s", viper.GetString("serve-addr"))
@@ -91,6 +101,14 @@ var serveCmd = &cobra.Command{
 			}
 		}
 
+		packageNames := make([]string, 0, len(cfg.Packages))
+		for _, pkg := range cfg.Packages {
+			packageNames = append(packageNames, pkg.Name)
+		}
+		if err := telemetry.RegisterPendingRunsGauge(dbStore, packageNames); err != nil {
+			log.Fatalf("failed to register pending runs gauge: %s", err)
+		}
+
 		var httpOpts []testerhttp.Option
 		if apiKey := viper.GetString("serve-api-key"); apiKey != "" {
 			httpOpts = append(httpOpts, testerhttp.WithAPIKey(apiKey))
@@ -155,7 +173,6 @@ var serveCmd = &cobra.Command{
 		apiHandler := testerhttp.NewAPIHandler(dbStore, cfg.Packages, httpOpts...)
 
 		mux := http.NewServeMux()
-		mux.Handle("/metrics", promhttp.Handler())
 		mux.Handle("/api/", apiHandler)
 
 		oktaAuthHandler := configureOktaAuth(uiHandler.RenderError)

go.mod

@@ -11,11 +11,15 @@ require (
 	github.com/jackc/pgx/v5 v5.11.0
 	github.com/jackc/tern/v2 v2.4.3
 	github.com/okta/okta-jwt-verifier-golang v1.3.1
-	github.com/prometheus/client_golang v1.24.1
 	github.com/slack-go/slack v0.29.0
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.12.1
+	go.opentelemetry.io/otel v1.47.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0
+	go.opentelemetry.io/otel/metric v1.47.0
+	go.opentelemetry.io/otel/sdk v1.47.0
+	go.opentelemetry.io/otel/sdk/metric v1.47.0
 	go.uber.org/mock v0.6.0
 	golang.org/x/sync v0.23.0
 	gotest.tools v2.2.0+incompatible
@@ -26,14 +30,17 @@ require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.5.0 // indirect
 	github.com/Masterminds/sprig/v3 v3.3.0 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect
 	github.com/fsnotify/fsnotify v1.10.1 // indirect
+	github.com/go-logr/logr v1.4.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/goccy/go-json v0.10.6 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
 	github.com/huandu/xstrings v1.6.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -49,22 +56,26 @@ require (
 	github.com/lestrrat-go/option v1.0.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627 // indirect
 	github.com/pelletier/go-toml/v2 v2.4.3 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.6.3 // indirect
-	github.com/prometheus/common v0.71.0 // indirect
-	github.com/prometheus/procfs v0.22.0 // indirect
 	github.com/sagikazarmark/locafero v0.12.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/log v1.47.0 // indirect
+	go.opentelemetry.io/otel/trace v1.47.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	golang.org/x/crypto v0.57.0 // indirect
+	golang.org/x/net v0.59.0 // indirect
 	golang.org/x/sys v0.48.0 // indirect
 	golang.org/x/text v0.42.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect
+	google.golang.org/grpc v1.83.2 // indirect
 	google.golang.org/protobuf v1.36.12 // indirect
 )