diff --git a/README.md b/README.md index 14abc7e..40b21eb 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,40 @@ How a run's outcome is recorded: - any other exit status (2 = panic/timeout, signal): the run fails with the exit code and output. +## Metrics + +Both `tester serve` and `tester run` push metrics over OTLP/HTTP; there is no +`/metrics` endpoint to scrape. Export is configured entirely through the +standard OpenTelemetry environment variables and is off when no endpoint is +set: + +```sh +OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-nyc3.digitalocean.com:443 +OTEL_EXPORTER_OTLP_HEADERS="authorization=Bearer $DO_API_TOKEN" +OTEL_EXPORTER_OTLP_COMPRESSION=gzip +OTEL_SERVICE_NAME=tester # tester-runner for workers +OTEL_METRIC_EXPORT_INTERVAL=60000 # ms, optional +``` + +`service.instance.id` defaults to the hostname; override with +`OTEL_RESOURCE_ATTRIBUTES=service.instance.id=...`. + +| metric | kind | attributes | source | +| --- | --- | --- | --- | +| `tester.test.duration` (s) | histogram | `name`, `state` | server | +| `tester.test.last_run` (unix s) | gauge | `name`, `state` | server | +| `tester.run.queue_wait` (s) | histogram | `package` | server | +| `tester.run.duration` (s) | histogram | `package`, `result` | server | +| `tester.run.claims` | counter | `result` = claimed / empty / error | server | +| `tester.run.scheduled` | counter | `package` | server | +| `tester.run.reset` | counter | `package`, `result` = reset / failed | server | +| `tester.runs.pending` | gauge | `package`, `state` = queued / running | server | +| `tester.runner.polls` | counter | `result` = claimed / empty / error | runner | +| `tester.runner.run.duration` (s) | histogram | `package`, `result` | runner | + +`tester.test.duration` keeps the bucket boundaries of the Prometheus +`tester_tb_run_duration_s` histogram it replaces. + ## Development ```sh diff --git a/cmd/tester/run.go b/cmd/tester/run.go index 34392ec..940e04d 100644 --- a/cmd/tester/run.go +++ b/cmd/tester/run.go @@ -9,6 +9,7 @@ import ( "time" "github.com/digitalocean/tester/runner" + "github.com/digitalocean/tester/telemetry" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -18,6 +19,16 @@ var runCmd = &cobra.Command{ Short: "start a test runner", Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { + telemetryShutdown, err := telemetry.Setup(context.Background(), "tester-runner") + if err != nil { + log.Fatalf("failed to configure telemetry: %s", err) + } + defer func() { + if err := telemetryShutdown(context.Background()); err != nil { + log.Printf("failed to flush telemetry: %s", err) + } + }() + opts := []runner.Option{runner.WithTesterAddr(viper.GetString("run-tester-addr"))} if apiKey := viper.GetString("run-api-key"); apiKey != "" { opts = append(opts, runner.WithAPIKey(apiKey)) diff --git a/cmd/tester/serve.go b/cmd/tester/serve.go index 5d24182..5ef08e0 100644 --- a/cmd/tester/serve.go +++ b/cmd/tester/serve.go @@ -20,8 +20,8 @@ import ( "github.com/digitalocean/tester/http/okta" "github.com/digitalocean/tester/scheduler" "github.com/digitalocean/tester/slack" + "github.com/digitalocean/tester/telemetry" "github.com/jackc/pgx/v5/pgxpool" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" "github.com/spf13/viper" "golang.org/x/sync/errgroup" @@ -59,6 +59,16 @@ var serveCmd = &cobra.Command{ pkgBin.Close() } + telemetryShutdown, err := telemetry.Setup(context.Background(), "tester") + if err != nil { + log.Fatalf("failed to configure telemetry: %s", err) + } + defer func() { + if err := telemetryShutdown(context.Background()); err != nil { + log.Printf("failed to flush telemetry: %s", err) + } + }() + l, err := net.Listen("tcp", viper.GetString("serve-addr")) if err != nil { log.Fatalf("failed to listen on %s", viper.GetString("serve-addr")) @@ -91,6 +101,14 @@ var serveCmd = &cobra.Command{ } } + packageNames := make([]string, 0, len(cfg.Packages)) + for _, pkg := range cfg.Packages { + packageNames = append(packageNames, pkg.Name) + } + if err := telemetry.RegisterPendingRunsGauge(dbStore, packageNames); err != nil { + log.Fatalf("failed to register pending runs gauge: %s", err) + } + var httpOpts []testerhttp.Option if apiKey := viper.GetString("serve-api-key"); apiKey != "" { httpOpts = append(httpOpts, testerhttp.WithAPIKey(apiKey)) @@ -155,7 +173,6 @@ var serveCmd = &cobra.Command{ apiHandler := testerhttp.NewAPIHandler(dbStore, cfg.Packages, httpOpts...) mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) mux.Handle("/api/", apiHandler) oktaAuthHandler := configureOktaAuth(uiHandler.RenderError) diff --git a/go.mod b/go.mod index a0078ec..4976d62 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,15 @@ require ( github.com/jackc/pgx/v5 v5.11.0 github.com/jackc/tern/v2 v2.4.3 github.com/okta/okta-jwt-verifier-golang v1.3.1 - github.com/prometheus/client_golang v1.24.1 github.com/slack-go/slack v0.29.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.12.1 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.23.0 gotest.tools v2.2.0+incompatible @@ -26,14 +30,17 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.5.0 // indirect github.com/Masterminds/sprig/v3 v3.3.0 // indirect - github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/huandu/xstrings v1.6.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -49,22 +56,26 @@ require ( github.com/lestrrat-go/option v1.0.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627 // indirect github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.6.3 // indirect - github.com/prometheus/common v0.71.0 // indirect - github.com/prometheus/procfs v0.22.0 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/trace v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect + golang.org/x/net v0.59.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/go.sum b/go.sum index 71e008f..1e3cd93 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -23,6 +23,11 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= @@ -30,6 +35,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -44,6 +51,8 @@ github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzq github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/huandu/xstrings v1.6.1 h1:yYdKSd6Sjv3fNZt9BDjl1FDsPNfQEaYi19L5LzK2JKs= github.com/huandu/xstrings v1.6.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -58,14 +67,10 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackc/tern/v2 v2.4.3 h1:g293d3OZgW7OFEhsYXgEv0C21jea2boNr0VR4K5I7OY= github.com/jackc/tern/v2 v2.4.3/go.mod h1:rMpMuRYcff5wWLptoTSO1qcDxJ4OodysvK17i2SVBys= -github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= -github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -94,8 +99,6 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/okta/okta-jwt-verifier-golang v1.3.1 h1:V+9W5KD3nG7xN0UYtnzXtkurGcs71bLwzPFuUGNMwdE= github.com/okta/okta-jwt-verifier-golang v1.3.1/go.mod h1:cHffA777f7Yi4K+yDzUp89sGD5v8sk04Pc3CiT1OMR8= github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627 h1:pSCLCl6joCFRnjpeojzOpEYs4q7Vditq8fySFG5ap3Y= @@ -105,16 +108,8 @@ github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= -github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= -github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= -github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= -github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8= -github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk= -github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= -github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= @@ -145,12 +140,28 @@ github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUm github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= -go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= -go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= @@ -166,6 +177,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -190,6 +203,14 @@ golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4f golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/api.go b/http/api.go index e7a4749..f16d03f 100644 --- a/http/api.go +++ b/http/api.go @@ -9,14 +9,16 @@ import ( "net/http" "sort" "strings" + "time" "github.com/digitalocean/tester" "github.com/digitalocean/tester/alerting" "github.com/digitalocean/tester/db" "github.com/digitalocean/tester/slack" + "github.com/digitalocean/tester/telemetry" "github.com/google/uuid" "github.com/gorilla/mux" - "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel/metric" ) // APIHandler is the http handler for presenting the API. @@ -105,12 +107,12 @@ func (h *APIHandler) submitTest(w http.ResponseWriter, r *http.Request) { return } - runLabels := prometheus.Labels{ - "name": test.Result.Name, - "state": string(test.Result.State), - } - RunDurationMetric.With(runLabels).Observe(test.Result.FinishedAt.Sub(test.Result.StartedAt).Seconds()) - RunLastMetric.With(runLabels).Set(float64(test.Result.StartedAt.Unix())) + testAttrs := metric.WithAttributes( + telemetry.AttrName.String(test.Result.Name), + telemetry.AttrState.String(string(test.Result.State)), + ) + telemetry.TestDuration.Record(r.Context(), telemetry.Seconds(test.Result.FinishedAt.Sub(test.Result.StartedAt)), testAttrs) + telemetry.TestLastRun.Record(r.Context(), float64(test.Result.StartedAt.Unix()), testAttrs) if test.Result.State == tester.TBStateFailed { go func() { @@ -187,14 +189,20 @@ func (h *APIHandler) claimRun(w http.ResponseWriter, r *http.Request) { run, err := h.db.ClaimRun(r.Context(), r.Header.Get("User-Agent"), packages, claimRunRequest.PackageBlacklist) if err != nil { if errors.Is(err, db.ErrNotFound) { + telemetry.RunClaims.Add(r.Context(), 1, metric.WithAttributes(telemetry.AttrResult.String(telemetry.ResultEmpty))) renderAPIError(w, http.StatusNotFound, fmt.Errorf("no runs for packages: %s", strings.Join(packages, ", "))) return } + telemetry.RunClaims.Add(r.Context(), 1, metric.WithAttributes(telemetry.AttrResult.String(telemetry.ResultError))) log.Printf("failed to claim run: %s", err) renderAPIError(w, http.StatusInternalServerError, err) return } + telemetry.RunClaims.Add(r.Context(), 1, metric.WithAttributes(telemetry.AttrResult.String(telemetry.ResultClaimed))) + telemetry.RunQueueWait.Record(r.Context(), telemetry.Seconds(run.StartedAt.Sub(run.EnqueuedAt)), + metric.WithAttributes(telemetry.AttrPackage.String(run.Package))) + w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(run) } @@ -222,6 +230,7 @@ func (h *APIHandler) completeRun(w http.ResponseWriter, r *http.Request) { renderAPIError(w, http.StatusInternalServerError, err) return } + recordRunDuration(r.Context(), run, telemetry.ResultCompleted) w.WriteHeader(http.StatusOK) } @@ -257,10 +266,21 @@ func (h *APIHandler) failRun(w http.ResponseWriter, r *http.Request) { renderAPIError(w, http.StatusInternalServerError, err) return } + recordRunDuration(r.Context(), run, telemetry.ResultFailed) w.WriteHeader(http.StatusOK) } +// recordRunDuration records claim -> now for a run that just finished. Runs +// that were never claimed (failed by the scheduler, say) have no duration. +func recordRunDuration(ctx context.Context, run *tester.Run, result string) { + if run.StartedAt.IsZero() { + return + } + telemetry.RunDuration.Record(ctx, telemetry.Seconds(time.Since(run.StartedAt)), + metric.WithAttributes(telemetry.AttrPackage.String(run.Package), telemetry.AttrResult.String(result))) +} + func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request) { pkgName := mux.Vars(r)["package_name"] pkg, ok := h.packages[pkgName] diff --git a/http/metrics.go b/http/metrics.go deleted file mode 100644 index 2634647..0000000 --- a/http/metrics.go +++ /dev/null @@ -1,48 +0,0 @@ -package http - -import "github.com/prometheus/client_golang/prometheus" - -const ( - // RunDurationMetricName is the name of the metric for test and benchmark run - // durations. - RunDurationMetricName = "run_duration_s" - - // RunLastMetricName is the name of the metric for the test and benchmark last - // run timestamp. - RunLastMetricName = "run_last_timestamp" -) - -// RunDurationMetric is the the metric for test and benchmark run durations. -var RunDurationMetric = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "tester", - Subsystem: "tb", - Name: RunDurationMetricName, - Help: "Amount of time tests or benchmarks take.", - Buckets: []float64{ - // TODO need to figure out more appropriate bucketing - 0.001, 0.01, - 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, - 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, - 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, - 180, 240, 300, 360, 420, 480, 540, 600, 660, 720, 780, 840, 900, - }, - }, - []string{"name", "state"}, -) - -// RunLastMetric is the the metric for test and benchmark last run timestamps. -var RunLastMetric = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "testr", - Subsystem: "tb", - Name: RunLastMetricName, - Help: "Timestamp of the last run for a test or benchmark.", - }, - []string{"name", "state"}, -) - -func init() { - prometheus.MustRegister(RunDurationMetric) - prometheus.MustRegister(RunLastMetric) -} diff --git a/runner/runner.go b/runner/runner.go index 4bcd9fd..0d61487 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -19,7 +19,9 @@ import ( "github.com/digitalocean/tester" testerhttp "github.com/digitalocean/tester/http" + "github.com/digitalocean/tester/telemetry" "github.com/google/uuid" + "go.opentelemetry.io/otel/metric" ) var ( @@ -306,11 +308,21 @@ func (r *Runner) claimRun(ctx context.Context) (*tester.Run, error) { func (r *Runner) runOnce(ctx context.Context) error { run, err := r.claimRun(ctx) if err != nil { + telemetry.RunnerPolls.Add(ctx, 1, metric.WithAttributes(telemetry.AttrResult.String(telemetry.ResultError))) return fmt.Errorf("claiming run: %w", err) } if run == nil { + telemetry.RunnerPolls.Add(ctx, 1, metric.WithAttributes(telemetry.AttrResult.String(telemetry.ResultEmpty))) return nil } + telemetry.RunnerPolls.Add(ctx, 1, metric.WithAttributes(telemetry.AttrResult.String(telemetry.ResultClaimed))) + + claimedAt := time.Now() + result := telemetry.ResultError + defer func() { + telemetry.RunnerRunDuration.Record(ctx, telemetry.Seconds(time.Since(claimedAt)), + metric.WithAttributes(telemetry.AttrPackage.String(run.Package), telemetry.AttrResult.String(result))) + }() pkg, err := r.getPackageInfo(ctx, run.Package) if err != nil { @@ -381,6 +393,7 @@ func (r *Runner) runOnce(ctx context.Context) error { if err := r.failRun(run.ID, errorMessage); err != nil { log.Printf("failed to mark run failed: %s", err) } + result = telemetry.ResultFailed return exitErr } } @@ -437,6 +450,7 @@ func (r *Runner) runOnce(ctx context.Context) error { if err != nil { log.Printf("failed to mark run complete: %s", err) } + result = telemetry.ResultCompleted log.Printf("finished run for %s", run.Package) return nil diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index ce99550..d4d6d5c 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -12,7 +12,9 @@ import ( "github.com/digitalocean/tester" "github.com/digitalocean/tester/db" + "github.com/digitalocean/tester/telemetry" "github.com/google/uuid" + "go.opentelemetry.io/otel/metric" "golang.org/x/sync/errgroup" ) @@ -202,6 +204,7 @@ func (s *Scheduler) scheduleRuns(ctx context.Context) error { continue } if scheduled { + telemetry.RunScheduled.Add(ctx, 1, metric.WithAttributes(telemetry.AttrPackage.String(pkg.Name))) log.Printf("scheduled run %s", pkg.Name) } } @@ -253,6 +256,8 @@ func (s *Scheduler) resetStaleRuns(ctx context.Context) error { if err != nil { return fmt.Errorf("failing timed out run %s (%s): %w", run.ID, run.Package, err) } + telemetry.RunReset.Add(ctx, 1, metric.WithAttributes( + telemetry.AttrPackage.String(run.Package), telemetry.AttrResult.String(telemetry.ResultFailed))) log.Printf("failed run %s (%s): exceeded run timeout %d time(s)", run.Package, run.ID, run.ResetCount+1) continue } @@ -264,6 +269,8 @@ func (s *Scheduler) resetStaleRuns(ctx context.Context) error { } return err } + telemetry.RunReset.Add(ctx, 1, metric.WithAttributes( + telemetry.AttrPackage.String(run.Package), telemetry.AttrResult.String(telemetry.ResultReset))) log.Printf("reset run %s (%s): exceeded run timeout, attempt %d of %d", run.Package, run.ID, run.ResetCount+2, s.maxResets+1) } diff --git a/telemetry/instruments.go b/telemetry/instruments.go new file mode 100644 index 0000000..c6aab3a --- /dev/null +++ b/telemetry/instruments.go @@ -0,0 +1,191 @@ +package telemetry + +import ( + "context" + "log" + "time" + + "github.com/digitalocean/tester" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// ScopeName is the instrumentation scope for every tester instrument. +const ScopeName = "github.com/digitalocean/tester" + +// Attribute keys shared by the instruments below. Kept short and stable: they +// become Prometheus labels on the far side of the edge. +const ( + AttrPackage = attribute.Key("package") + AttrName = attribute.Key("name") + AttrState = attribute.Key("state") + AttrResult = attribute.Key("result") +) + +// Result values for AttrResult. +const ( + ResultClaimed = "claimed" + ResultEmpty = "empty" + ResultError = "error" + ResultCompleted = "completed" + ResultFailed = "failed" + ResultReset = "reset" +) + +// testDurationBuckets mirrors the buckets of the Prometheus histogram this +// replaces (tester_tb_run_duration_s), so history stays comparable. +var testDurationBuckets = []float64{ + 0.001, 0.01, + 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, + 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, + 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, + 180, 240, 300, 360, 420, 480, 540, 600, 660, 720, 780, 840, 900, +} + +// runDurationBuckets covers whole-run timescales: seconds to hours. Queue +// wait was measured at ~140 min median before the scheduler fixes, so the +// upper buckets exist to make regressions visible, not because they are +// acceptable. +var runDurationBuckets = []float64{ + 1, 5, 15, 30, 60, 120, 300, 600, 900, 1200, 1800, 2700, 3600, 5400, 7200, 10800, 14400, +} + +// Server-side instruments (tester serve). +var ( + // TestDuration is the wall time of one test as reported by the runner. + TestDuration metric.Float64Histogram + // TestLastRun is the unix time of the most recent report for a test. + TestLastRun metric.Float64Gauge + // RunQueueWait is enqueued_at -> started_at for a claimed run. + RunQueueWait metric.Float64Histogram + // RunDuration is started_at -> finished_at for a completed or failed run. + RunDuration metric.Float64Histogram + // RunClaims counts claim requests by result. + RunClaims metric.Int64Counter + // RunScheduled counts runs enqueued by the scheduler. + RunScheduled metric.Int64Counter + // RunReset counts runs the scheduler reset or failed for exceeding the + // run timeout. + RunReset metric.Int64Counter +) + +// Runner-side instruments (tester run). +var ( + // RunnerPolls counts claim attempts by result. Its presence at a steady + // rate is the runner heartbeat; ResultError distinguishes a runner that + // cannot reach the server from one that is idle. + RunnerPolls metric.Int64Counter + // RunnerRunDuration is claim -> report for a run as seen by the runner. + RunnerRunDuration metric.Float64Histogram +) + +func init() { + // otel's global MeterProvider delegates to whatever Setup installs later, + // so instruments created here work whether or not export is enabled. + // Setup re-binds them to the real provider once it exists. + bind(otel.Meter(ScopeName)) +} + +// bind (re)creates every instrument on m. Called from init against the +// global delegating meter, from Setup against the installed SDK provider, +// and from tests against a provider with a manual reader. +func bind(m metric.Meter) { + var err error + + TestDuration, err = m.Float64Histogram("tester.test.duration", + metric.WithDescription("Wall time of a test as reported by the runner."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(testDurationBuckets...)) + must(err) + TestLastRun, err = m.Float64Gauge("tester.test.last_run", + metric.WithDescription("Unix time of the most recent report for a test."), + metric.WithUnit("s")) + must(err) + RunQueueWait, err = m.Float64Histogram("tester.run.queue_wait", + metric.WithDescription("Time a run waited in the queue before a runner claimed it."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(runDurationBuckets...)) + must(err) + RunDuration, err = m.Float64Histogram("tester.run.duration", + metric.WithDescription("Time from claim to completion or failure of a run."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(runDurationBuckets...)) + must(err) + RunClaims, err = m.Int64Counter("tester.run.claims", + metric.WithDescription("Claim requests handled by the server, by result."), + metric.WithUnit("{claim}")) + must(err) + RunScheduled, err = m.Int64Counter("tester.run.scheduled", + metric.WithDescription("Runs enqueued by the scheduler."), + metric.WithUnit("{run}")) + must(err) + RunReset, err = m.Int64Counter("tester.run.reset", + metric.WithDescription("Runs reset or failed by the scheduler for exceeding the run timeout."), + metric.WithUnit("{run}")) + must(err) + + RunnerPolls, err = m.Int64Counter("tester.runner.polls", + metric.WithDescription("Claim attempts made by the runner, by result."), + metric.WithUnit("{poll}")) + must(err) + RunnerRunDuration, err = m.Float64Histogram("tester.runner.run.duration", + metric.WithDescription("Time from claim to report of a run as seen by the runner."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(runDurationBuckets...)) + must(err) +} + +// PendingRunsSource is the subset of db.DB the pending-runs gauge needs. +type PendingRunsSource interface { + ListPendingRuns(ctx context.Context) ([]*tester.Run, error) +} + +// RegisterPendingRunsGauge observes tester.runs.pending{package,state} from +// the database on every export. state is "queued" (not started) or "running". +// Every configured package is reported in both states, with 0 when empty, so +// "queue depth for package X" is always a present series and alert rules do +// not have to reason about absent data. +func RegisterPendingRunsGauge(src PendingRunsSource, packages []string) error { + m := otel.Meter(ScopeName) + pending, err := m.Int64ObservableGauge("tester.runs.pending", + metric.WithDescription("Runs that have not finished, by package and state."), + metric.WithUnit("{run}")) + if err != nil { + return err + } + _, err = m.RegisterCallback(func(ctx context.Context, o metric.Observer) error { + runs, err := src.ListPendingRuns(ctx) + if err != nil { + return err + } + type key struct{ pkg, state string } + counts := make(map[key]int64, 2*len(packages)) + for _, pkg := range packages { + counts[key{pkg, "queued"}] = 0 + counts[key{pkg, "running"}] = 0 + } + for _, run := range runs { + state := "queued" + if !run.StartedAt.IsZero() { + state = "running" + } + counts[key{run.Package, state}]++ + } + for k, n := range counts { + o.ObserveInt64(pending, n, metric.WithAttributes(AttrPackage.String(k.pkg), AttrState.String(k.state))) + } + return nil + }, pending) + return err +} + +// Seconds converts a duration to the float seconds every duration instrument +// here records. +func Seconds(d time.Duration) float64 { return d.Seconds() } + +func must(err error) { + if err != nil { + log.Fatalf("telemetry: creating instrument: %s", err) + } +} diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go new file mode 100644 index 0000000..9593666 --- /dev/null +++ b/telemetry/telemetry.go @@ -0,0 +1,120 @@ +// Package telemetry wires tester's metrics to OpenTelemetry and pushes them +// over OTLP/HTTP. +// +// tester used to expose Prometheus metrics on /metrics for something to +// scrape. On App Platform nothing can scrape a specific instance, which is +// one reason the worker was pinned to a single replica. Pushing from every +// process removes that constraint and matches how the OHP cluster ships +// metrics to DigitalOcean's observability edge. +// +// All exporter configuration comes from the standard OTEL_* environment +// variables, so the deploy spec, not the code, owns endpoint and credentials: +// +// OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-nyc3.digitalocean.com:443 +// OTEL_EXPORTER_OTLP_HEADERS=authorization=Bearer +// OTEL_EXPORTER_OTLP_COMPRESSION=gzip +// OTEL_SERVICE_NAME=tester +// OTEL_METRIC_EXPORT_INTERVAL=60000 (ms, optional) +// +// When no endpoint is configured Setup leaves the no-op global provider in +// place and logs once; instruments stay usable and cost nothing. +package telemetry + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" +) + +// Enabled reports whether an OTLP endpoint is configured in the environment. +func Enabled() bool { + return os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" || + os.Getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") != "" +} + +// Setup installs the global MeterProvider. defaultServiceName is used when +// OTEL_SERVICE_NAME is not set; the process hostname becomes +// service.instance.id unless OTEL_RESOURCE_ATTRIBUTES overrides it. The +// returned shutdown flushes pending metrics and must be called on exit. +func Setup(ctx context.Context, defaultServiceName string) (shutdown func(context.Context) error, err error) { + noop := func(context.Context) error { return nil } + + if !Enabled() { + log.Printf("telemetry: OTEL_EXPORTER_OTLP_ENDPOINT not set, metrics export disabled") + return noop, nil + } + + res, err := buildResource(ctx, defaultServiceName) + if err != nil { + return noop, err + } + + exporter, err := otlpmetrichttp.New(ctx) + if err != nil { + return noop, fmt.Errorf("creating OTLP metric exporter: %w", err) + } + + provider := sdkmetric.NewMeterProvider( + sdkmetric.WithResource(res), + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter)), + ) + otel.SetMeterProvider(provider) + bind(provider.Meter(ScopeName)) + + log.Printf("telemetry: exporting metrics over OTLP/HTTP to %s as service.name=%q service.instance.id=%q", + endpointForLog(), attr(res, semconv.ServiceNameKey), attr(res, semconv.ServiceInstanceIDKey)) + + return func(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return provider.Shutdown(ctx) + }, nil +} + +// buildResource describes this process. Defaults are applied first and the +// environment second, so OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES win +// over defaultServiceName and the hostname. +func buildResource(ctx context.Context, defaultServiceName string) (*resource.Resource, error) { + hostname, err := os.Hostname() + if err != nil || hostname == "" { + hostname = "unknown" + } + + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceName(defaultServiceName), + semconv.ServiceInstanceID(hostname), + ), + resource.WithFromEnv(), + resource.WithTelemetrySDK(), + ) + if err != nil && !errors.Is(err, resource.ErrPartialResource) { + return nil, fmt.Errorf("building resource: %w", err) + } + return res, nil +} + +func attr(res *resource.Resource, key attribute.Key) string { + v, ok := res.Set().Value(key) + if !ok { + return "" + } + return v.AsString() +} + +func endpointForLog() string { + if v := os.Getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"); v != "" { + return v + } + return os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") +} diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go new file mode 100644 index 0000000..b7c9e4f --- /dev/null +++ b/telemetry/telemetry_test.go @@ -0,0 +1,160 @@ +package telemetry + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/digitalocean/tester" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" +) + +func TestSetup_DisabledWithoutEndpoint(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "") + + assert.False(t, Enabled()) + shutdown, err := Setup(context.Background(), "tester") + require.NoError(t, err) + require.NoError(t, shutdown(context.Background())) + + // Instruments remain safe to use against the no-op provider. + RunClaims.Add(context.Background(), 1, metric.WithAttributes(AttrResult.String(ResultEmpty))) +} + +func TestSetup_ExportsOverOTLPHTTP(t *testing.T) { + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/metrics" && r.Method == http.MethodPost { + requests.Add(1) + assert.Equal(t, "Bearer test-token", r.Header.Get("Authorization")) + assert.Equal(t, "application/x-protobuf", r.Header.Get("Content-Type")) + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", srv.URL) + t.Setenv("OTEL_EXPORTER_OTLP_INSECURE", "true") + t.Setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer test-token") + t.Setenv("OTEL_SERVICE_NAME", "tester-test") + + assert.True(t, Enabled()) + shutdown, err := Setup(context.Background(), "tester") + require.NoError(t, err) + + RunClaims.Add(context.Background(), 1, metric.WithAttributes(AttrResult.String(ResultClaimed))) + + // Shutdown flushes the periodic reader, which must produce at least one + // export to the fake edge. + require.NoError(t, shutdown(context.Background())) + assert.GreaterOrEqual(t, requests.Load(), int32(1), "expected at least one OTLP metrics POST") + + // Leave the global provider in a known state for other tests. + otel.SetMeterProvider(sdkmetric.NewMeterProvider()) +} + +func TestSetup_ResourceDefaultsAndEnvPrecedence(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:1") // never contacted: no export before shutdown w/o data + t.Setenv("OTEL_EXPORTER_OTLP_INSECURE", "true") + t.Setenv("OTEL_SERVICE_NAME", "") + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "service.instance.id=override") + + shutdown, err := Setup(context.Background(), "tester-default") + require.NoError(t, err) + defer func() { + _ = shutdown(context.Background()) + otel.SetMeterProvider(sdkmetric.NewMeterProvider()) + }() + + _, ok := otel.GetMeterProvider().(*sdkmetric.MeterProvider) + require.True(t, ok, "Setup must install an SDK MeterProvider") + + // Re-derive the resource the same way Setup does and check precedence. + // (The SDK does not expose the provider's resource directly.) + res, err := buildResource(context.Background(), "tester-default") + require.NoError(t, err) + attrs := res.Set() + name, _ := attrs.Value(semconv.ServiceNameKey) + assert.Equal(t, "tester-default", name.AsString(), "default service.name applies when env is empty") + inst, _ := attrs.Value(semconv.ServiceInstanceIDKey) + assert.Equal(t, "override", inst.AsString(), "OTEL_RESOURCE_ATTRIBUTES wins over the hostname default") +} + +func TestInstruments_RecordExpectedSeries(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(provider) + bind(provider.Meter(ScopeName)) + defer func() { + otel.SetMeterProvider(sdkmetric.NewMeterProvider()) + bind(otel.Meter(ScopeName)) + }() + + ctx := context.Background() + TestDuration.Record(ctx, 1.5, metric.WithAttributes(AttrName.String("TestA"), AttrState.String("passed"))) + TestLastRun.Record(ctx, 1700000000, metric.WithAttributes(AttrName.String("TestA"), AttrState.String("passed"))) + RunQueueWait.Record(ctx, 42, metric.WithAttributes(AttrPackage.String("agents"))) + RunClaims.Add(ctx, 1, metric.WithAttributes(AttrResult.String(ResultClaimed))) + RunnerPolls.Add(ctx, 2, metric.WithAttributes(AttrResult.String(ResultEmpty))) + + src := &fakePending{runs: []*tester.Run{ + {ID: uuid.New(), Package: "agents"}, + {ID: uuid.New(), Package: "agents", StartedAt: time.Now()}, + {ID: uuid.New(), Package: "dbaas"}, + }} + require.NoError(t, RegisterPendingRunsGauge(src, []string{"agents", "dbaas", "idle"})) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(ctx, &rm)) + + byName := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + assert.Equal(t, ScopeName, sm.Scope.Name) + for _, m := range sm.Metrics { + byName[m.Name] = m + } + } + + for _, name := range []string{ + "tester.test.duration", "tester.test.last_run", "tester.run.queue_wait", + "tester.run.claims", "tester.runner.polls", "tester.runs.pending", + } { + assert.Contains(t, byName, name) + } + assert.Equal(t, "s", byName["tester.test.duration"].Unit) + assert.Equal(t, "{claim}", byName["tester.run.claims"].Unit) + + hist, ok := byName["tester.test.duration"].Data.(metricdata.Histogram[float64]) + require.True(t, ok) + require.Len(t, hist.DataPoints, 1) + assert.Equal(t, testDurationBuckets, hist.DataPoints[0].Bounds, "must keep the legacy Prometheus buckets") + + pending, ok := byName["tester.runs.pending"].Data.(metricdata.Gauge[int64]) + require.True(t, ok) + got := map[string]int64{} + for _, dp := range pending.DataPoints { + pkg, _ := dp.Attributes.Value(AttrPackage) + state, _ := dp.Attributes.Value(AttrState) + got[pkg.AsString()+"/"+state.AsString()] = dp.Value + } + assert.Equal(t, map[string]int64{ + "agents/queued": 1, "agents/running": 1, + "dbaas/queued": 1, "dbaas/running": 0, + "idle/queued": 0, "idle/running": 0, + }, got, "every configured package reports both states, zero when empty") +} + +type fakePending struct{ runs []*tester.Run } + +func (f *fakePending) ListPendingRuns(context.Context) ([]*tester.Run, error) { return f.runs, nil }