From b6879bc513de250f37159e78e18106a3ff8e7942 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Fri, 14 Aug 2026 17:55:51 +0000 Subject: [PATCH 1/6] [backend-scheduler] expose pending job depth and fix job-duration bucketing Nothing exposed queue depth. jobs_created_total counts dispatch and jobs_active counts work already handed to a worker, so a submission that fanned out 145 jobs reported 1 -- the other 144 were invisible. jobs_pending{tenant, job_type} reports what is enqueued and not yet dispatched. Depth is also the only signal that can drive scale-up. jobs_active is bounded by the worker count and rises only as capacity rises, so reading it to decide whether to add capacity is circular. This matters most for redaction, which suppresses the tenant's compaction while it runs: an autoscaler driven by compaction backlog watches that backlog vanish exactly when redaction work starts, and scales down mid-run. The gauge resets before each publish. It is keyed by tenant, and a tenant whose queue drains stops appearing in the snapshot rather than reporting zero, so without the reset its last value would persist for the process lifetime and an autoscaler would hold scale forever. job_duration_seconds moves off the client defaults, which stop at 10s and put no boundary between 2.5s and 5s -- where about 60% of redaction jobs land fleet-wide, so p50, p90 and p99 were all interpolations inside one bucket. Now powers of two from 10ms to ~11m. tempodb_cache_store_size_bytes was the only histogram in the tree without native-histogram configuration; it now matches the other 33. --- .chloggen/backend-scheduler-jobs-pending.yaml | 13 ++++ modules/backendscheduler/backendscheduler.go | 1 + modules/backendscheduler/metrics.go | 37 +++++++++-- .../backendscheduler/metrics_pending_test.go | 64 +++++++++++++++++++ modules/backendscheduler/work/interface.go | 3 + .../work/pending_counts_test.go | 63 ++++++++++++++++++ modules/backendscheduler/work/work.go | 25 ++++++++ tempodb/backend/cache/cache.go | 18 ++++-- 8 files changed, 214 insertions(+), 10 deletions(-) create mode 100644 .chloggen/backend-scheduler-jobs-pending.yaml create mode 100644 modules/backendscheduler/metrics_pending_test.go create mode 100644 modules/backendscheduler/work/pending_counts_test.go diff --git a/.chloggen/backend-scheduler-jobs-pending.yaml b/.chloggen/backend-scheduler-jobs-pending.yaml new file mode 100644 index 00000000000..060417a8aad --- /dev/null +++ b/.chloggen/backend-scheduler-jobs-pending.yaml @@ -0,0 +1,13 @@ +change_type: enhancement +component: backend-scheduler +note: add `tempo_backend_scheduler_jobs_pending`, widen the job-duration histogram buckets, and complete native-histogram coverage. +issues: [] +subtext: | + `tempo_backend_scheduler_jobs_pending{tenant, job_type}` reports queue depth: jobs enqueued and not yet dispatched to a worker. Nothing exposed this before, so a submission that fanned out hundreds of jobs showed only the handful already running. `jobs_active` counts work already handed out, so it is bounded by the worker count and cannot indicate that more capacity is needed — only depth can, which makes this the signal to autoscale backend-workers on. + + It matters most for redaction, which suppresses the tenant's compaction while it runs. An autoscaler driven by compaction backlog therefore sees that backlog disappear exactly when redaction work begins, and scales down mid-run. + + The `backend_scheduler_job_duration_seconds` buckets move from the client library defaults to powers of two between 10ms and ~11m. The defaults stop at 10s and place no boundary between 2.5s and 5s, where roughly 60% of redaction jobs land, so every percentile drawn from them interpolated inside a single bucket. Dashboards using `histogram_quantile` are unaffected; queries pinned to specific `le` values need updating. + + `tempodb_cache_store_size_bytes` was the only histogram in the tree without native-histogram configuration; it now matches the other 33. +user: zalegrala diff --git a/modules/backendscheduler/backendscheduler.go b/modules/backendscheduler/backendscheduler.go index b71dd3d9490..602d14a4441 100644 --- a/modules/backendscheduler/backendscheduler.go +++ b/modules/backendscheduler/backendscheduler.go @@ -230,6 +230,7 @@ func (s *BackendScheduler) running(ctx context.Context) error { s.work.Prune(ctx) s.checkPendingRescans(ctx) s.cleanupOrphanedBatches(ctx) + s.recordPendingJobs() case <-backendFlushTicker.C: err = s.flushWorkCacheToBackend(ctx) metricWorkFlushes.Inc() diff --git a/modules/backendscheduler/metrics.go b/modules/backendscheduler/metrics.go index 68c8dbae944..874886db4d1 100644 --- a/modules/backendscheduler/metrics.go +++ b/modules/backendscheduler/metrics.go @@ -30,6 +30,11 @@ var ( Name: "backend_scheduler_jobs_active", Help: "Number of currently active jobs", }, []string{"tenant", "job_type"}) + metricJobsPending = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "tempo", + Name: "backend_scheduler_jobs_pending", + Help: "Number of jobs enqueued and not yet dispatched to a worker, by tenant and type. This is queue depth: jobs_active counts work already handed out, so it is bounded by the worker count and cannot indicate that more capacity is needed.", + }, []string{"tenant", "job_type"}) metricJobsRetry = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "tempo", Name: "backend_scheduler_jobs_retry_total", @@ -70,10 +75,18 @@ var ( NativeHistogramMinResetDuration: 1 * time.Hour, }) metricJobDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "tempo", - Name: "backend_scheduler_job_duration_seconds", - Help: "Duration of of a job in seconds", - Buckets: prometheus.DefBuckets, + Namespace: "tempo", + Name: "backend_scheduler_job_duration_seconds", + Help: "Duration of a job in seconds", + // DefBuckets stops at 10s and puts nothing between 2.5s and 5s, where roughly 60% of + // redaction jobs land — every percentile drawn from it interpolates inside one bucket, so + // p50, p90 and p99 move together and say nothing. It also cannot represent a job slower + // than 10s at all, and a redaction over a large block runs for minutes. + // + // Powers of two from 10ms to ~11m: fast retention jobs stay resolved, the redaction mass + // splits across the 2.56s and 5.12s boundaries, and long jobs land in a real bucket + // instead of +Inf. + Buckets: prometheus.ExponentialBuckets(0.01, 2, 17), NativeHistogramBucketFactor: 1.1, NativeHistogramMaxBucketNumber: 100, NativeHistogramMinResetDuration: 1 * time.Hour, @@ -101,3 +114,19 @@ func redactionModeLabel(mode tempopb.RedactionMode) string { } return "apply" } + +// recordPendingJobs publishes the queue depth per tenant and job type. +// +// Reset first: this is a gauge keyed by tenant, and a tenant whose queue drains stops appearing in +// the snapshot entirely. Without the reset its last non-zero value would persist forever, so a +// finished redaction would look permanently backlogged — and an autoscaler reading it would never +// scale back down. +func (s *BackendScheduler) recordPendingJobs() { + metricJobsPending.Reset() + + for tenant, byType := range s.work.PendingJobCounts() { + for jobType, n := range byType { + metricJobsPending.WithLabelValues(tenant, jobType.String()).Set(float64(n)) + } + } +} diff --git a/modules/backendscheduler/metrics_pending_test.go b/modules/backendscheduler/metrics_pending_test.go new file mode 100644 index 00000000000..14ea4924396 --- /dev/null +++ b/modules/backendscheduler/metrics_pending_test.go @@ -0,0 +1,64 @@ +package backendscheduler + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" + + "github.com/grafana/tempo/modules/backendscheduler/work" + "github.com/grafana/tempo/pkg/tempopb" +) + +// TestRecordPendingJobs covers publishing queue depth, and in particular that a drained queue stops +// being reported. +// +// The gauge is keyed by tenant, and a tenant whose queue empties simply stops appearing in the +// snapshot — it never reports zero. Without the Reset, its last non-zero value would persist for the +// process lifetime: a finished redaction would look permanently backlogged, and an autoscaler +// triggering on this would hold the scale-up forever. +func TestRecordPendingJobs(t *testing.T) { + s := &BackendScheduler{work: work.New(work.Config{})} + + redaction := func(id, tenant, block string) *work.Job { + return &work.Job{ + ID: id, + Type: tempopb.JobType_JOB_TYPE_REDACTION, + JobDetail: tempopb.JobDetail{ + Tenant: tenant, + Redaction: &tempopb.RedactionDetail{BlockId: block}, + }, + } + } + + require.NoError(t, s.work.AddPendingJobs([]*work.Job{ + redaction("r1", "tenant-a", "block-1"), + redaction("r2", "tenant-a", "block-2"), + redaction("r3", "tenant-b", "block-1"), + { + ID: "c1", + Type: tempopb.JobType_JOB_TYPE_COMPACTION, + JobDetail: tempopb.JobDetail{ + Tenant: "tenant-a", + Compaction: &tempopb.CompactionDetail{Input: []string{"block-9"}}, + }, + }, + })) + + s.recordPendingJobs() + require.Equal(t, 2.0, testutil.ToFloat64(metricJobsPending.WithLabelValues("tenant-a", "JOB_TYPE_REDACTION"))) + require.Equal(t, 1.0, testutil.ToFloat64(metricJobsPending.WithLabelValues("tenant-b", "JOB_TYPE_REDACTION"))) + require.Equal(t, 1.0, testutil.ToFloat64(metricJobsPending.WithLabelValues("tenant-a", "JOB_TYPE_COMPACTION"))) + require.Equal(t, 3, testutil.CollectAndCount(metricJobsPending), "one series per tenant and type with queued work") + + // Drain every redaction job. NextPendingJob is type-scoped, so the compaction job stays queued + // and tenant-a keeps exactly one series while tenant-b loses its only one. + for s.work.NextPendingJob(tempopb.JobType_JOB_TYPE_REDACTION) != nil { //nolint:revive // drain + } + + s.recordPendingJobs() + require.Equal(t, 1, testutil.CollectAndCount(metricJobsPending), + "drained queues must lose their series rather than keep their last value") + require.Equal(t, 1.0, testutil.ToFloat64(metricJobsPending.WithLabelValues("tenant-a", "JOB_TYPE_COMPACTION")), + "the surviving series is the compaction queue that was never drained") +} diff --git a/modules/backendscheduler/work/interface.go b/modules/backendscheduler/work/interface.go index ae616cf9306..48e93411565 100644 --- a/modules/backendscheduler/work/interface.go +++ b/modules/backendscheduler/work/interface.go @@ -47,6 +47,9 @@ type Interface interface { // Acquires pendingMtx exactly once and returns a snapshot. BusyBlocksForTenant(tenantID string) map[string]string + // PendingJobCounts returns enqueued, not-yet-dispatched job counts per tenant and type. + PendingJobCounts() map[string]map[tempopb.JobType]int + // TenantPending returns true when an exclusive tenant operation exists whose // full scope is not yet reflected in the job queue — i.e. an apply-mode redaction batch // (just created or in its rescan-wait window). Gates compaction and retention. A dry-run diff --git a/modules/backendscheduler/work/pending_counts_test.go b/modules/backendscheduler/work/pending_counts_test.go new file mode 100644 index 00000000000..60fd43638ab --- /dev/null +++ b/modules/backendscheduler/work/pending_counts_test.go @@ -0,0 +1,63 @@ +package work + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/tempo/pkg/tempopb" +) + +// TestPendingJobCounts covers the queue-depth snapshot that backs the jobs_pending gauge. +// +// Depth is the only signal that can drive scale-up. jobs_active counts work already handed to a +// worker, so it is bounded by the worker count and rises only as capacity rises — reading it to +// decide whether to add capacity is circular. If this undercounts, an autoscaler stops adding +// workers while work is still queued. +func TestPendingJobCounts(t *testing.T) { + w := New(Config{}).(*Work) + + require.Empty(t, w.PendingJobCounts(), "no jobs means no series at all, not zero-valued ones") + + require.NoError(t, w.AddPendingJobs([]*Job{ + createRedactionJob("r1", "tenant-a", "block-1"), + createRedactionJob("r2", "tenant-a", "block-2"), + createRedactionJob("r3", "tenant-b", "block-1"), + createCompactionJob("c1", "tenant-a", []string{"block-9"}), + })) + + counts := w.PendingJobCounts() + require.Equal(t, 2, counts["tenant-a"][tempopb.JobType_JOB_TYPE_REDACTION]) + require.Equal(t, 1, counts["tenant-a"][tempopb.JobType_JOB_TYPE_COMPACTION]) + require.Equal(t, 1, counts["tenant-b"][tempopb.JobType_JOB_TYPE_REDACTION]) + + // Dispatching moves a job out of the queue: it is active now, not pending. Counting it in both + // would hold an autoscaler up after the queue had actually drained. + require.NotNil(t, w.NextPendingJob(tempopb.JobType_JOB_TYPE_REDACTION)) + + counts = w.PendingJobCounts() + remaining := counts["tenant-a"][tempopb.JobType_JOB_TYPE_REDACTION] + counts["tenant-b"][tempopb.JobType_JOB_TYPE_REDACTION] + require.Equal(t, 2, remaining, "a dispatched job must leave the pending depth") + + // A drained type disappears rather than reporting zero, which is what lets the gauge's Reset + // clear the series instead of leaving a stale non-zero value an autoscaler would keep reading. + for w.NextPendingJob(tempopb.JobType_JOB_TYPE_REDACTION) != nil { //nolint:revive // drain + } + + counts = w.PendingJobCounts() + for tenant, byType := range counts { + _, ok := byType[tempopb.JobType_JOB_TYPE_REDACTION] + require.False(t, ok, "a drained redaction queue must not appear for tenant %s", tenant) + } + require.Equal(t, 1, counts["tenant-a"][tempopb.JobType_JOB_TYPE_COMPACTION], "other job types are unaffected") + + // The dequeue path deletes a queue as it empties, so an empty slice only arises from another + // path (a rebuild, or a future removal). Reported as-is it would publish a zero-valued series + // that never clears, which is the same stale-signal problem the gauge's Reset exists to avoid. + w.pendingMtx.Lock() + w.pendingByTenant["tenant-c"] = map[tempopb.JobType][]string{tempopb.JobType_JOB_TYPE_RETENTION: {}} + w.pendingMtx.Unlock() + + _, ok := w.PendingJobCounts()["tenant-c"] + require.False(t, ok, "an empty queue must produce no series at all") +} diff --git a/modules/backendscheduler/work/work.go b/modules/backendscheduler/work/work.go index 3855a103f98..27ee3943838 100644 --- a/modules/backendscheduler/work/work.go +++ b/modules/backendscheduler/work/work.go @@ -924,6 +924,31 @@ func (w *Work) IsBlockBusy(tenantID, blockID string) bool { // BusyBlocksForTenant returns a map of blockID -> jobID for every block // currently referenced by a pending, registered, or active job for the tenant. // Acquires pendingMtx exactly once and returns a snapshot. +// PendingJobCounts returns the number of enqueued, not-yet-dispatched jobs per tenant and type. +// +// This is queue DEPTH, which no other metric carries: jobs_active counts work already handed to a +// worker, so it is bounded by the worker count and can never signal that more capacity is needed. +// Only the pending depth can. +func (w *Work) PendingJobCounts() map[string]map[tempopb.JobType]int { + w.pendingMtx.Lock() + defer w.pendingMtx.Unlock() + + counts := make(map[string]map[tempopb.JobType]int, len(w.pendingByTenant)) + for tenant, byType := range w.pendingByTenant { + for jobType, queue := range byType { + if len(queue) == 0 { + continue + } + if counts[tenant] == nil { + counts[tenant] = make(map[tempopb.JobType]int, len(byType)) + } + counts[tenant][jobType] = len(queue) + } + } + + return counts +} + func (w *Work) BusyBlocksForTenant(tenantID string) map[string]string { result := make(map[string]string) w.pendingMtx.Lock() diff --git a/tempodb/backend/cache/cache.go b/tempodb/backend/cache/cache.go index 96314c58ca1..eca48cca610 100644 --- a/tempodb/backend/cache/cache.go +++ b/tempodb/backend/cache/cache.go @@ -20,24 +20,30 @@ import ( "github.com/grafana/tempo/tempodb/backend" ) +// metricsNamespace is the Prometheus namespace shared by this package's metrics. +const metricsNamespace = "tempodb" + // cacheStoreSizeBytes records the byte size of every item written to a tempodb backend cache, labelled by role. var cacheStoreSizeBytes = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "tempodb", - Name: "cache_store_size_bytes", - Help: "Distribution of item sizes written to tempodb backend caches, by role.", - Buckets: prometheus.ExponentialBuckets(512, 2, 15), // 512 B, 1 KiB, ..., 8 MiB + Namespace: metricsNamespace, + Name: "cache_store_size_bytes", + Help: "Distribution of item sizes written to tempodb backend caches, by role.", + Buckets: prometheus.ExponentialBuckets(512, 2, 15), // 512 B, 1 KiB, ..., 8 MiB + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"role"}) // cacheRequests counts cache lookups by role and outcome (hit / miss). var cacheRequests = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "tempodb", + Namespace: metricsNamespace, Name: "cache_requests_total", Help: "Cache lookup outcome by role.", }, []string{"role", "outcome"}) // cacheRequestBytes counts bytes served on hit and bytes fetched from backend on miss, by role. var cacheRequestBytes = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "tempodb", + Namespace: metricsNamespace, Name: "cache_request_bytes_total", Help: "Bytes served by cache (hit) or fetched on miss, by role.", }, []string{"role", "outcome"}) From abae7b7f28410262e1e3ed7ddc87d1e20ec50eab Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Fri, 14 Aug 2026 19:47:42 +0000 Subject: [PATCH 2/6] [mixin] backend work: show redaction progress and query native histograms Active Redaction Jobs reads the same on the first block as on the last -- it counts what a worker is running now, which is bounded by the worker count. Pending Redaction Jobs plots the queue depth per tenant, so the dashboard answers how much is left rather than only whether something is running. Both job-duration panels gain native-histogram queries next to the classic-bucket ones. The classic buckets are coarse where redaction actually lands, and the native series resolves it properly wherever native histograms are retained; keeping both means the panel renders on stacks that have either. Compiled output regenerated with make tempo-mixin. --- ...kendwork-dashboard-redaction-progress.yaml | 9 + .../dashboards/tempo-backendwork.json | 203 +++++++++++++++--- .../dashboards/tempo-backendwork.json | 201 ++++++++++++++--- 3 files changed, 357 insertions(+), 56 deletions(-) create mode 100644 .chloggen/backendwork-dashboard-redaction-progress.yaml diff --git a/.chloggen/backendwork-dashboard-redaction-progress.yaml b/.chloggen/backendwork-dashboard-redaction-progress.yaml new file mode 100644 index 00000000000..0ba6f15fd26 --- /dev/null +++ b/.chloggen/backendwork-dashboard-redaction-progress.yaml @@ -0,0 +1,9 @@ +change_type: enhancement +component: operations +note: "tempo-mixin: add a pending-redaction-jobs panel and native-histogram queries to the Backend Work dashboard." +issues: [] +subtext: | + "Pending Redaction Jobs" plots `tempo_backend_scheduler_jobs_pending` per tenant, so the dashboard shows how much of a redaction is left. "Active Redaction Jobs" only shows what a worker is running right now, which is bounded by the worker count and reads the same on the first block as on the last. + + Both job-duration panels now carry native-histogram queries alongside the classic-bucket ones. The native series renders wherever native histograms are retained and gives full resolution; the classic queries stay for stacks that keep only buckets. +user: zalegrala diff --git a/operations/tempo-mixin-compiled/dashboards/tempo-backendwork.json b/operations/tempo-mixin-compiled/dashboards/tempo-backendwork.json index a9b79d7d203..d311195456d 100644 --- a/operations/tempo-mixin-compiled/dashboards/tempo-backendwork.json +++ b/operations/tempo-mixin-compiled/dashboards/tempo-backendwork.json @@ -1478,6 +1478,7 @@ "type": "prometheus", "uid": "${metrics}" }, + "description": "Classic-bucket and native-histogram queries are both present. The native series gives full resolution where the classic buckets are coarse; it renders once native histograms are retained for this metric.", "fieldConfig": { "defaults": { "color": { @@ -1582,6 +1583,28 @@ "legendFormat": "p50 {{job_type}}", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "p99 (native)", + "range": true, + "refId": "N99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "p50 (native)", + "range": true, + "refId": "N50" } ], "title": "Job Duration", @@ -1814,6 +1837,7 @@ "type": "prometheus", "uid": "${metrics}" }, + "description": "Jobs enqueued and not yet dispatched to a worker. Active shows what is running now (bounded by worker count); this shows how much is left, so it is the progress indicator and the signal to autoscale workers on.", "fieldConfig": { "defaults": { "color": { @@ -1878,6 +1902,108 @@ "x": 0, "y": 28 }, + "id": 44, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(tempo_backend_scheduler_jobs_pending{cluster=~\"$cluster\", namespace=~\"$namespace\", job_type=\"JOB_TYPE_REDACTION\"}) by (tenant)", + "legendFormat": "{{tenant}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending Redaction Jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 28 + }, "id": 36, "options": { "legend": { @@ -1977,7 +2103,7 @@ "gridPos": { "h": 6, "w": 8, - "x": 8, + "x": 16, "y": 28 }, "id": 37, @@ -2079,8 +2205,8 @@ "gridPos": { "h": 6, "w": 8, - "x": 16, - "y": 28 + "x": 0, + "y": 34 }, "id": 38, "options": { @@ -2180,8 +2306,8 @@ }, "gridPos": { "h": 6, - "w": 6, - "x": 0, + "w": 8, + "x": 8, "y": 34 }, "id": 39, @@ -2282,8 +2408,8 @@ }, "gridPos": { "h": 6, - "w": 6, - "x": 6, + "w": 8, + "x": 16, "y": 34 }, "id": 40, @@ -2324,6 +2450,7 @@ "type": "prometheus", "uid": "${metrics}" }, + "description": "Classic-bucket and native-histogram queries are both present. The native series gives full resolution where the classic buckets are coarse; it renders once native histograms are retained for this metric.", "fieldConfig": { "defaults": { "color": { @@ -2386,8 +2513,8 @@ "gridPos": { "h": 6, "w": 12, - "x": 12, - "y": 34 + "x": 0, + "y": 40 }, "id": 41, "options": { @@ -2439,6 +2566,28 @@ "legendFormat": "p50", "range": true, "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", job_type=\"JOB_TYPE_REDACTION\"}[$__rate_interval])))", + "legendFormat": "p99 (native)", + "range": true, + "refId": "N99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", job_type=\"JOB_TYPE_REDACTION\"}[$__rate_interval])))", + "legendFormat": "p50 (native)", + "range": true, + "refId": "N50" } ], "title": "Redaction Job Duration", @@ -2511,7 +2660,7 @@ "gridPos": { "h": 6, "w": 12, - "x": 0, + "x": 12, "y": 40 }, "id": 42, @@ -2614,8 +2763,8 @@ "gridPos": { "h": 6, "w": 12, - "x": 12, - "y": 40 + "x": 0, + "y": 46 }, "id": 43, "options": { @@ -2656,7 +2805,7 @@ "h": 1, "w": 24, "x": 0, - "y": 46 + "y": 52 }, "id": 8, "panels": [ @@ -2732,7 +2881,7 @@ "h": 6, "w": 5, "x": 0, - "y": 47 + "y": 53 }, "id": 9, "options": { @@ -2836,7 +2985,7 @@ "h": 6, "w": 4, "x": 5, - "y": 47 + "y": 53 }, "id": 11, "options": { @@ -2939,7 +3088,7 @@ "h": 6, "w": 5, "x": 9, - "y": 47 + "y": 53 }, "id": 10, "options": { @@ -3042,7 +3191,7 @@ "h": 6, "w": 4, "x": 14, - "y": 47 + "y": 53 }, "id": 13, "options": { @@ -3146,7 +3295,7 @@ "h": 6, "w": 4, "x": 18, - "y": 47 + "y": 53 }, "id": 12, "options": { @@ -3249,7 +3398,7 @@ "h": 6, "w": 2, "x": 22, - "y": 47 + "y": 53 }, "id": 29, "options": { @@ -3290,7 +3439,7 @@ "h": 1, "w": 24, "x": 0, - "y": 53 + "y": 59 }, "id": 25, "panels": [ @@ -3411,7 +3560,7 @@ "h": 8, "w": 6, "x": 12, - "y": 54 + "y": 60 }, "id": 7, "options": { @@ -3583,7 +3732,7 @@ "h": 8, "w": 6, "x": 18, - "y": 54 + "y": 60 }, "id": 6, "options": { @@ -3662,7 +3811,7 @@ "h": 1, "w": 24, "x": 0, - "y": 62 + "y": 68 }, "id": 1, "panels": [ @@ -3738,7 +3887,7 @@ "h": 8, "w": 6, "x": 0, - "y": 63 + "y": 69 }, "id": 2, "options": { @@ -3921,7 +4070,7 @@ "h": 8, "w": 6, "x": 6, - "y": 63 + "y": 69 }, "id": 3, "options": { @@ -4082,7 +4231,7 @@ "h": 8, "w": 6, "x": 12, - "y": 63 + "y": 69 }, "id": 4, "options": { @@ -4254,7 +4403,7 @@ "h": 8, "w": 6, "x": 18, - "y": 63 + "y": 69 }, "id": 5, "options": { diff --git a/operations/tempo-mixin/dashboards/tempo-backendwork.json b/operations/tempo-mixin/dashboards/tempo-backendwork.json index bc1cd2914b2..93af8e8b911 100644 --- a/operations/tempo-mixin/dashboards/tempo-backendwork.json +++ b/operations/tempo-mixin/dashboards/tempo-backendwork.json @@ -1496,10 +1496,33 @@ "legendFormat": "p50 {{job_type}}", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "p99 (native)", + "range": true, + "refId": "N99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "p50 (native)", + "range": true, + "refId": "N50" } ], "title": "Job Duration", - "type": "timeseries" + "type": "timeseries", + "description": "Classic-bucket and native-histogram queries are both present. The native series gives full resolution where the classic buckets are coarse; it renders once native histograms are retained for this metric." }, { "datasource": { @@ -1774,6 +1797,103 @@ "x": 0, "y": 28 }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(tempo_backend_scheduler_jobs_pending{cluster=~\"$cluster\", namespace=~\"$namespace\", job_type=\"JOB_TYPE_REDACTION\"}) by (tenant)", + "legendFormat": "{{tenant}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending Redaction Jobs", + "type": "timeseries", + "description": "Jobs enqueued and not yet dispatched to a worker. Active shows what is running now (bounded by worker count); this shows how much is left, so it is the progress indicator and the signal to autoscale workers on." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 28 + }, "id": 36, "options": { "legend": { @@ -1867,7 +1987,7 @@ "gridPos": { "h": 6, "w": 8, - "x": 8, + "x": 16, "y": 28 }, "id": 37, @@ -1963,8 +2083,8 @@ "gridPos": { "h": 6, "w": 8, - "x": 16, - "y": 28 + "x": 0, + "y": 34 }, "id": 38, "options": { @@ -2058,8 +2178,8 @@ }, "gridPos": { "h": 6, - "w": 6, - "x": 0, + "w": 8, + "x": 8, "y": 34 }, "id": 39, @@ -2154,8 +2274,8 @@ }, "gridPos": { "h": 6, - "w": 6, - "x": 6, + "w": 8, + "x": 16, "y": 34 }, "id": 40, @@ -2252,8 +2372,8 @@ "gridPos": { "h": 6, "w": 12, - "x": 12, - "y": 34 + "x": 0, + "y": 40 }, "id": 41, "options": { @@ -2303,10 +2423,33 @@ "legendFormat": "p50", "range": true, "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", job_type=\"JOB_TYPE_REDACTION\"}[$__rate_interval])))", + "legendFormat": "p99 (native)", + "range": true, + "refId": "N99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", job_type=\"JOB_TYPE_REDACTION\"}[$__rate_interval])))", + "legendFormat": "p50 (native)", + "range": true, + "refId": "N50" } ], "title": "Redaction Job Duration", - "type": "timeseries" + "type": "timeseries", + "description": "Classic-bucket and native-histogram queries are both present. The native series gives full resolution where the classic buckets are coarse; it renders once native histograms are retained for this metric." }, { "datasource": { @@ -2370,7 +2513,7 @@ "gridPos": { "h": 6, "w": 12, - "x": 0, + "x": 12, "y": 40 }, "id": 42, @@ -2467,8 +2610,8 @@ "gridPos": { "h": 6, "w": 12, - "x": 12, - "y": 40 + "x": 0, + "y": 46 }, "id": 43, "options": { @@ -2508,7 +2651,7 @@ "h": 1, "w": 24, "x": 0, - "y": 46 + "y": 52 }, "id": 8, "panels": [], @@ -2578,7 +2721,7 @@ "h": 6, "w": 5, "x": 0, - "y": 47 + "y": 53 }, "id": 9, "options": { @@ -2676,7 +2819,7 @@ "h": 6, "w": 4, "x": 5, - "y": 47 + "y": 53 }, "id": 11, "options": { @@ -2773,7 +2916,7 @@ "h": 6, "w": 5, "x": 9, - "y": 47 + "y": 53 }, "id": 10, "options": { @@ -2870,7 +3013,7 @@ "h": 6, "w": 4, "x": 14, - "y": 47 + "y": 53 }, "id": 13, "options": { @@ -2968,7 +3111,7 @@ "h": 6, "w": 4, "x": 18, - "y": 47 + "y": 53 }, "id": 12, "options": { @@ -3065,7 +3208,7 @@ "h": 6, "w": 2, "x": 22, - "y": 47 + "y": 53 }, "id": 29, "options": { @@ -3104,7 +3247,7 @@ "h": 1, "w": 24, "x": 0, - "y": 53 + "y": 59 }, "id": 25, "panels": [], @@ -3219,7 +3362,7 @@ "h": 8, "w": 6, "x": 12, - "y": 54 + "y": 60 }, "id": 7, "options": { @@ -3385,7 +3528,7 @@ "h": 8, "w": 6, "x": 18, - "y": 54 + "y": 60 }, "id": 6, "options": { @@ -3462,7 +3605,7 @@ "h": 1, "w": 24, "x": 0, - "y": 62 + "y": 68 }, "id": 1, "panels": [], @@ -3532,7 +3675,7 @@ "h": 8, "w": 6, "x": 0, - "y": 63 + "y": 69 }, "id": 2, "options": { @@ -3709,7 +3852,7 @@ "h": 8, "w": 6, "x": 6, - "y": 63 + "y": 69 }, "id": 3, "options": { @@ -3864,7 +4007,7 @@ "h": 8, "w": 6, "x": 12, - "y": 63 + "y": 69 }, "id": 4, "options": { @@ -4030,7 +4173,7 @@ "h": 8, "w": 6, "x": 18, - "y": 63 + "y": 69 }, "id": 5, "options": { From 263ae10b65514f1e363fd13c60c19dd9b7f029eb Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Fri, 14 Aug 2026 21:54:34 +0000 Subject: [PATCH 3/6] [docs] compaction: document the pending-jobs metric The metrics table listed jobs_active but had nothing for queue depth. Notes the distinction that matters for autoscaling: active is bounded by the worker count, pending is not. --- .../tempo/reference-tempo-architecture/components/compaction.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sources/tempo/reference-tempo-architecture/components/compaction.md b/docs/sources/tempo/reference-tempo-architecture/components/compaction.md index aa29d2a8099..85a06591042 100644 --- a/docs/sources/tempo/reference-tempo-architecture/components/compaction.md +++ b/docs/sources/tempo/reference-tempo-architecture/components/compaction.md @@ -107,6 +107,7 @@ This endpoint is useful for diagnosing stalled jobs, verifying that workers are | `tempo_backend_scheduler_jobs_completed_total` | Jobs completed successfully | | `tempo_backend_scheduler_jobs_failed_total` | Jobs that failed | | `tempo_backend_scheduler_jobs_active` | Jobs currently assigned to a worker | +| `tempo_backend_scheduler_jobs_pending` | Jobs enqueued and not yet assigned to a worker. Unlike `jobs_active`, which is bounded by the number of workers, this is queue depth and indicates whether more worker capacity is needed | | `tempo_backend_scheduler_job_duration_seconds` | Job execution duration histogram | | `tempodb_blocklist_length` | Number of live blocks per tenant; high values indicate compaction is falling behind | | `tempodb_compaction_outstanding_blocks` | Outstanding blocks awaiting compaction per tenant; the primary autoscaling signal | From de6c74e2f10f9cf4091bbc234e15f728ea2bff14 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 17 Aug 2026 15:08:02 +0000 Subject: [PATCH 4/6] [backend-scheduler] restore the BusyBlocksForTenant doc and align native histogram series Adding PendingJobCounts above BusyBlocksForTenant put the new function between that function and its doc comment. The two comment blocks then ran together with no blank line, so GoDoc attached BusyBlocksForTenant's description to PendingJobCounts and left BusyBlocksForTenant -- exported, and part of work.Interface -- undocumented. Each comment sits above its own function again. On the Job Duration panel the classic queries group by (le, job_type), producing one series per type, while the native queries aggregated across all types into a single line. Read together that invites treating the native line as another rendering of one classic series. Native now groups by (job_type) with the type in the legend. Redaction Job Duration plots p99/p90/p50 classic but only had p99/p50 native; added p90 so the two sets pair up. Compiled mixin regenerated. --- modules/backendscheduler/work/work.go | 6 +++--- .../dashboards/tempo-backendwork.json | 19 +++++++++++++++---- .../dashboards/tempo-backendwork.json | 19 +++++++++++++++---- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/modules/backendscheduler/work/work.go b/modules/backendscheduler/work/work.go index 27ee3943838..d7a5d44acd7 100644 --- a/modules/backendscheduler/work/work.go +++ b/modules/backendscheduler/work/work.go @@ -921,9 +921,6 @@ func (w *Work) IsBlockBusy(tenantID, blockID string) bool { return inPending || inRunning } -// BusyBlocksForTenant returns a map of blockID -> jobID for every block -// currently referenced by a pending, registered, or active job for the tenant. -// Acquires pendingMtx exactly once and returns a snapshot. // PendingJobCounts returns the number of enqueued, not-yet-dispatched jobs per tenant and type. // // This is queue DEPTH, which no other metric carries: jobs_active counts work already handed to a @@ -949,6 +946,9 @@ func (w *Work) PendingJobCounts() map[string]map[tempopb.JobType]int { return counts } +// BusyBlocksForTenant returns a map of blockID -> jobID for every block +// currently referenced by a pending, registered, or active job for the tenant. +// Acquires pendingMtx exactly once and returns a snapshot. func (w *Work) BusyBlocksForTenant(tenantID string) map[string]string { result := make(map[string]string) w.pendingMtx.Lock() diff --git a/operations/tempo-mixin-compiled/dashboards/tempo-backendwork.json b/operations/tempo-mixin-compiled/dashboards/tempo-backendwork.json index d311195456d..c7ebb35f948 100644 --- a/operations/tempo-mixin-compiled/dashboards/tempo-backendwork.json +++ b/operations/tempo-mixin-compiled/dashboards/tempo-backendwork.json @@ -1590,8 +1590,8 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])))", - "legendFormat": "p99 (native)", + "expr": "histogram_quantile(0.99, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (job_type))", + "legendFormat": "p99 {{job_type}} (native)", "range": true, "refId": "N99" }, @@ -1601,8 +1601,8 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])))", - "legendFormat": "p50 (native)", + "expr": "histogram_quantile(0.5, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (job_type))", + "legendFormat": "p50 {{job_type}} (native)", "range": true, "refId": "N50" } @@ -2578,6 +2578,17 @@ "range": true, "refId": "N99" }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", job_type=\"JOB_TYPE_REDACTION\"}[$__rate_interval])))", + "legendFormat": "p90 (native)", + "range": true, + "refId": "N90" + }, { "datasource": { "type": "prometheus", diff --git a/operations/tempo-mixin/dashboards/tempo-backendwork.json b/operations/tempo-mixin/dashboards/tempo-backendwork.json index 93af8e8b911..490f6a2d22b 100644 --- a/operations/tempo-mixin/dashboards/tempo-backendwork.json +++ b/operations/tempo-mixin/dashboards/tempo-backendwork.json @@ -1503,8 +1503,8 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])))", - "legendFormat": "p99 (native)", + "expr": "histogram_quantile(0.99, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (job_type))", + "legendFormat": "p99 {{job_type}} (native)", "range": true, "refId": "N99" }, @@ -1514,8 +1514,8 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])))", - "legendFormat": "p50 (native)", + "expr": "histogram_quantile(0.5, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (job_type))", + "legendFormat": "p50 {{job_type}} (native)", "range": true, "refId": "N50" } @@ -2435,6 +2435,17 @@ "range": true, "refId": "N99" }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(tempo_backend_scheduler_job_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", job_type=\"JOB_TYPE_REDACTION\"}[$__rate_interval])))", + "legendFormat": "p90 (native)", + "range": true, + "refId": "N90" + }, { "datasource": { "type": "prometheus", From 5b0732aacbe047d9f49a61a070c8037fa9ea55e0 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 17 Aug 2026 16:26:30 +0000 Subject: [PATCH 5/6] [docs] correct the pending-jobs rationale The changelog claimed an autoscaler on compaction backlog sees that backlog disappear when redaction starts, and scales down mid-run. That is not true: measureTenants uses newBlockSelectorForMeasurement precisely so the outstanding-blocks metric keeps reporting real work while a tenant's compaction is gated, and TestCompactionProvider_MeasureTenantsIgnoresTenantPending has asserted it since #6992. The real gap is narrower. Compaction is gated for the tenant, so the backlog it reports cannot be dispatched -- an autoscaler on that signal holds scale for work it will not be given. Redaction queue depth is the direct input, and the only signal that tracks progress. --- .chloggen/backend-scheduler-jobs-pending.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/backend-scheduler-jobs-pending.yaml b/.chloggen/backend-scheduler-jobs-pending.yaml index 060417a8aad..46a29b8b6bd 100644 --- a/.chloggen/backend-scheduler-jobs-pending.yaml +++ b/.chloggen/backend-scheduler-jobs-pending.yaml @@ -5,7 +5,7 @@ issues: [] subtext: | `tempo_backend_scheduler_jobs_pending{tenant, job_type}` reports queue depth: jobs enqueued and not yet dispatched to a worker. Nothing exposed this before, so a submission that fanned out hundreds of jobs showed only the handful already running. `jobs_active` counts work already handed out, so it is bounded by the worker count and cannot indicate that more capacity is needed — only depth can, which makes this the signal to autoscale backend-workers on. - It matters most for redaction, which suppresses the tenant's compaction while it runs. An autoscaler driven by compaction backlog therefore sees that backlog disappear exactly when redaction work begins, and scales down mid-run. + It matters most for redaction. Compaction is gated for a tenant being redacted, so no compaction jobs are created for it, yet the outstanding-blocks metric deliberately keeps reporting that backlog (see `newBlockSelectorForMeasurement`) — an autoscaler on that signal is holding scale for work it cannot dispatch. Redaction depth is the direct input, and the only one that tracks progress. The `backend_scheduler_job_duration_seconds` buckets move from the client library defaults to powers of two between 10ms and ~11m. The defaults stop at 10s and place no boundary between 2.5s and 5s, where roughly 60% of redaction jobs land, so every percentile drawn from them interpolated inside a single bucket. Dashboards using `histogram_quantile` are unaffected; queries pinned to specific `le` values need updating. From b777bd3d883f8fb25002ee8113bc7b023efbe180 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Tue, 18 Aug 2026 17:52:30 +0000 Subject: [PATCH 6/6] [backend-scheduler] publish jobs_pending without a reset window Reset() cleared the whole vector before repopulating it, so a scrape landing inside that window saw series missing and read the total lower than it is -- the spurious scale-down this metric exists to prevent. Current values are now set first and only drained label sets are deleted, so a mid-publish scrape can catch a slightly stale value but never a low one. Errors bias toward holding scale. Drained queues are still removed: a tenant whose queue empties stops appearing in the snapshot rather than reporting zero, so its last value would otherwise persist for the process lifetime. Also publish once before entering the maintenance loop. On the tick alone the metric was absent for a full MaintenanceInterval after start, so anything reading it right after a restart saw nothing rather than the queue that survived the restart. Both raised by Copilot as suppressed comments. --- modules/backendscheduler/backendscheduler.go | 9 ++++++ modules/backendscheduler/metrics.go | 28 +++++++++++++++---- .../backendscheduler/metrics_pending_test.go | 18 +++++++++--- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/modules/backendscheduler/backendscheduler.go b/modules/backendscheduler/backendscheduler.go index 602d14a4441..2754060cb11 100644 --- a/modules/backendscheduler/backendscheduler.go +++ b/modules/backendscheduler/backendscheduler.go @@ -56,6 +56,10 @@ type BackendScheduler struct { } mergedJobs chan *work.Job + + // publishedPendingLabels records the jobs_pending label sets published last tick, so a drained + // queue can be deleted without resetting the whole vector. Touched only from the maintenance loop. + publishedPendingLabels map[[2]string]struct{} } // ListJobs returns all jobs in the work cache @@ -220,6 +224,11 @@ func (s *BackendScheduler) running(ctx context.Context) error { backendFlushTicker := time.NewTicker(s.cfg.BackendFlushInterval) defer backendFlushTicker.Stop() + // Publish once up front: on the tick alone the metric would be absent for a full + // MaintenanceInterval after start, so a dashboard or autoscaler reading it right after a restart + // would see nothing rather than the queue that survived the restart. + s.recordPendingJobs() + var err error for { diff --git a/modules/backendscheduler/metrics.go b/modules/backendscheduler/metrics.go index 874886db4d1..484da944693 100644 --- a/modules/backendscheduler/metrics.go +++ b/modules/backendscheduler/metrics.go @@ -117,16 +117,32 @@ func redactionModeLabel(mode tempopb.RedactionMode) string { // recordPendingJobs publishes the queue depth per tenant and job type. // -// Reset first: this is a gauge keyed by tenant, and a tenant whose queue drains stops appearing in -// the snapshot entirely. Without the reset its last non-zero value would persist forever, so a -// finished redaction would look permanently backlogged — and an autoscaler reading it would never -// scale back down. +// Current values are set BEFORE drained series are removed, rather than resetting the vector first. +// A scrape landing inside a Reset would see series missing and read the total lower than it is — +// which is precisely the spurious scale-down this metric exists to prevent. Setting first means a +// scrape can only ever catch a slightly stale value, so the error biases toward holding scale. +// +// Drained queues must still be removed: a tenant whose queue empties stops appearing in the +// snapshot rather than reporting zero, so its last non-zero value would otherwise persist for the +// process lifetime and an autoscaler would hold scale forever. +// +// Called only from the maintenance loop, so the label bookkeeping needs no lock. func (s *BackendScheduler) recordPendingJobs() { - metricJobsPending.Reset() + current := make(map[[2]string]struct{}, len(s.publishedPendingLabels)) for tenant, byType := range s.work.PendingJobCounts() { for jobType, n := range byType { - metricJobsPending.WithLabelValues(tenant, jobType.String()).Set(float64(n)) + labels := [2]string{tenant, jobType.String()} + current[labels] = struct{}{} + metricJobsPending.WithLabelValues(labels[0], labels[1]).Set(float64(n)) + } + } + + for labels := range s.publishedPendingLabels { + if _, still := current[labels]; !still { + metricJobsPending.DeleteLabelValues(labels[0], labels[1]) } } + + s.publishedPendingLabels = current } diff --git a/modules/backendscheduler/metrics_pending_test.go b/modules/backendscheduler/metrics_pending_test.go index 14ea4924396..fd2fd8df1dd 100644 --- a/modules/backendscheduler/metrics_pending_test.go +++ b/modules/backendscheduler/metrics_pending_test.go @@ -11,13 +11,18 @@ import ( ) // TestRecordPendingJobs covers publishing queue depth, and in particular that a drained queue stops -// being reported. +// being reported while a surviving one keeps its value. // -// The gauge is keyed by tenant, and a tenant whose queue empties simply stops appearing in the -// snapshot — it never reports zero. Without the Reset, its last non-zero value would persist for the +// The gauge is keyed by tenant, and a tenant whose queue empties stops appearing in the snapshot — it +// never reports zero. Without removing those series their last non-zero value would persist for the // process lifetime: a finished redaction would look permanently backlogged, and an autoscaler // triggering on this would hold the scale-up forever. +// +// Removal is deliberately done by deleting drained label sets rather than resetting the vector. A +// scrape landing inside a Reset would observe series missing and read the total lower than it is, +// which is the spurious scale-down this metric exists to prevent. func TestRecordPendingJobs(t *testing.T) { + metricJobsPending.Reset() // isolate from other tests in this package s := &BackendScheduler{work: work.New(work.Config{})} redaction := func(id, tenant, block string) *work.Job { @@ -60,5 +65,10 @@ func TestRecordPendingJobs(t *testing.T) { require.Equal(t, 1, testutil.CollectAndCount(metricJobsPending), "drained queues must lose their series rather than keep their last value") require.Equal(t, 1.0, testutil.ToFloat64(metricJobsPending.WithLabelValues("tenant-a", "JOB_TYPE_COMPACTION")), - "the surviving series is the compaction queue that was never drained") + "the queue that was never drained keeps its value; only drained series are removed") + + // Publishing again with nothing changed must be stable, not oscillate as labels are re-recorded. + s.recordPendingJobs() + require.Equal(t, 1, testutil.CollectAndCount(metricJobsPending), "a repeat publish must be idempotent") + require.Equal(t, 1.0, testutil.ToFloat64(metricJobsPending.WithLabelValues("tenant-a", "JOB_TYPE_COMPACTION"))) }