From b74241bbf79a002e3e773648113d59c9db742480 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Fri, 14 Aug 2026 19:50:24 +0000 Subject: [PATCH 1/3] Add native histogram metrics for backend block sizes at flush and compaction Adds tempo_block_builder_flush_size_bytes (block-builder flush path) and tempodb_compaction_output_block_size_bytes (compaction output), mirroring the existing live-store tempo_live_store_local_flush_size_bytes metric. Closes the observability gap needed to inform max_input_blocks tuning and TCO analysis. --- .../block-builder-flush-size-metric.yaml | 24 ++++++++++++++++++ .../compaction-output-block-size-metric.yaml | 25 +++++++++++++++++++ modules/blockbuilder/tenant_store.go | 11 ++++++++ tempodb/compactor.go | 12 +++++++++ 4 files changed, 72 insertions(+) create mode 100644 .chloggen/block-builder-flush-size-metric.yaml create mode 100644 .chloggen/compaction-output-block-size-metric.yaml diff --git a/.chloggen/block-builder-flush-size-metric.yaml b/.chloggen/block-builder-flush-size-metric.yaml new file mode 100644 index 00000000000..de4abe7465e --- /dev/null +++ b/.chloggen/block-builder-flush-size-metric.yaml @@ -0,0 +1,24 @@ +# Changelog entry. Generate a copy with `make chlog-new`, then fill in the fields. + +# One of: breaking | change | feature | enhancement | bug_fix | security +change_type: enhancement + +# Component or area of the change. Must be one of the components in .chloggen/config.yaml, +# e.g. distributor, querier, query-frontend, storage, operations. +component: block-builder + +# A brief description of the change. Surround with quotes ("") if it must start with a backtick (`). +note: Add `tempo_block_builder_flush_size_bytes` native histogram recording the size of blocks flushed by the block-builder. + +# (Optional) PR number(s), e.g. [7339]. Leave blank to auto-fill at release. See +# .chloggen/README.md. +issues: [] + +# (Optional) Additional lines rendered under the note. Use a pipe (|) for multiline text. +subtext: | + The block-builder flush path had no visibility into the size of the blocks it writes to the + backend. This closes that gap alongside the existing live-store `tempo_live_store_local_flush_size_bytes` + metric, informing `max_input_blocks` tuning and TCO analysis. + +# The GitHub handle (without the leading @) of the change's author. Rendered as "(@handle)". +user: zalegrala diff --git a/.chloggen/compaction-output-block-size-metric.yaml b/.chloggen/compaction-output-block-size-metric.yaml new file mode 100644 index 00000000000..7f9f708c72c --- /dev/null +++ b/.chloggen/compaction-output-block-size-metric.yaml @@ -0,0 +1,25 @@ +# Changelog entry. Generate a copy with `make chlog-new`, then fill in the fields. + +# One of: breaking | change | feature | enhancement | bug_fix | security +change_type: enhancement + +# Component or area of the change. Must be one of the components in .chloggen/config.yaml, +# e.g. distributor, querier, query-frontend, storage, operations. +component: compactor + +# A brief description of the change. Surround with quotes ("") if it must start with a backtick (`). +note: Add `tempodb_compaction_output_block_size_bytes` native histogram recording the size of blocks produced by compaction. + +# (Optional) PR number(s), e.g. [7339]. Leave blank to auto-fill at release. See +# .chloggen/README.md. +issues: [] + +# (Optional) Additional lines rendered under the note. Use a pipe (|) for multiline text. +subtext: | + No existing metric captured the final size of compacted output blocks (the existing + `tempodb_compaction_bytes_written_total` is a cumulative counter fed per row-group flush, not a + per-finished-block size). This gives operators the data needed to tune `max_input_blocks` and + reason about TCO. + +# The GitHub handle (without the leading @) of the change's author. Rendered as "(@handle)". +user: zalegrala diff --git a/modules/blockbuilder/tenant_store.go b/modules/blockbuilder/tenant_store.go index 53b1d20b4ae..30f192a9dda 100644 --- a/modules/blockbuilder/tenant_store.go +++ b/modules/blockbuilder/tenant_store.go @@ -30,6 +30,17 @@ var metricBlockBuilderFlushedBlocks = promauto.NewCounterVec( }, []string{"tenant"}, ) +var metricBlockBuilderFlushSize = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "tempo", + Subsystem: "block_builder", + Name: "flush_size_bytes", + Help: "Size in bytes of blocks flushed by the block-builder.", + Buckets: prometheus.ExponentialBuckets(1024*1024, 2, 10), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, +}) + type tenantStore struct { tenantID string idGenerator util.IDGenerator diff --git a/tempodb/compactor.go b/tempodb/compactor.go index a68737eaa88..bd8751e139d 100644 --- a/tempodb/compactor.go +++ b/tempodb/compactor.go @@ -68,6 +68,15 @@ var ( Name: "compaction_spans_deduped_total", Help: "Total number of spans that are deduped per replication factor.", }, []string{"replication_factor"}) + metricCompactionOutputBlockSize = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "tempodb", + Name: "compaction_output_block_size_bytes", + Help: "Size in bytes of blocks produced by compaction.", + Buckets: prometheus.ExponentialBuckets(1024*1024, 2, 10), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }) errCompactionJobNoLongerOwned = fmt.Errorf("compaction job no longer owned") ) @@ -333,6 +342,9 @@ func (rw *readerWriter) CompactWithConfig(ctx context.Context, blockMetas []*bac } metricCompactionBlocks.WithLabelValues(compactionLevelLabel).Add(float64(len(blockMetas))) + for _, meta := range newCompactedBlocks { + metricCompactionOutputBlockSize.Observe(float64(meta.Size_)) + } logArgs := []interface{}{ "msg", From 4b9636e1fdc5a59882272eb004b8b1b6c028fbc5 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Fri, 14 Aug 2026 21:56:40 +0000 Subject: [PATCH 2/3] fixup: actually observe the block-builder flush-size metric metricBlockBuilderFlushSize was declared but never observed, tripping the unused-var lint check. --- modules/blockbuilder/tenant_store.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/blockbuilder/tenant_store.go b/modules/blockbuilder/tenant_store.go index 30f192a9dda..d865df52289 100644 --- a/modules/blockbuilder/tenant_store.go +++ b/modules/blockbuilder/tenant_store.go @@ -194,6 +194,7 @@ func (s *tenantStore) Flush(ctx context.Context, r tempodb.Reader, w tempodb.Wri span.AddEvent("wrote block to backend", trace.WithAttributes(attribute.String("block_id", newMeta.BlockID.String()))) metricBlockBuilderFlushedBlocks.WithLabelValues(s.tenantID).Inc() + metricBlockBuilderFlushSize.Observe(float64(newMeta.Size_)) if err := s.wal.LocalBackend().ClearBlock(uuid.UUID(newMeta.BlockID), s.tenantID); err != nil { return err From 68a9fb904391d974ad944cd158cfd15d1e1869f0 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Tue, 18 Aug 2026 14:24:45 +0000 Subject: [PATCH 3/3] chloggen: cross-reference the compaction output metric in the block-builder entry Addresses review feedback: this PR adds two metrics, and the block-builder fragment's note read as if it were the only one. --- .chloggen/block-builder-flush-size-metric.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.chloggen/block-builder-flush-size-metric.yaml b/.chloggen/block-builder-flush-size-metric.yaml index de4abe7465e..da374541dcb 100644 --- a/.chloggen/block-builder-flush-size-metric.yaml +++ b/.chloggen/block-builder-flush-size-metric.yaml @@ -18,7 +18,9 @@ issues: [] subtext: | The block-builder flush path had no visibility into the size of the blocks it writes to the backend. This closes that gap alongside the existing live-store `tempo_live_store_local_flush_size_bytes` - metric, informing `max_input_blocks` tuning and TCO analysis. + metric, informing `max_input_blocks` tuning and TCO analysis. Paired with a + `tempodb_compaction_output_block_size_bytes` metric added in the same PR for compaction output blocks + (see the sibling `.chloggen` entry). # The GitHub handle (without the leading @) of the change's author. Rendered as "(@handle)". user: zalegrala