diff --git a/.chloggen/block-builder-flush-size-metric.yaml b/.chloggen/block-builder-flush-size-metric.yaml new file mode 100644 index 00000000000..da374541dcb --- /dev/null +++ b/.chloggen/block-builder-flush-size-metric.yaml @@ -0,0 +1,26 @@ +# Changelog entry. Generate a copy with `make chlog-new`, then fill in the fields. + +# One of: breaking | change | feature | enhancement | bug_fix | security +change_type: enhancement + +# Component or area of the change. Must be one of the components in .chloggen/config.yaml, +# e.g. distributor, querier, query-frontend, storage, operations. +component: block-builder + +# A brief description of the change. Surround with quotes ("") if it must start with a backtick (`). +note: Add `tempo_block_builder_flush_size_bytes` native histogram recording the size of blocks flushed by the block-builder. + +# (Optional) PR number(s), e.g. [7339]. Leave blank to auto-fill at release. See +# .chloggen/README.md. +issues: [] + +# (Optional) Additional lines rendered under the note. Use a pipe (|) for multiline text. +subtext: | + The block-builder flush path had no visibility into the size of the blocks it writes to the + backend. This closes that gap alongside the existing live-store `tempo_live_store_local_flush_size_bytes` + metric, informing `max_input_blocks` tuning and TCO analysis. Paired with a + `tempodb_compaction_output_block_size_bytes` metric added in the same PR for compaction output blocks + (see the sibling `.chloggen` entry). + +# The GitHub handle (without the leading @) of the change's author. Rendered as "(@handle)". +user: zalegrala diff --git a/.chloggen/compaction-output-block-size-metric.yaml b/.chloggen/compaction-output-block-size-metric.yaml new file mode 100644 index 00000000000..7f9f708c72c --- /dev/null +++ b/.chloggen/compaction-output-block-size-metric.yaml @@ -0,0 +1,25 @@ +# Changelog entry. Generate a copy with `make chlog-new`, then fill in the fields. + +# One of: breaking | change | feature | enhancement | bug_fix | security +change_type: enhancement + +# Component or area of the change. Must be one of the components in .chloggen/config.yaml, +# e.g. distributor, querier, query-frontend, storage, operations. +component: compactor + +# A brief description of the change. Surround with quotes ("") if it must start with a backtick (`). +note: Add `tempodb_compaction_output_block_size_bytes` native histogram recording the size of blocks produced by compaction. + +# (Optional) PR number(s), e.g. [7339]. Leave blank to auto-fill at release. See +# .chloggen/README.md. +issues: [] + +# (Optional) Additional lines rendered under the note. Use a pipe (|) for multiline text. +subtext: | + No existing metric captured the final size of compacted output blocks (the existing + `tempodb_compaction_bytes_written_total` is a cumulative counter fed per row-group flush, not a + per-finished-block size). This gives operators the data needed to tune `max_input_blocks` and + reason about TCO. + +# The GitHub handle (without the leading @) of the change's author. Rendered as "(@handle)". +user: zalegrala diff --git a/modules/blockbuilder/tenant_store.go b/modules/blockbuilder/tenant_store.go index 53b1d20b4ae..d865df52289 100644 --- a/modules/blockbuilder/tenant_store.go +++ b/modules/blockbuilder/tenant_store.go @@ -30,6 +30,17 @@ var metricBlockBuilderFlushedBlocks = promauto.NewCounterVec( }, []string{"tenant"}, ) +var metricBlockBuilderFlushSize = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "tempo", + Subsystem: "block_builder", + Name: "flush_size_bytes", + Help: "Size in bytes of blocks flushed by the block-builder.", + Buckets: prometheus.ExponentialBuckets(1024*1024, 2, 10), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, +}) + type tenantStore struct { tenantID string idGenerator util.IDGenerator @@ -183,6 +194,7 @@ func (s *tenantStore) Flush(ctx context.Context, r tempodb.Reader, w tempodb.Wri span.AddEvent("wrote block to backend", trace.WithAttributes(attribute.String("block_id", newMeta.BlockID.String()))) metricBlockBuilderFlushedBlocks.WithLabelValues(s.tenantID).Inc() + metricBlockBuilderFlushSize.Observe(float64(newMeta.Size_)) if err := s.wal.LocalBackend().ClearBlock(uuid.UUID(newMeta.BlockID), s.tenantID); err != nil { return err diff --git a/tempodb/compactor.go b/tempodb/compactor.go index a68737eaa88..bd8751e139d 100644 --- a/tempodb/compactor.go +++ b/tempodb/compactor.go @@ -68,6 +68,15 @@ var ( Name: "compaction_spans_deduped_total", Help: "Total number of spans that are deduped per replication factor.", }, []string{"replication_factor"}) + metricCompactionOutputBlockSize = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "tempodb", + Name: "compaction_output_block_size_bytes", + Help: "Size in bytes of blocks produced by compaction.", + Buckets: prometheus.ExponentialBuckets(1024*1024, 2, 10), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }) errCompactionJobNoLongerOwned = fmt.Errorf("compaction job no longer owned") ) @@ -333,6 +342,9 @@ func (rw *readerWriter) CompactWithConfig(ctx context.Context, blockMetas []*bac } metricCompactionBlocks.WithLabelValues(compactionLevelLabel).Add(float64(len(blockMetas))) + for _, meta := range newCompactedBlocks { + metricCompactionOutputBlockSize.Observe(float64(meta.Size_)) + } logArgs := []interface{}{ "msg",