Add block-size observability at flush and compaction

.chloggen/block-builder-flush-size-metric.yaml

@@ -0,0 +1,24 @@
+# Changelog entry. Generate a copy with `make chlog-new`, then fill in the fields.
+
+# One of: breaking | change | feature | enhancement | bug_fix | security
+change_type: enhancement
+
+# Component or area of the change. Must be one of the components in .chloggen/config.yaml,
+# e.g. distributor, querier, query-frontend, storage, operations.
+component: block-builder
+
+# A brief description of the change. Surround with quotes ("") if it must start with a backtick (`).
+note: Add `tempo_block_builder_flush_size_bytes` native histogram recording the size of blocks flushed by the block-builder.
+
+# (Optional) PR number(s), e.g. [7339]. Leave blank to auto-fill at release. See
+# .chloggen/README.md.
+issues: []
+
+# (Optional) Additional lines rendered under the note. Use a pipe (|) for multiline text.
+subtext: |
+  The block-builder flush path had no visibility into the size of the blocks it writes to the
+  backend. This closes that gap alongside the existing live-store `tempo_live_store_local_flush_size_bytes`
+  metric, informing `max_input_blocks` tuning and TCO analysis.
+
+# The GitHub handle (without the leading @) of the change's author. Rendered as "(@handle)".
+user: zalegrala

.chloggen/compaction-output-block-size-metric.yaml

@@ -0,0 +1,25 @@
+# Changelog entry. Generate a copy with `make chlog-new`, then fill in the fields.
+
+# One of: breaking | change | feature | enhancement | bug_fix | security
+change_type: enhancement
+
+# Component or area of the change. Must be one of the components in .chloggen/config.yaml,
+# e.g. distributor, querier, query-frontend, storage, operations.
+component: compactor
+
+# A brief description of the change. Surround with quotes ("") if it must start with a backtick (`).
+note: Add `tempodb_compaction_output_block_size_bytes` native histogram recording the size of blocks produced by compaction.
+
+# (Optional) PR number(s), e.g. [7339]. Leave blank to auto-fill at release. See
+# .chloggen/README.md.
+issues: []
+
+# (Optional) Additional lines rendered under the note. Use a pipe (|) for multiline text.
+subtext: |
+  No existing metric captured the final size of compacted output blocks (the existing
+  `tempodb_compaction_bytes_written_total` is a cumulative counter fed per row-group flush, not a
+  per-finished-block size). This gives operators the data needed to tune `max_input_blocks` and
+  reason about TCO.
+
+# The GitHub handle (without the leading @) of the change's author. Rendered as "(@handle)".
+user: zalegrala

modules/blockbuilder/tenant_store.go

@@ -30,6 +30,17 @@ var metricBlockBuilderFlushedBlocks = promauto.NewCounterVec(
 	}, []string{"tenant"},
 )
 
+var metricBlockBuilderFlushSize = promauto.NewHistogram(prometheus.HistogramOpts{
+	Namespace:                       "tempo",
+	Subsystem:                       "block_builder",
+	Name:                            "flush_size_bytes",
+	Help:                            "Size in bytes of blocks flushed by the block-builder.",
+	Buckets:                         prometheus.ExponentialBuckets(1024*1024, 2, 10),
+	NativeHistogramBucketFactor:     1.1,
+	NativeHistogramMaxBucketNumber:  100,
+	NativeHistogramMinResetDuration: 1 * time.Hour,
+})
+
 type tenantStore struct {
 	tenantID         string
 	idGenerator      util.IDGenerator
@@ -183,6 +194,7 @@ func (s *tenantStore) Flush(ctx context.Context, r tempodb.Reader, w tempodb.Wri
 	span.AddEvent("wrote block to backend", trace.WithAttributes(attribute.String("block_id", newMeta.BlockID.String())))
 
 	metricBlockBuilderFlushedBlocks.WithLabelValues(s.tenantID).Inc()
+	metricBlockBuilderFlushSize.Observe(float64(newMeta.Size_))
 
 	if err := s.wal.LocalBackend().ClearBlock(uuid.UUID(newMeta.BlockID), s.tenantID); err != nil {
 		return err

tempodb/compactor.go

@@ -68,6 +68,15 @@ var (
 		Name:      "compaction_spans_deduped_total",
 		Help:      "Total number of spans that are deduped per replication factor.",
 	}, []string{"replication_factor"})
+	metricCompactionOutputBlockSize = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace:                       "tempodb",
+		Name:                            "compaction_output_block_size_bytes",
+		Help:                            "Size in bytes of blocks produced by compaction.",
+		Buckets:                         prometheus.ExponentialBuckets(1024*1024, 2, 10),
+		NativeHistogramBucketFactor:     1.1,
+		NativeHistogramMaxBucketNumber:  100,
+		NativeHistogramMinResetDuration: 1 * time.Hour,
+	})
 
 	errCompactionJobNoLongerOwned = fmt.Errorf("compaction job no longer owned")
 )
@@ -333,6 +342,9 @@ func (rw *readerWriter) CompactWithConfig(ctx context.Context, blockMetas []*bac
 	}
 
 	metricCompactionBlocks.WithLabelValues(compactionLevelLabel).Add(float64(len(blockMetas)))
+	for _, meta := range newCompactedBlocks {
+		metricCompactionOutputBlockSize.Observe(float64(meta.Size_))
+	}
 
 	logArgs := []interface{}{
 		"msg",