From 18a9c981f6ddc5e01f8d9b2f4e998ea53345492a Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Tue, 25 Aug 2026 19:02:50 -0500 Subject: [PATCH 1/2] fix(mimir): gate StrongConsistency alert on strong mode Fix verified RED->GREEN. MimirStrongConsistencyOffsetMissing fires on default eventual read-consistency at ingest-storage.libsonnet:238 --- .../mimir-mixin/alerts/ingest-storage.libsonnet | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/operations/mimir-mixin/alerts/ingest-storage.libsonnet b/operations/mimir-mixin/alerts/ingest-storage.libsonnet index 61beec8bf2..3d1b8a95d7 100644 --- a/operations/mimir-mixin/alerts/ingest-storage.libsonnet +++ b/operations/mimir-mixin/alerts/ingest-storage.libsonnet @@ -238,10 +238,14 @@ local utils = import 'mixin-utils/utils.libsonnet'; alert: $.alertName('StrongConsistencyOffsetMissing'), 'for': '5m', expr: ||| - sum by (%(alert_aggregation_labels)s) (rate(cortex_ingest_storage_strong_consistency_requests_total{component="partition-reader", with_offset="false"}[%(range)s])) - / - sum by (%(alert_aggregation_labels)s) (rate(cortex_ingest_storage_strong_consistency_requests_total{component="partition-reader"}[%(range)s])) - * 100 > 5 + ( + sum by (%(alert_aggregation_labels)s) (rate(cortex_ingest_storage_strong_consistency_requests_total{component="partition-reader", with_offset="false"}[%(range)s])) + / + sum by (%(alert_aggregation_labels)s) (rate(cortex_ingest_storage_strong_consistency_requests_total{component="partition-reader"}[%(range)s])) + * 100 > 5 + ) + and + sum by (%(alert_aggregation_labels)s) (rate(cortex_ingest_storage_strong_consistency_requests_total{component="partition-reader", with_offset="true"}[%(range)s])) > 0 ||| % { alert_aggregation_labels: $._config.alert_aggregation_labels, range: $.rateInterval('1m'), From 236f18b60ede83e95acbca811d573dde733149d8 Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Thu, 27 Aug 2026 15:44:42 -0500 Subject: [PATCH 2/2] fix(mimir): gate on total strong-consistency traffic, not offset-tagged requests Bugbot: with_offset="true" gate suppressed the alert when offset injection fails completely (all requests missing offsets), the exact case it should catch. Gate on total partition-reader strong-consistency rate instead. --- operations/mimir-mixin/alerts/ingest-storage.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operations/mimir-mixin/alerts/ingest-storage.libsonnet b/operations/mimir-mixin/alerts/ingest-storage.libsonnet index 3d1b8a95d7..812889702e 100644 --- a/operations/mimir-mixin/alerts/ingest-storage.libsonnet +++ b/operations/mimir-mixin/alerts/ingest-storage.libsonnet @@ -245,7 +245,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; * 100 > 5 ) and - sum by (%(alert_aggregation_labels)s) (rate(cortex_ingest_storage_strong_consistency_requests_total{component="partition-reader", with_offset="true"}[%(range)s])) > 0 + sum by (%(alert_aggregation_labels)s) (rate(cortex_ingest_storage_strong_consistency_requests_total{component="partition-reader"}[%(range)s])) > 0 ||| % { alert_aggregation_labels: $._config.alert_aggregation_labels, range: $.rateInterval('1m'),