Ruler: cache parsed rule files during rule sync

CHANGELOG.md

@@ -22,6 +22,7 @@
 * [ENHANCEMENT] Validation: Add an optional `reason` field to `limited_queries` rules, aligning them with `blocked_queries`. When set, the reason is included in the client-facing error and the query-frontend's `"query limited"` log line. #16407
 * [ENHANCEMENT] Compactor: Add the experimental `-compactor.scheduler-client.enable-ring-based-cleanup` option, which when disabled stops a scheduler-mode compactor from running the ring-based background blocks cleaner. #16457
 * [ENHANCEMENT] Block-builder-scheduler: Add `cortex_blockbuilder_scheduler_end_offset_probe_failed_total`, counting failures to list a cluster's end offsets, and `cortex_blockbuilder_scheduler_startup_jobs_skipped_total`, counting observed jobs that startup recovery could not import. #16134
+* [ENHANCEMENT] Ruler: Add the experimental `-ruler.rule-file-parse-caching-enabled` option to cache parsed rule files and avoid re-parsing unchanged rule files on every rule sync. #16512
 * [FEATURE] Querier: Add experimental per-tenant limit `-querier.max-blocks-per-store-request` to cap the number of blocks a single store-gateway request may reference. Disabled by default. #16292
 * [FEATURE] Validation: Add optional `id`, `note`, `created_by`, `created_at`, and `expires_at` fields to `blocked_queries` and `limited_queries` rules, for tooling to attach ownership/context metadata to a rule. For rules with `expires_at` set, the earliest `expires_at` per tenant and `id` (rules without an `id` are grouped together) is exported as the `cortex_blocked_query_rule_expires_at`/`cortex_limited_query_rule_expires_at` metrics, so an alert can fire on stale rules; this is informational only and never affects enforcement. The query-frontend's `"query blocked"` log line now also includes the matched rule's `id` and whether it is expired, and rate-limited queries are now logged with a new `"query limited"` line carrying the same fields. #16395
 * [BUGFIX] Query-frontend: Wait for the querier ring to be populated during startup, up to 30 seconds, before reporting the query-frontend as ready. Previously a query-frontend could become ready before it had seen any querier in the ring and fail every query it received until the ring was populated. Only applies when remote execution is enabled, and can be disabled with the experimental `-query-frontend.wait-for-querier-ring-on-startup=false`. #16333

docs/sources/mimir/configure/about-versioning.md

@@ -83,6 +83,8 @@ The following features are currently experimental:
   - `-ruler.max-independent-rule-evaluation-concurrency-per-tenant`
   - `-ruler.independent-rule-evaluation-concurrency-min-duration-percentage`
   - `-ruler.rule-evaluation-write-enabled`
+  - Cache the result of parsing rule files on disk to avoid re-parsing unchanged rule files on every rule sync.
+    - `-ruler.rule-file-parse-caching-enabled`
   - Push rule-result series to remote distributors over native gRPC instead of using the internal distributor.
     - `-ruler.distributor.address`
     - `-ruler.distributor.remote-timeout`

pkg/ruler/compat.go

@@ -463,13 +463,30 @@ func DefaultTenantManagerFactory(
 			Help: "Number of queries that did not fetch any series by ruler.",
 		}, []string{"user"})
 	}
+	var ruleFileParseCacheHits *prometheus.CounterVec
+	var ruleFileParseCacheMisses *prometheus.CounterVec
+	if cfg.RuleFileParseCachingEnabled {
+		ruleFileParseCacheHits = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_ruler_rule_file_parse_cache_hits_total",
+			Help: "Total number of rule file parses served from the parse cache instead of re-parsing.",
+		}, []string{"user"})
+		ruleFileParseCacheMisses = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_ruler_rule_file_parse_cache_misses_total",
+			Help: "Total number of rule file parses that were not served from the cache.",
+		}, []string{"user"})
+	}
 	return func(ctx context.Context, userID string, notifier *notifier.Manager, logger log.Logger, reg prometheus.Registerer) RulesManager {
 		var queryTime prometheus.Counter
 		var zeroFetchedSeriesCount prometheus.Counter
 		if rulerQuerySeconds != nil {
 			queryTime = rulerQuerySeconds.WithLabelValues(userID)
 			zeroFetchedSeriesCount = zeroFetchedSeriesQueries.WithLabelValues(userID)
 		}
+		var cacheHits, cacheMisses prometheus.Counter
+		if cfg.RuleFileParseCachingEnabled {
+			cacheHits = ruleFileParseCacheHits.WithLabelValues(userID)
+			cacheMisses = ruleFileParseCacheMisses.WithLabelValues(userID)
+		}
 		// Wrap the query function with our custom logic.
 		wrappedQueryFunc := WrapQueryFuncWithReadConsistency(queryFunc, overrides, userID, logger)
 		remoteQuerier := cfg.QueryFrontend.Address != ""
@@ -502,7 +519,7 @@ func DefaultTenantManagerFactory(
 			OutageTolerance:            cfg.OutageTolerance,
 			ForGracePeriod:             cfg.ForGracePeriod,
 			ResendDelay:                cfg.ResendDelay,
-			GroupLoader:                NewFSLoader(rulesFS),
+			GroupLoader:                NewFSLoader(rulesFS, cfg.RuleFileParseCachingEnabled, cacheHits, cacheMisses),
 			RestoreNewRuleGroups:       true,
 			DefaultRuleQueryOffset: func() time.Duration {
 				// Delay the evaluation of all rules by a set interval to give a buffer

pkg/ruler/mapper.go

@@ -9,14 +9,17 @@ import (
 	"bytes"
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/url"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
+	"sync"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
 	"github.com/prometheus/prometheus/model/rulefmt"
@@ -181,42 +184,157 @@ func (m *mapper) writeRuleGroupsIfNewer(groups []rulefmt.RuleGroup, filename str
 }
 
 // FSLoader a GroupLoader implementation that reads files from a given afero.Fs.
+//
+// If cacheEnabled is set, it caches the parsed result of each file.
+// The cache key is the file's exact byte content plus the parsing options passed to Load.
+// A file that hasn't changed since the last Load call skips rulefmt.Parse entirely.
+//
+// The cache holds two generations, cur and prev, to bound memory.
+// This avoids needing an external signal for when a file is removed or renamed.
+// rules.Manager.LoadGroups calls Load at most once per path per Update pass.
+// manager.Update holds its own lock. Passes for one tenant never interleave.
+// A path already present in cur means a new pass has started.
+// Load then rotates the generations: prev = cur, cur = a new empty map.
+//
+// A hit in prev gets promoted into cur.
+// This keeps a stable file triggering rotation on every later pass.
+// Without promotion, a stable file would fall out of the cache after a single pass.
+//
+// A path that stops being loaded ages out within two passes.
+// Its entry moves into prev on the next rotation.
+// It is dropped for good when prev is next overwritten.
+// This bounds the cache at roughly 2x the live file count, regardless of how many distinct paths a tenant has ever used.
+//
+// Known gap: this needs at least one stable path across two consecutive passes to trigger a rotation.
+// A tenant whose entire namespace set changes to new paths on every single pass never triggers one.
+// That is considered acceptable. It requires zero stable namespaces ever, not just occasional renames.
 type FSLoader struct {
 	fs     afero.Fs
 	parser parser.Parser
 	logger *slog.Logger
+
+	cacheEnabled bool
+	cacheHits    prometheus.Counter
+	cacheMisses  prometheus.Counter
+
+	mu        sync.Mutex
+	cur, prev map[string]cachedRuleGroups
+}
+
+type cachedRuleGroups struct {
+	rawBytes             []byte
+	ignoreUnknownFields  bool
+	nameValidationScheme model.ValidationScheme
+	parsed               *rulefmt.RuleGroups
 }
 
-func NewFSLoader(fs afero.Fs) FSLoader {
-	return FSLoader{
-		fs:     fs,
-		parser: promqlext.NewPromQLParser(),
-		logger: promslog.NewNopLogger(),
+// NewFSLoader returns a GroupLoader that reads rule files from fs. When
+// cacheEnabled is true, cacheHits and cacheMisses must be non-nil and are
+// incremented on every Load call.
+func NewFSLoader(fs afero.Fs, cacheEnabled bool, cacheHits, cacheMisses prometheus.Counter) *FSLoader {
+	loader := &FSLoader{
+		fs:           fs,
+		parser:       promqlext.NewPromQLParser(),
+		logger:       promslog.NewNopLogger(),
+		cacheEnabled: cacheEnabled,
+		cacheHits:    cacheHits,
+		cacheMisses:  cacheMisses,
 	}
+	if cacheEnabled {
+		loader.cur = make(map[string]cachedRuleGroups)
+	}
+	return loader
 }
 
-func (f FSLoader) Load(identifier string, ignoreUnknownFields bool, nameValidationScheme model.ValidationScheme) (*rulefmt.RuleGroups, []error) {
+func (f *FSLoader) Load(identifier string, ignoreUnknownFields bool, nameValidationScheme model.ValidationScheme) (*rulefmt.RuleGroups, []error) {
 	return f.parseFile(f.fs, identifier, ignoreUnknownFields, nameValidationScheme)
 }
 
-func (f FSLoader) Parse(query string) (parser.Expr, error) {
+func (f *FSLoader) Parse(query string) (parser.Expr, error) {
 	return f.parser.ParseExpr(query)
 }
 
 // parseFile reads and parses rules from a file.
 // Duplicate of Prometheus' rulefmt.ParseFile, but injects the FS.
-func (f FSLoader) parseFile(fs afero.Fs, file string, ignoreUnknownFields bool, nameValidationScheme model.ValidationScheme) (*rulefmt.RuleGroups, []error) {
+func (f *FSLoader) parseFile(fs afero.Fs, file string, ignoreUnknownFields bool, nameValidationScheme model.ValidationScheme) (*rulefmt.RuleGroups, []error) {
 	b, err := afero.ReadFile(fs, file)
 	if err != nil {
 		return nil, []error{fmt.Errorf("%s: %w", file, err)}
 	}
+
+	if f.cacheEnabled {
+		if rgs, ok := f.lookupCache(file, b, ignoreUnknownFields, nameValidationScheme); ok {
+			f.cacheHits.Inc()
+			return rgs, nil
+		}
+		f.cacheMisses.Inc()
+	}
+
 	rgs, errs := rulefmt.Parse(b, ignoreUnknownFields, nameValidationScheme, f.parser, f.logger)
 	for i := range errs {
 		errs[i] = fmt.Errorf("%s: %w", file, errs[i])
 	}
+	if len(errs) == 0 && f.cacheEnabled {
+		// Cache a copy, not rgs itself: its SourceTenants field is mutated in place by the caller.
+		f.storeCache(file, b, ignoreUnknownFields, nameValidationScheme, copyRuleGroups(rgs))
+	}
 	return rgs, errs
 }
 
+func (f *FSLoader) lookupCache(path string, rawBytes []byte, ignoreUnknownFields bool, nameValidationScheme model.ValidationScheme) (*rulefmt.RuleGroups, bool) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	if _, ok := f.cur[path]; ok {
+		f.prev = f.cur
+		f.cur = make(map[string]cachedRuleGroups)
+	}
+
+	entry, ok := f.prev[path]
+	if !ok ||
+		entry.ignoreUnknownFields != ignoreUnknownFields ||
+		entry.nameValidationScheme != nameValidationScheme ||
+		!bytes.Equal(entry.rawBytes, rawBytes) {
+		return nil, false
+	}
+
+	f.cur[path] = entry
+	return copyRuleGroups(entry.parsed), true
+}
+
+func (f *FSLoader) storeCache(path string, rawBytes []byte, ignoreUnknownFields bool, nameValidationScheme model.ValidationScheme, rgs *rulefmt.RuleGroups) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	f.cur[path] = cachedRuleGroups{
+		rawBytes:             rawBytes,
+		ignoreUnknownFields:  ignoreUnknownFields,
+		nameValidationScheme: nameValidationScheme,
+		parsed:               rgs,
+	}
+}
+
+// copyRuleGroups returns a deep copy of rgs.
+func copyRuleGroups(rgs *rulefmt.RuleGroups) *rulefmt.RuleGroups {
+	if rgs == nil {
+		return nil
+	}
+	out := &rulefmt.RuleGroups{
+		Groups: make([]rulefmt.RuleGroup, len(rgs.Groups)),
+	}
+	for i, g := range rgs.Groups {
+		g.SourceTenants = slices.Clone(g.SourceTenants)
+		g.Rules = make([]rulefmt.Rule, len(rgs.Groups[i].Rules))
+		for j, r := range rgs.Groups[i].Rules {
+			r.Labels = maps.Clone(r.Labels)
+			r.Annotations = maps.Clone(r.Annotations)
+			g.Rules[j] = r
+		}
+		out.Groups[i] = g
+	}
+	return out
+}
+
 // cleanRuleGroupExprs returns a copy of groups with leading/trailing whitespace
 // trimmed from rule expressions. This avoids yaml.v3 emitting explicit
 // indentation indicators (e.g. "|4") for expressions that start with newlines