Horizontal scaling for very large deployments

charts/promforecast/templates/configmap.yaml

@@ -27,5 +27,29 @@ data:
     {{- $_ := set $ha "redis" $redis }}
     {{- end }}
     {{- $_ := set $cfg "highAvailability" $ha }}
+    {{- /*
+        Sharding follows the same merge philosophy: the chart-level `sharding`
+        block is the easy switch, and we propagate `enabled`, `mode`, and the
+        shard count (from `replicaCount`, the single source of truth so the
+        StatefulSet replica count and the forecaster's view can't drift). The
+        inner `config.sharding` block stays available for advanced overrides.
+        For leader_assigned we also seed the query-cache Redis address (the
+        forecaster reads the membership/assignment store from there when HA is
+        off) unless the operator already set one.
+    */}}
+    {{- $sh := default dict (get $cfg "sharding") }}
+    {{- if .Values.sharding.enabled }}
+    {{- $_ := set $sh "enabled" true }}
+    {{- $_ := set $sh "mode" .Values.sharding.mode }}
+    {{- $_ := set $sh "replica_count" (.Values.replicaCount | int) }}
+    {{- if eq .Values.sharding.mode "leader_assigned" }}
+    {{- $qc := default dict (get $cfg "query_cache") }}
+    {{- if and (.Values.sharding.redis.address) (not (get $qc "redis_address")) }}
+    {{- $_ := set $qc "redis_address" .Values.sharding.redis.address }}
+    {{- $_ := set $cfg "query_cache" $qc }}
+    {{- end }}
+    {{- end }}
+    {{- end }}
+    {{- $_ := set $cfg "sharding" $sh }}
     {{- toYaml $cfg | nindent 4 }}
 {{- end }}

charts/promforecast/templates/deployment.yaml

@@ -1,19 +1,27 @@
+{{- /*
+    consistent_hash sharding renders a StatefulSet (see statefulset.yaml) for
+    stable per-replica ordinals; the Deployment covers every other case —
+    default, HA, and leader_assigned sharding (whose assigner distributes by
+    identity, so pods need no stable ordinal).
+*/}}
+{{- if not (and .Values.sharding.enabled (eq .Values.sharding.mode "consistent_hash")) }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ include "promforecast.fullname" . }}
   labels: {{- include "promforecast.labels" . | nindent 4 }}
 spec:
   {{- /*
-      HA mode uses RollingUpdate so a fresh leader can take over before the
-      old replica is fully torn down (the elector's best-effort release
-      shaves another few seconds off failover). Single-replica installs
-      keep Recreate to honour the stateless invariant — a brief gap during
-      pod replacement is fine because the next refit reproduces everything.
+      HA and leader_assigned sharding both run multiple replicas that must roll
+      without a full outage, so we force RollingUpdate (maxUnavailable: 0) when
+      the chart's default Recreate would otherwise drop every replica at once.
+      Single-replica installs keep Recreate to honour the stateless invariant —
+      a brief gap during pod replacement is fine because the next refit
+      reproduces everything.
   */}}
   replicas: {{ .Values.replicaCount }}
   strategy:
-    {{- if and .Values.highAvailability.enabled (eq (.Values.strategy.type | default "Recreate") "Recreate") }}
+    {{- if and (or .Values.highAvailability.enabled .Values.sharding.enabled) (eq (.Values.strategy.type | default "Recreate") "Recreate") }}
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
@@ -83,6 +91,31 @@ spec:
                   key: {{ .Values.highAvailability.redis.secretKey }}
             {{- end }}
             {{- end }}
+            {{- /*
+                leader_assigned sharding: every replica heartbeats into the
+                shared Redis and reads the published assignment. The pod name
+                is the shard identity; the namespace scopes the assigner Lease.
+                (consistent_hash sharding renders the StatefulSet instead.)
+            */}}
+            {{- if and .Values.sharding.enabled (eq .Values.sharding.mode "leader_assigned") }}
+            - name: PROMFORECAST_SHARDING_ENABLED
+              value: "true"
+            - name: PROMFORECAST_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: PROMFORECAST_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            {{- if .Values.sharding.redis.existingSecret }}
+            - name: PROMFORECAST_REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.sharding.redis.existingSecret }}
+                  key: {{ .Values.sharding.redis.secretKey }}
+            {{- end }}
+            {{- end }}
             {{- with .Values.extraEnv }}{{- toYaml . | nindent 12 }}{{- end }}
           volumeMounts:
             - name: config
@@ -107,3 +140,4 @@ spec:
       {{- with .Values.affinity }}
       affinity: {{- toYaml . | nindent 8 }}
       {{- end }}
+{{- end }}

charts/promforecast/templates/networkpolicy.yaml

@@ -37,7 +37,14 @@ spec:
         configured port (default 6379); operators with a non-default Redis
         port can override via networkPolicy.redisPort.
     */}}
-    {{- if .Values.highAvailability.enabled }}
+    {{- /*
+        leader_assigned sharding has the same egress needs as HA: the Lease
+        API on the kube-apiserver plus Redis (membership registry +
+        assignment). consistent_hash sharding needs neither — its egress is
+        just DNS + the TSDB (covered by networkPolicy.egress).
+    */}}
+    {{- $needsCoordEgress := or .Values.highAvailability.enabled (and .Values.sharding.enabled (eq .Values.sharding.mode "leader_assigned")) }}
+    {{- if $needsCoordEgress }}
     - to:
         - namespaceSelector: {}
       ports:

charts/promforecast/templates/rbac.yaml

@@ -1,4 +1,9 @@
-{{- if and .Values.rbac.create .Values.highAvailability.enabled }}
+{{- /*
+    The Lease API is needed by HA (leader election) and by leader_assigned
+    sharding (the assigner Lease). consistent_hash sharding needs no Lease.
+*/}}
+{{- $needsLease := or .Values.highAvailability.enabled (and .Values.sharding.enabled (eq .Values.sharding.mode "leader_assigned")) }}
+{{- if and .Values.rbac.create $needsLease }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:

charts/promforecast/templates/statefulset.yaml

@@ -0,0 +1,104 @@
+{{- /*
+    consistent_hash sharding needs each replica to have a STABLE shard index
+    in [0, replicas). A Deployment's pods get random names, so there is no
+    ordinal to derive an index from; a StatefulSet gives every pod a stable
+    ordinal name (<name>-0, <name>-1, …). The forecaster reads the trailing
+    ordinal of PROMFORECAST_POD_NAME as its shard index. This template renders
+    ONLY for consistent_hash sharding; every other case (default, HA, and
+    leader_assigned sharding, where the assigner distributes by identity)
+    renders the Deployment instead.
+*/}}
+{{- if and .Values.sharding.enabled (eq .Values.sharding.mode "consistent_hash") }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "promforecast.fullname" . }}
+  labels: {{- include "promforecast.labels" . | nindent 4 }}
+spec:
+  # The shard count. Each replica owns hash(group) % replicas == ordinal.
+  replicas: {{ .Values.replicaCount }}
+  serviceName: {{ include "promforecast.fullname" . }}
+  # OrderedReady would gate pod N on pod N-1 becoming Ready — needless here
+  # since shards are independent. Parallel brings the whole fleet up at once.
+  podManagementPolicy: Parallel
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels: {{- include "promforecast.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "promforecast.selectorLabels" . | nindent 8 }}
+        {{- with .Values.podLabels }}{{- toYaml . | nindent 8 }}{{- end }}
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- with .Values.podAnnotations }}{{- toYaml . | nindent 8 }}{{- end }}
+    spec:
+      serviceAccountName: {{ include "promforecast.serviceAccountName" . }}
+      securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: forecaster
+          image: {{ include "promforecast.image" . }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - --config=/etc/promforecast/config.yaml
+          ports:
+            - name: http
+              containerPort: 9091
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          resources: {{- toYaml .Values.resources | nindent 12 }}
+          securityContext: {{- toYaml .Values.securityContext | nindent 12 }}
+          env:
+            - name: PROMFORECAST_SHARDING_ENABLED
+              value: "true"
+            # The pod name carries the StatefulSet ordinal; the forecaster
+            # parses the trailing -N as its shard index and uses the full name
+            # as the shard_id label value.
+            - name: PROMFORECAST_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: PROMFORECAST_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            {{- with .Values.extraEnv }}{{- toYaml . | nindent 12 }}{{- end }}
+          volumeMounts:
+            - name: config
+              mountPath: /etc/promforecast
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+            {{- with .Values.extraVolumeMounts }}{{- toYaml . | nindent 12 }}{{- end }}
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "promforecast.configMapName" . }}
+        - name: tmp
+          emptyDir: {}
+        {{- with .Values.extraVolumes }}{{- toYaml . | nindent 8 }}{{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity: {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/promforecast/values.schema.json

@@ -50,6 +50,38 @@
         }
       }
     },
+    "sharding": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "enabled": { "type": "boolean" },
+        "mode": { "type": "string", "enum": ["consistent_hash", "leader_assigned"] },
+        "redis": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "address": { "type": "string" },
+            "existingSecret": { "type": "string" },
+            "secretKey": { "type": "string" }
+          }
+        }
+      },
+      "if": {
+        "properties": {
+          "enabled": { "const": true },
+          "mode": { "const": "leader_assigned" }
+        },
+        "required": ["enabled", "mode"]
+      },
+      "then": {
+        "properties": {
+          "redis": {
+            "properties": { "address": { "minLength": 1 } },
+            "required": ["address"]
+          }
+        }
+      }
+    },
     "existingConfigMap": { "type": "string" },
     "config": { "type": "object" },
     "service": {

charts/promforecast/values.yaml

@@ -28,6 +28,29 @@ highAvailability:
     existingSecret: ""
     secretKey: password
 
+# Horizontal sharding: run multiple *active* replicas, each fitting a disjoint
+# subset of the configured groups so the aggregate fit budget scales with the
+# replica count. Distinct from highAvailability (one active leader plus hot
+# followers serving a cached snapshot) and mutually exclusive with it. The
+# `replicaCount` field above controls the shard count. See
+# docs/scaling/sharded-ha.md for the sizing guide and mode trade-offs.
+sharding:
+  enabled: false
+  # consistent_hash — each replica owns a group iff hash(group) % replicas
+  #   equals its own shard index. With this mode the chart renders a
+  #   StatefulSet so each pod's stable ordinal (<name>-0, <name>-1, …) is its
+  #   shard index; no coordination backend is required.
+  # leader_assigned — a coordination.k8s.io/v1 Lease elects one replica to
+  #   compute a balanced group→shard assignment over the live membership and
+  #   publish it to Redis; every replica reads it. Rebalances on replica
+  #   join/leave. Requires `redis.address`. Renders a Deployment (pods need no
+  #   stable ordinal — the assigner distributes by identity).
+  mode: consistent_hash
+  redis:
+    address: ""     # required for mode=leader_assigned, e.g. redis-master.redis.svc:6379
+    existingSecret: ""
+    secretKey: password
+
 # Forecaster YAML config. Either inline below, or set existingConfigMap.
 existingConfigMap: ""
 config:
@@ -175,6 +198,26 @@ config:
       password: ""
       snapshot_ttl: 5m
 
+  # Sharding runtime view of the top-level `sharding` switch. `helm install`
+  # populates `enabled`, `mode`, and `replica_count` from the chart fields and
+  # `replicaCount` automatically, so operators usually only edit the top-level
+  # values. `shard_index` and `shard_id` are left at their derive-from-env
+  # defaults (-1 / "") — under Kubernetes the StatefulSet pod ordinal becomes
+  # the index and the pod name becomes the id. Override the inner block to
+  # tune the leader_assigned heartbeat/Lease or to pin a shard id off-cluster.
+  sharding:
+    enabled: false
+    mode: consistent_hash
+    replica_count: 1
+    shard_index: -1
+    shard_id: ""
+    heartbeat_interval_seconds: 10
+    member_ttl_seconds: 30
+    lease_name: promforecast-shard-assigner
+    lease_duration_seconds: 30
+    renew_deadline_seconds: 20
+    retry_period_seconds: 4
+
   # Optional OpenTelemetry tracing. Disabled by default; flip
   # `telemetry.enabled` and set an `otlp.endpoint` to ship traces of the
   # query → fit → export pipeline to an OTel Collector. The Docker image

docs/README.md

@@ -59,6 +59,7 @@ scheduling, and serving telemetry.
 - [Cardinality budgeting](operations/cardinality.md) — per-family multipliers and the `--strict-cardinality` CI gate
 - [Scheduling](operations/scheduling.md) — per-query refresh intervals, max-concurrent-fits cap
 - [High availability](operations/ha.md) — Lease-based leader election, Redis snapshot cache
+- [Sharded HA (horizontal scaling)](scaling/sharded-ha.md) — N active replicas each fitting a disjoint slice of groups, via `consistent_hash` or `leader_assigned`; the `shard_id` label and `forecast_shard_owns_group` gauge; sizing guide
 - [Query cache](operations/cache.md) — in-memory LRU + optional Redis dedup for overlapping PromQL
 - [Telemetry](operations/telemetry.md) — internal metrics catalogue + per-series staleness + self-observability (fit success ratio, regressor stability, per-group CPU/memory)
 - [Warm-up status](operations/warmup.md) — fresh-install "what is still loading?" endpoint and CLI

docs/operations/ha.md

@@ -48,6 +48,11 @@ config:
 
 Defaults match the Kubernetes control plane. The forecaster rejects any config where `renew_deadline_seconds >= lease_duration_seconds`.
 
-## Why not multi-leader sharding?
-
-True sharding (different replicas fitting different groups) is planned for a future major version. For now, followers carry zero fit cost — the simplest and safest model for v0.x deployments.
+## Sharding — when one leader isn't enough
+
+HA scales to one well-sized leader plus hot followers; the followers carry zero
+fit cost. Once a single leader can no longer fit everything within
+`refresh_interval`, switch to **[sharded HA](../scaling/sharded-ha.md)**, where
+every replica is active and fits a disjoint subset of the groups. HA and
+sharding are mutually exclusive — pick HA for zero-downtime failover when one
+replica keeps up, sharding for horizontal fit throughput.