fix(compose): reserve resources on observability/WUD/mailpit + guardrail (#126)

The base stack, GlitchTip, and BullMQ all declare deploy.resources.reservations, but the observability overlay (8 services), WUD, and mailpit shipped only limits — so under host memory pressure those containers had no guaranteed floor and were first to be OOM-killed/throttled, losing exactly the observability an operator needs mid-incident. Adds env-overridable reservations (~20% cpus / 25% memory of limits, mirroring base) and a new check_resource_reservations guardrail (render_full_config; fails any restart!=no service without reservations) wired into the 'all' run and a per-step CI invocation. Dev-only mailpit gets reservations for consistency but stays outside the full-stack guardrail scope, like the other hardening checks.

Audit: F001

Author: agjs

.github/workflows/infra-compose-validate-compose.yml

@@ -124,6 +124,12 @@ jobs:
         # so CI and local guardrails cannot drift.
         run: ../scripts/validate-guardrails.sh no-new-privileges
         working-directory: infra/compose/compose
+      - name: Guardrail — long-running services reserve resources (no starvation)
+        if: steps.filter.outputs.code == 'true'
+        # Single-source: the same script runs in the local pre-push gate,
+        # so CI and local guardrails cannot drift.
+        run: ../scripts/validate-guardrails.sh resource-reservations
+        working-directory: infra/compose/compose
       - name: Guardrail — prod rejects unpinned image tags (dev.sh fail-closed)
         if: steps.filter.outputs.code == 'true'
         # Single-source: the same script runs in the local pre-push gate,

infra/compose/compose/docker-compose.mailpit.yml

@@ -44,6 +44,9 @@ services:
         limits:
           cpus: "${MAILPIT_LIMITS_CPUS:-0.25}"
           memory: "${MAILPIT_LIMITS_MEMORY:-128M}"
+        reservations:
+          cpus: "${MAILPIT_RESERVATIONS_CPUS:-0.05}"
+          memory: "${MAILPIT_RESERVATIONS_MEMORY:-32M}"
 
 volumes:
   mailpit_data:

infra/compose/compose/docker-compose.observability.yml

@@ -50,6 +50,9 @@ services:
         limits:
           cpus: "${PROMETHEUS_LIMITS_CPUS:-0.5}"
           memory: "${PROMETHEUS_LIMITS_MEMORY:-512M}"
+        reservations:
+          cpus: "${PROMETHEUS_RESERVATIONS_CPUS:-0.1}"
+          memory: "${PROMETHEUS_RESERVATIONS_MEMORY:-128M}"
 
   alertmanager:
     profiles: ["observability"]
@@ -83,6 +86,9 @@ services:
         limits:
           cpus: "${ALERTMANAGER_LIMITS_CPUS:-0.25}"
           memory: "${ALERTMANAGER_LIMITS_MEMORY:-128M}"
+        reservations:
+          cpus: "${ALERTMANAGER_RESERVATIONS_CPUS:-0.05}"
+          memory: "${ALERTMANAGER_RESERVATIONS_MEMORY:-32M}"
 
   grafana:
     profiles: ["observability"]
@@ -123,6 +129,9 @@ services:
         limits:
           cpus: "${GRAFANA_LIMITS_CPUS:-0.5}"
           memory: "${GRAFANA_LIMITS_MEMORY:-512M}"
+        reservations:
+          cpus: "${GRAFANA_RESERVATIONS_CPUS:-0.1}"
+          memory: "${GRAFANA_RESERVATIONS_MEMORY:-128M}"
 
   loki:
     profiles: ["observability"]
@@ -147,6 +156,9 @@ services:
         limits:
           cpus: "${LOKI_LIMITS_CPUS:-0.5}"
           memory: "${LOKI_LIMITS_MEMORY:-512M}"
+        reservations:
+          cpus: "${LOKI_RESERVATIONS_CPUS:-0.1}"
+          memory: "${LOKI_RESERVATIONS_MEMORY:-128M}"
 
   promtail:
     profiles: ["observability"]
@@ -175,6 +187,9 @@ services:
         limits:
           cpus: "${PROMTAIL_LIMITS_CPUS:-0.25}"
           memory: "${PROMTAIL_LIMITS_MEMORY:-128M}"
+        reservations:
+          cpus: "${PROMTAIL_RESERVATIONS_CPUS:-0.05}"
+          memory: "${PROMTAIL_RESERVATIONS_MEMORY:-32M}"
 
   tempo:
     profiles: ["observability"]
@@ -200,6 +215,9 @@ services:
         limits:
           cpus: "${TEMPO_LIMITS_CPUS:-0.5}"
           memory: "${TEMPO_LIMITS_MEMORY:-512M}"
+        reservations:
+          cpus: "${TEMPO_RESERVATIONS_CPUS:-0.1}"
+          memory: "${TEMPO_RESERVATIONS_MEMORY:-128M}"
 
   postgres-exporter:
     profiles: ["observability"]
@@ -226,6 +244,9 @@ services:
         limits:
           cpus: "${POSTGRES_EXPORTER_LIMITS_CPUS:-0.1}"
           memory: "${POSTGRES_EXPORTER_LIMITS_MEMORY:-64M}"
+        reservations:
+          cpus: "${POSTGRES_EXPORTER_RESERVATIONS_CPUS:-0.05}"
+          memory: "${POSTGRES_EXPORTER_RESERVATIONS_MEMORY:-32M}"
 
   node-exporter:
     profiles: ["observability"]
@@ -256,6 +277,9 @@ services:
         limits:
           cpus: "${NODE_EXPORTER_LIMITS_CPUS:-0.1}"
           memory: "${NODE_EXPORTER_LIMITS_MEMORY:-64M}"
+        reservations:
+          cpus: "${NODE_EXPORTER_RESERVATIONS_CPUS:-0.05}"
+          memory: "${NODE_EXPORTER_RESERVATIONS_MEMORY:-32M}"
 
 volumes:
   prometheus_data:

infra/compose/compose/docker-compose.wud.yml

@@ -66,6 +66,9 @@ services:
         limits:
           cpus: "${WUD_LIMITS_CPUS:-0.25}"
           memory: "${WUD_LIMITS_MEMORY:-256M}"
+        reservations:
+          cpus: "${WUD_RESERVATIONS_CPUS:-0.05}"
+          memory: "${WUD_RESERVATIONS_MEMORY:-64M}"
 
 volumes:
   wud_data:

infra/compose/scripts/validate-guardrails.sh

@@ -9,7 +9,8 @@
 #   ./validate-guardrails.sh [check]
 #
 # Checks: healthchecks | digest-pins | credential-fallbacks | valkey-auth
-#         | rooted-caps | no-new-privileges | prod-image-tags | all (default)
+#         | rooted-caps | no-new-privileges | resource-reservations
+#         | prod-image-tags | all (default)
 #
 # Requires: docker (compose config rendering), python3.
 
@@ -274,13 +275,49 @@ check_prod_image_tags() {
   ok "prod-image-tags"
 }
 
+check_resource_reservations() {
+  # Render the FULL stack (base prod + every prod-capable overlay) and require
+  # that every long-running service declares deploy.resources.reservations, not
+  # just limits. Limits cap a container; reservations guarantee its floor — the
+  # scheduler honors them so an observability/exporter container can't be
+  # starved or OOM-killed under host memory pressure, which is exactly when an
+  # operator needs metrics and dashboards. The base stack, GlitchTip, and
+  # BullMQ already set reservations; the observability + WUD overlays drifted.
+  # One-shot jobs (restart: no) are exempt — same criterion as
+  # check_healthchecks. Dev-only overlays (mailpit) are out of scope here, like
+  # the other full-stack guardrails.
+  render_full_config
+  python3 - <<'EOF'
+import json
+
+with open("/tmp/guardrails-full-config.json", encoding="utf-8") as handle:
+    services = json.load(handle)["services"]
+
+missing = sorted(
+    name
+    for name, svc in services.items()
+    if svc.get("restart") != "no"
+    and (svc.get("deploy") or {}).get("resources", {}).get("reservations") is None
+)
+
+if missing:
+    raise SystemExit(
+        "services missing deploy.resources.reservations: " + ", ".join(missing)
+    )
+
+print("reservations present on: " + ", ".join(sorted(services)))
+EOF
+  ok "resource-reservations"
+}
+
 case "$CHECK" in
   healthchecks)         check_healthchecks ;;
   digest-pins)          check_digest_pins ;;
   credential-fallbacks) check_credential_fallbacks ;;
   valkey-auth)          check_valkey_auth ;;
   rooted-caps)          check_rooted_caps ;;
   no-new-privileges)    check_no_new_privileges ;;
+  resource-reservations) check_resource_reservations ;;
   prod-image-tags)      check_prod_image_tags ;;
   all)
     check_digest_pins
@@ -289,10 +326,11 @@ case "$CHECK" in
     check_healthchecks
     check_rooted_caps
     check_no_new_privileges
+    check_resource_reservations
     check_prod_image_tags
     c_green "✓ all compose guardrails passed"
     ;;
   *)
-    fail "unknown check: $CHECK (healthchecks|digest-pins|credential-fallbacks|valkey-auth|rooted-caps|no-new-privileges|prod-image-tags|all)"
+    fail "unknown check: $CHECK (healthchecks|digest-pins|credential-fallbacks|valkey-auth|rooted-caps|no-new-privileges|resource-reservations|prod-image-tags|all)"
     ;;
 esac