From 56e6633fcd27a48f0de45363cbeb9364405893fa Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Mon, 13 Jul 2026 13:13:12 -0500
Subject: [PATCH 01/15] feat: expand alert catalog to 30 runbooks with a
 security category

Adds 10 runbooks (20 -> 30): a new security category (unexpected-container-image, apiserver-auth-failure-spike, privileged-container-started, interactive-shell-in-container, rbac-privilege-escalation, security-tooling-down) plus SRE gap-fillers (cpu-throttling-high, image-pull-backoff, pods-unschedulable, postgres-connections-near-max). Each is wired with a scaffoldSets entry, add/remove/validate autocomplete, help text, and inline Quick Diagnostics. Adds embedded docs ALERT_CATALOG.md (docs alerts) and ALERT_REQUIREMENTS.md (docs requirements).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 docs/ALERT_CATALOG.md                      | 111 +++++++++++++++++++++
 docs/ALERT_REQUIREMENTS.md                 |  87 ++++++++++++++++
 runbooks/INDEX.md                          |  21 +++-
 runbooks/apiserver-auth-failure-spike.md   |  90 +++++++++++++++++
 runbooks/cpu-throttling-high.md            |  89 +++++++++++++++++
 runbooks/image-pull-backoff.md             |  74 ++++++++++++++
 runbooks/interactive-shell-in-container.md |  90 +++++++++++++++++
 runbooks/pods-unschedulable.md             |  80 +++++++++++++++
 runbooks/postgres-connections-near-max.md  |  88 ++++++++++++++++
 runbooks/privileged-container-started.md   |  92 +++++++++++++++++
 runbooks/rbac-privilege-escalation.md      |  88 ++++++++++++++++
 runbooks/security-tooling-down.md          |  80 +++++++++++++++
 runbooks/unexpected-container-image.md     |  96 ++++++++++++++++++
 server/cmd_docs.go                         |   3 +
 server/cmd_scaffold.go                     |  12 +++
 server/commands.go                         |  15 ++-
 16 files changed, 1110 insertions(+), 6 deletions(-)
 create mode 100644 docs/ALERT_CATALOG.md
 create mode 100644 docs/ALERT_REQUIREMENTS.md
 create mode 100644 runbooks/apiserver-auth-failure-spike.md
 create mode 100644 runbooks/cpu-throttling-high.md
 create mode 100644 runbooks/image-pull-backoff.md
 create mode 100644 runbooks/interactive-shell-in-container.md
 create mode 100644 runbooks/pods-unschedulable.md
 create mode 100644 runbooks/postgres-connections-near-max.md
 create mode 100644 runbooks/privileged-container-started.md
 create mode 100644 runbooks/rbac-privilege-escalation.md
 create mode 100644 runbooks/security-tooling-down.md
 create mode 100644 runbooks/unexpected-container-image.md

diff --git a/docs/ALERT_CATALOG.md b/docs/ALERT_CATALOG.md
new file mode 100644
index 0000000..c901369
--- /dev/null
+++ b/docs/ALERT_CATALOG.md
@@ -0,0 +1,111 @@
+# Alert Catalog
+
+Every alert type the runbook library covers, grouped the same way as the
+`/alertmanager add <set>` group sets and the `groups:` in your Prometheus
+rules. Read it in chat with `/alertmanager docs alerts`.
+
+Each runbook slug maps to a rendered page at
+`/plugins/com.mattermost.alertmanager/public/runbooks/<slug>.html` and,
+when the alert fires, its first three Quick-Diagnostics blocks are inlined
+into the Mattermost post.
+
+## Alert message anatomy (what an alert can carry)
+
+These are the "options" every alert message is built from — set them in
+the Prometheus rule's `labels` / `annotations`:
+
+| Field | Where | Effect in the Mattermost post |
+|---|---|---|
+| `severity: critical` | label | Red sidebar, title `[CRITICAL:...]`, pages |
+| `severity: warning` | label | Yellow sidebar, chat-only |
+| `severity: info` | label | Blue sidebar, informational |
+| (status = resolved) | auto | Green `[✓ RESOLVED:...]` variant |
+| `runbook: <slug>` | label | Routes the alert to its per-runbook receiver |
+| `summary` | annotation | Title line / one-liner |
+| `description` | annotation | **Description:** block |
+| `runbook_url` | annotation | **Runbook:** link (falls back to the plugin-hosted page) |
+| `dashboard_url` | annotation | **Dashboard:** link (optional) |
+| any other labels | label | Listed under **Details:** and used to auto-fill `<label>` placeholders in the inline diagnostics |
+
+Diagnostic auto-fill labels: `alertname`, `app`, `cluster`, `container`,
+`deployment`, `instance`, `job`, `namespace`, `node`, `pod`, `service`.
+
+## Compute (`compute`)
+
+| Alert | slug | Severity | Fires when |
+|---|---|---|---|
+| High CPU Usage | `high-cpu-usage` | warning→critical | Container >80% CPU for 10m |
+| High Memory Usage | `high-memory-usage` | warning→critical | Container >85% mem limit, or OOMKilled |
+| Container OOMKilled | (uses `high-memory-usage`) | critical | Container killed for exceeding its memory limit |
+| Pod CrashLoopBackOff | `pod-crashloopbackoff` | critical | Restart count climbing fast |
+| Pod Not Ready | `pod-not-ready` | warning→critical | Readiness probe failing 5m+ |
+| Deployment Replicas Unavailable | `deployment-replicas-unavailable` | critical | Available < desired replicas |
+| Node Not Ready | `node-not-ready` | critical | Node `Ready=False` / pressure |
+| CPU Throttling High | `cpu-throttling-high` | warning | Throttled >25% of CFS periods |
+| Pod ImagePullBackOff | `image-pull-backoff` | critical | Image can't be pulled; pod never starts |
+| Pods Unschedulable | `pods-unschedulable` | warning | Pods stuck Pending; scheduler can't place |
+
+## Application (`application`)
+
+| Alert | slug | Severity | Fires when |
+|---|---|---|---|
+| High HTTP 5xx Error Rate | `high-http-error-rate` | critical | 5xx >5% over 10m |
+| High API Latency | `high-api-latency` | warning | p95 above SLO for 10m |
+| Service Endpoint Down | `service-endpoint-down` | critical | Probe failing on a known endpoint |
+| Request Rate Anomaly | `request-rate-anomaly` | warning | Sudden spike/drop in request rate |
+
+## Database (`database`)
+
+| Alert | slug | Severity | Fires when |
+|---|---|---|---|
+| Database Connectivity Loss | `database-connectivity-loss` | critical | Zero active DB connections |
+| Database Replication Lag | `database-replication-lag` | warning→critical | Replica lag > N seconds |
+| Database High Latency | `database-high-latency` | warning | p95 query time > N ms |
+| Postgres Connections Near Max | `postgres-connections-near-max` | warning→critical | In-use connections >80% of max |
+
+## Storage (`storage`)
+
+| Alert | slug | Severity | Fires when |
+|---|---|---|---|
+| Persistent Volume Full | `persistent-volume-full` | warning→critical | PV/PVC usage >85% |
+| Disk Fill Rate High | `disk-fill-rate-high` | warning | Projected to fill in <24h |
+
+## Networking (`networking`)
+
+| Alert | slug | Severity | Fires when |
+|---|---|---|---|
+| Ingress High 5xx | `ingress-high-5xx` | critical | LB/ingress 5xx spike |
+| Certificate Expiring Soon | `certificate-expiring-soon` | warning | TLS cert expires <14d |
+| DNS Resolution Failure | `dns-resolution-failure` | critical | Cluster DNS lookups failing |
+
+## Observability (`observability`)
+
+| Alert | slug | Severity | Fires when |
+|---|---|---|---|
+| Prometheus Scrape Target Down | `prometheus-scrape-target-down` | warning | `up == 0` for 5m+ |
+| Alertmanager Notification Failure | `alertmanager-notification-failure` | critical | Delivery failures climbing |
+
+## Security (`security`)
+
+Several require tooling beyond stock Prometheus/kube-state-metrics — the
+rule is valid but never fires without it.
+
+| Alert | slug | Severity | Fires when | Requires |
+|---|---|---|---|---|
+| Unexpected Container Image | `unexpected-container-image` | warning | Image outside the registry allowlist | kube-state-metrics |
+| API Server Auth Failure Spike | `apiserver-auth-failure-spike` | warning | 401/403 spike at the API server | apiserver metrics |
+| Privileged / Root Container Started | `privileged-container-started` | critical | Privileged/root/hostPath container starts | Kyverno/Gatekeeper |
+| Interactive Shell in Container | `interactive-shell-in-container` | warning | Shell spawned in a running container | Falco |
+| RBAC Privilege Escalation | `rbac-privilege-escalation` | critical | cluster-admin binding / wildcard role created | audit logs + Falco |
+| Security Tooling Down | `security-tooling-down` | critical | Falco/Kyverno/audit sensor stopped reporting | Falco/Kyverno scrape targets |
+
+## Wiring a set into a channel
+
+```
+/alertmanager add <team> <channel> <am-url> security
+/alertmanager add <team> <channel> <am-url> storage
+/alertmanager validate security --simulate runbook=unexpected-container-image severity=warning
+```
+
+`all` wires every slug above (30) behind one shared webhook. See
+`/alertmanager docs slash_commands` for the full command reference.
diff --git a/docs/ALERT_REQUIREMENTS.md b/docs/ALERT_REQUIREMENTS.md
new file mode 100644
index 0000000..5bba957
--- /dev/null
+++ b/docs/ALERT_REQUIREMENTS.md
@@ -0,0 +1,87 @@
+# Alert Requirements
+
+What each runbook's alert needs in order to **fire**. If the metric source
+in the "Provided by" column isn't being scraped, the rule is valid but
+never triggers. Read in chat with `/alertmanager docs requirements`.
+
+Legend for "Extra tooling":
+- **—** works with metrics a standard cluster already scrapes
+  (kube-state-metrics, cAdvisor/kubelet, node-exporter, kube-apiserver).
+- **⚠ named tool** requires deploying that tool first, or the alert is dark.
+
+## Compute
+
+| Runbook | Key metric(s) | Provided by | Extra tooling |
+|---|---|---|---|
+| `high-cpu-usage` | `container_cpu_usage_seconds_total` | cAdvisor (kubelet) | — |
+| `high-memory-usage` | `container_memory_working_set_bytes`, `container_spec_memory_limit_bytes` | cAdvisor | — |
+| `high-memory-usage` (OOMKilled) | `kube_pod_container_status_last_terminated_reason` | kube-state-metrics | — |
+| `pod-crashloopbackoff` | `kube_pod_container_status_restarts_total` | kube-state-metrics | — |
+| `pod-not-ready` | `kube_pod_status_ready` | kube-state-metrics | — |
+| `deployment-replicas-unavailable` | `kube_deployment_status_replicas_available` | kube-state-metrics | — |
+| `node-not-ready` | `kube_node_status_condition` | kube-state-metrics | — |
+| `cpu-throttling-high` | `container_cpu_cfs_throttled_periods_total`, `..._periods_total` | cAdvisor | — |
+| `image-pull-backoff` | `kube_pod_container_status_waiting_reason` | kube-state-metrics | — |
+| `pods-unschedulable` | `kube_pod_status_unschedulable` | kube-state-metrics | — |
+
+## Application
+
+| Runbook | Key metric(s) | Provided by | Extra tooling |
+|---|---|---|---|
+| `high-http-error-rate` | `http_requests_total{status=~"5.."}` | app instrumentation | ⚠ app must expose request metrics |
+| `high-api-latency` | `http_request_duration_seconds_bucket` | app instrumentation | ⚠ app must expose a latency histogram |
+| `service-endpoint-down` | `probe_success` | blackbox-exporter | ⚠ blackbox-exporter + probe config |
+| `request-rate-anomaly` | `http_requests_total` | app instrumentation | ⚠ app must expose request metrics |
+
+## Database
+
+| Runbook | Key metric(s) | Provided by | Extra tooling |
+|---|---|---|---|
+| `database-connectivity-loss` | `mattermost_db_master_connections_total` | Mattermost metrics endpoint | ⚠ enable MM metrics |
+| `database-replication-lag` | `pg_replication_lag_seconds` | postgres-exporter | ⚠ postgres-exporter |
+| `database-high-latency` | query-time histogram | postgres-exporter / app | ⚠ postgres-exporter or app metrics |
+| `postgres-connections-near-max` | `pg_stat_activity_count`, `pg_settings_max_connections` | postgres-exporter | ⚠ postgres-exporter |
+
+## Storage
+
+| Runbook | Key metric(s) | Provided by | Extra tooling |
+|---|---|---|---|
+| `persistent-volume-full` | `kubelet_volume_stats_available_bytes`, `..._capacity_bytes` | kubelet | — |
+| `disk-fill-rate-high` | `node_filesystem_avail_bytes` | node-exporter | ⚠ node-exporter |
+
+## Networking
+
+| Runbook | Key metric(s) | Provided by | Extra tooling |
+|---|---|---|---|
+| `ingress-high-5xx` | ingress 5xx counter | nginx/traefik ingress metrics | ⚠ ingress-controller metrics |
+| `certificate-expiring-soon` | `probe_ssl_earliest_cert_expiry` | blackbox-exporter | ⚠ blackbox-exporter |
+| `dns-resolution-failure` | `coredns_dns_responses_total` | CoreDNS metrics | ⚠ CoreDNS metrics scrape |
+
+## Observability
+
+| Runbook | Key metric(s) | Provided by | Extra tooling |
+|---|---|---|---|
+| `prometheus-scrape-target-down` | `up` | Prometheus (built-in) | — |
+| `alertmanager-notification-failure` | `alertmanager_notifications_failed_total` | Alertmanager (built-in) | — |
+
+## Security
+
+| Runbook | Key metric(s) | Provided by | Extra tooling |
+|---|---|---|---|
+| `unexpected-container-image` | `kube_pod_container_info` | kube-state-metrics | — (tune allowlist regex) |
+| `apiserver-auth-failure-spike` | `apiserver_request_total{code=~"401\|403"}` | kube-apiserver | — |
+| `privileged-container-started` | `kyverno_policy_results_total` (or equiv) | policy engine | ⚠ **Kyverno / Gatekeeper** |
+| `interactive-shell-in-container` | `falco_events{rule="Terminal shell in container"}` | Falco | ⚠ **Falco** |
+| `rbac-privilege-escalation` | `falco_events{rule=~"...RoleBinding..."}` | Falco k8s-audit plugin | ⚠ **audit logs + Falco** |
+| `security-tooling-down` | `up{job=~"falco\|kyverno.*"}` | the sensors' scrape targets | ⚠ sensors must expose `/metrics` |
+
+## Bootstrapping checklist
+
+To light up the **—** rows (most of the catalog) you need, at minimum:
+kube-state-metrics, cAdvisor/kubelet, node-exporter, and kube-apiserver
+scraped by Prometheus. Everything else is opt-in per the ⚠ column.
+
+The three ⚠ **Falco/Kyverno** security rows are the highest-lift — they
+need runtime/policy/audit tooling deployed. Until then those alerts (and
+`security-tooling-down` watching them) stay dark. See
+`/alertmanager docs alerts` for the plain catalog.
diff --git a/runbooks/INDEX.md b/runbooks/INDEX.md
index a32a018..bb08a48 100644
--- a/runbooks/INDEX.md
+++ b/runbooks/INDEX.md
@@ -1,6 +1,6 @@
 # Runbook Library
 
-20 runbooks covering the most common SRE alert categories. Each
+30 runbooks covering the most common SRE and security alert categories. Each
 follows the same structure (see [TEMPLATE.md](TEMPLATE.md)) so the
 on-call experience is consistent across alert types.
 
@@ -14,6 +14,9 @@ on-call experience is consistent across alert types.
 | [Pod Not Ready](pod-not-ready.md) | Readiness probe failing 5+ min | warning → critical if majority of replicas |
 | [Deployment Replicas Unavailable](deployment-replicas-unavailable.md) | Available replicas < spec replicas for 5+ min | critical |
 | [Node Not Ready](node-not-ready.md) | K8s node `Ready=False` or pressure conditions | critical |
+| [CPU Throttling High](cpu-throttling-high.md) | Container throttled >25% of CFS periods | warning |
+| [Pod ImagePullBackOff](image-pull-backoff.md) | Image can't be pulled — pod never starts | critical |
+| [Pods Unschedulable](pods-unschedulable.md) | Pods stuck Pending — scheduler can't place them | warning |
 
 ## Application
 
@@ -31,6 +34,7 @@ on-call experience is consistent across alert types.
 | [Database Connectivity Loss](database-connectivity-loss.md) | App can't reach DB; zero active connections | critical |
 | [Database Replication Lag](database-replication-lag.md) | Replica lag > N seconds for 5+ min | warning → critical at higher thresholds |
 | [Database High Latency](database-high-latency.md) | p95 query time > N ms sustained | warning |
+| [Postgres Connections Near Max](postgres-connections-near-max.md) | In-use connections > 80% of max_connections | warning → critical at 95% |
 
 ## Storage
 
@@ -54,6 +58,21 @@ on-call experience is consistent across alert types.
 | [Prometheus Scrape Target Down](prometheus-scrape-target-down.md) | `up == 0` for a target for 5+ min | warning |
 | [Alertmanager Notification Failure](alertmanager-notification-failure.md) | AM delivery dead-letter rate climbing | critical |
 
+## Security
+
+New in this release. Several depend on tooling beyond stock
+Prometheus/kube-state-metrics — noted in the Requires column. Without
+that tooling the rule is valid but never fires.
+
+| Runbook | When it fires | Severity | Requires |
+|---|---|---|---|
+| [Unexpected Container Image](unexpected-container-image.md) | Pod runs an image outside the registry allowlist | warning | kube-state-metrics |
+| [API Server Auth Failure Spike](apiserver-auth-failure-spike.md) | Spike in 401/403 from the kube-apiserver | warning | apiserver metrics |
+| [Privileged / Root Container Started](privileged-container-started.md) | Privileged / root / hostPath container starts | critical | Kyverno/Gatekeeper |
+| [Interactive Shell in Container](interactive-shell-in-container.md) | Shell spawned inside a running container | warning | Falco |
+| [RBAC Privilege Escalation](rbac-privilege-escalation.md) | cluster-admin binding or wildcard role created | critical | audit logs + Falco |
+| [Security Tooling Down](security-tooling-down.md) | Falco/Kyverno/audit sensor stopped reporting | critical | Falco/Kyverno scrape targets |
+
 ## How these get referenced
 
 Every Prometheus rule in your alert config should set its
diff --git a/runbooks/apiserver-auth-failure-spike.md b/runbooks/apiserver-auth-failure-spike.md
new file mode 100644
index 0000000..567ef48
--- /dev/null
+++ b/runbooks/apiserver-auth-failure-spike.md
@@ -0,0 +1,90 @@
+# API Server Auth Failure Spike
+
+!!! danger "Severity: warning"
+    **Target response: 20m.** A spike in `401`/`403` responses from the
+    Kubernetes API server — credential brute force, a stolen or expired
+    token, or a broken RBAC change hammering the API.
+
+## What this alert means
+
+Every request to the API server is authenticated and authorized. A
+sustained rate of `401` (bad/expired credentials) or `403` (authenticated
+but not permitted) is abnormal and worth a look.
+
+```promql
+# Rate of auth failures across the API server.
+sum(rate(apiserver_request_total{code=~"401|403"}[5m])) > 1
+```
+
+Benign causes exist (a rotated token not yet redeployed), but a *spike*
+is also exactly what credential stuffing or a compromised-but-underprivileged
+token looks like as it probes for access.
+
+## Quick diagnostics
+
+```promql
+# WHERE: Grafana → Explore (Prometheus) or Prometheus /graph.
+# WHAT: break the failures down by response code and, if your apiserver
+#   exposes it, by the requesting identity/useragent.
+# READ: all 401 = credential problem (expired/rotated/bad token).
+#   Mostly 403 = an identity authenticating fine but probing resources it
+#   can't touch — more suspicious. A single useragent dominating = one
+#   actor; spread across many = broader issue or a bad rollout.
+sum by (code, useragent) (rate(apiserver_request_total{code=~"401|403"}[5m]))
+```
+
+```bash
+# WHERE: shell with cluster-admin kubectl context.
+# WHAT: recent auth failures from the API server audit log (path varies
+#   by distro; adjust). Shows the user, source IP, and resource.
+# READ: repeated failures from one user/IP against sensitive resources
+#   (secrets, clusterrolebindings) = treat as an attack, not a misconfig.
+grep -E '"(responseStatus)":.*"code":(401|403)' /var/log/kubernetes/audit.log | tail -50
+```
+
+```bash
+# WHERE: shell with kubectl context.
+# WHAT: list service-account tokens that may have expired/rotated recently.
+# READ: a workload using a token whose secret was just rotated will 401 in
+#   a tight loop — correlate the failing identity with a recent rotation.
+kubectl get secrets -A --field-selector type=kubernetes.io/service-account-token -o wide | head -30
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Warning | No — chat only | 20m | Possible credential abuse or broken auth |
+
+Escalate to **critical/page** if 403s target secrets/RBAC resources or the
+source is external to the cluster.
+
+## Diagnostic steps
+
+1. **Classify** — 401 vs 403 split (query above). 401 → credentials; 403 → authorization.
+2. **Attribute** — identify the failing identity + source IP from audit logs.
+3. **Correlate** — recent token rotation, cert expiry, or RBAC change (`kubectl get events`, GitOps history)?
+4. **Contain if malicious** — revoke the token/binding, rotate the affected credential, block the source IP at the network layer.
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| Uniform 401 from one workload | Rotated token not redeployed | Restart the workload to pick up the new token |
+| 403 probing many resources | One identity testing access | Revoke, investigate as compromise |
+| Burst after an RBAC merge | Over-tight Role | Fix the Role binding |
+
+## Escalation
+
+1. **Primary** — `@sre-oncall` in `#mm-incidents`.
+2. **Security** — `@security-oncall` if the pattern looks like probing
+   (403s against secrets/RBAC) or the source is off-cluster.
+
+## Required Prometheus labels
+
+Diagnostics use API-server-level series (`code`, `useragent`) from
+`apiserver_request_total`. No per-pod labels required.
+
+## Related runbooks
+
+- [RBAC Privilege Escalation](rbac-privilege-escalation.md) — the follow-on if probing succeeds.
diff --git a/runbooks/cpu-throttling-high.md b/runbooks/cpu-throttling-high.md
new file mode 100644
index 0000000..afbd6d2
--- /dev/null
+++ b/runbooks/cpu-throttling-high.md
@@ -0,0 +1,89 @@
+# CPU Throttling High
+
+!!! danger "Severity: warning"
+    **Target response: 30m.** A container is being CPU-throttled by the
+    Linux CFS scheduler for a significant fraction of its runtime — latency
+    without the CPU-usage graph looking "hot."
+
+## What this alert means
+
+CFS throttling = the kernel capping a container at its CPU *limit*. The
+container can be throttled hard while sitting well under 100% average
+usage, so `HighCPUUsage` won't catch it. Throttling shows up as latency,
+timeouts, and slow health checks.
+
+```promql
+# Fraction of scheduler periods in which the container was throttled.
+sum by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total[5m]))
+/
+sum by (namespace, pod, container) (rate(container_cpu_cfs_periods_total[5m]))
+> 0.25
+```
+
+Sustained throttling >25% means the container spends a quarter of its
+scheduling periods waiting for CPU it's not allowed to use — often the
+hidden cause of a co-firing `HighAPILatency`.
+
+## Quick diagnostics
+
+```promql
+# WHERE: Grafana → Explore (Prometheus) or Prometheus /graph.
+# WHAT: the throttled-periods ratio for the alerting container over time.
+# READ: 0 = healthy. >0.1 = throttled 10%+ of periods (noticeable). >0.25
+#   = the alert threshold, real latency impact. Rising trend = getting worse.
+rate(container_cpu_cfs_throttled_seconds_total{namespace="<namespace>",pod="<pod>",container="<container>"}[5m])
+```
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: the CPU request vs limit for the throttled container.
+# READ: a low limit relative to request (or a limit barely above request)
+#   throttles under normal bursts. limit == request with bursty work is a
+#   classic throttling setup. No limit set = shouldn't throttle (recheck).
+kubectl get pod -n <namespace> <pod> -o jsonpath='{range .spec.containers[*]}{.name}: req={.resources.requests.cpu} lim={.resources.limits.cpu}{"\n"}{end}'
+```
+
+```promql
+# WHERE: Grafana → Explore or Prometheus /graph.
+# WHAT: actual CPU usage vs the limit, to size a fix.
+# READ: if usage rides just under the limit and throttling is high, the
+#   limit is too low — raise it. If usage is spiky, the limit is fine on
+#   average but too low for bursts — raise the limit, not the request.
+sum by (pod) (rate(container_cpu_usage_seconds_total{namespace="<namespace>",pod="<pod>"}[5m]))
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Warning | No — chat only | 30m | Latency / timeouts under load |
+
+## Diagnostic steps
+
+1. **Confirm throttling is current** (ratio query above).
+2. **Check limits** — request vs limit for the container.
+3. **Correlate** — does it line up with a latency alert or a traffic peak?
+4. **Right-size** — raise the CPU limit (or remove it for latency-critical, well-behaved workloads) and redeploy.
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| Limit ≈ request, bursty app | High throttle ratio at peaks | Raise limit to cover bursts |
+| Limit far below usage | Usage rides the ceiling | Raise limit to p95 usage + headroom |
+| Throttling only at deploy | Startup CPU spike | Raise limit or add startup probe slack |
+
+## Escalation
+
+1. **Primary** — `@sre-oncall` in `#mm-incidents`.
+2. **Service owner** — to agree new limits if the workload is theirs.
+
+## Required Prometheus labels
+
+Diagnostics use `namespace`, `pod`, `container`. Provided by cAdvisor
+(`container_cpu_cfs_*`).
+
+## Related runbooks
+
+- [High API Latency](high-api-latency.md) — throttling is a frequent hidden cause.
+- [High CPU Usage](high-cpu-usage.md) — the sibling that fires on raw usage, not throttling.
diff --git a/runbooks/image-pull-backoff.md b/runbooks/image-pull-backoff.md
new file mode 100644
index 0000000..83f21e6
--- /dev/null
+++ b/runbooks/image-pull-backoff.md
@@ -0,0 +1,74 @@
+# Pod ImagePullBackOff
+
+!!! danger "Severity: critical"
+    **Target response: 15m.** A pod can't pull its container image —
+    it's stuck `ImagePullBackOff`/`ErrImagePull` and will never start.
+    A rollout is effectively down.
+
+## What this alert means
+
+Kubernetes couldn't fetch the image: wrong tag, private registry with no
+/expired pull secret, registry outage, or rate limiting. Unlike a
+crashloop (image runs then dies), the container never starts at all.
+
+```promql
+kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"} == 1
+```
+
+If this is a new deploy, the rollout is blocked; if it's a scale-up, you
+can't add capacity.
+
+## Quick diagnostics
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: the pull error message straight from the pod events.
+# READ: "not found" = bad tag/name. "unauthorized"/"denied" = pull secret
+#   missing/expired. "toomanyrequests" = registry rate limit (Docker Hub).
+kubectl describe pod -n <namespace> <pod> | grep -A3 -iE "failed|pull|back-off"
+```
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: the image the pod wants + whether an imagePullSecret is attached.
+# READ: no imagePullSecrets on a private-registry image = the cause. Wrong
+#   registry host or tag in the image string = a bad manifest.
+kubectl get pod -n <namespace> <pod> -o jsonpath='image={.spec.containers[0].image}{"\n"}pullSecrets={.spec.imagePullSecrets}{"\n"}'
+```
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: confirm the pull secret exists and is the right type in this ns.
+# READ: missing secret, or type != kubernetes.io/dockerconfigjson = fix
+#   the secret. Present + correct = suspect registry outage / rate limit.
+kubectl get secret -n <namespace> -o custom-columns=NAME:.metadata.name,TYPE:.type | grep -i docker
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Critical | Yes — page on-call | 15m | Rollout blocked / can't scale |
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| "not found" | `kubectl describe` event | Fix the image tag in the manifest |
+| "unauthorized" | no/expired pull secret | Recreate `imagePullSecret`, attach to SA |
+| "toomanyrequests" | Docker Hub rate limit | Authenticate pulls / mirror through your registry |
+
+## Escalation
+
+1. **Primary** — `@sre-oncall` in `#mm-incidents`.
+2. **Registry owner** — if the registry itself is down/unreachable.
+
+## Required Prometheus labels
+
+Diagnostics use `namespace`, `pod`. From `kube_pod_container_status_waiting_reason`
+(kube-state-metrics).
+
+## Related runbooks
+
+- [Deployment Replicas Unavailable](deployment-replicas-unavailable.md) — the deployment-level symptom.
+- [Unexpected Container Image](unexpected-container-image.md) — if the "wrong" image is also unapproved.
diff --git a/runbooks/interactive-shell-in-container.md b/runbooks/interactive-shell-in-container.md
new file mode 100644
index 0000000..ca945fb
--- /dev/null
+++ b/runbooks/interactive-shell-in-container.md
@@ -0,0 +1,90 @@
+# Interactive Shell in Container
+
+!!! danger "Severity: warning"
+    **Target response: 15m.** A shell was spawned inside a running
+    container (`kubectl exec`, or a process spawning `/bin/sh`) —
+    hands-on-keyboard activity in a workload that should be immutable.
+
+## What this alert means
+
+Production containers should not have humans (or attacker payloads)
+opening shells in them. This alert fires on a Falco runtime rule that
+detects a terminal/shell spawned inside a container.
+
+```promql
+# Requires Falco exporting events to Prometheus.
+sum by (k8s_ns_name, k8s_pod_name) (
+  rate(falco_events{rule="Terminal shell in container"}[5m])
+) > 0
+```
+
+**Dependency:** this is a *runtime* signal. Vanilla Prometheus cannot see
+it — you need **Falco** (or an equivalent eBPF runtime sensor) shipping
+events. Without Falco this alert never fires. See Required labels.
+
+## Quick diagnostics
+
+```bash
+# WHERE: shell with kubectl context set to the affected cluster.
+# WHAT: who currently has an exec session / what's running in the pod.
+# READ: an interactive shell (bash/sh with a tty) that you can't tie to a
+#   known operator action = investigate. A one-off debug by a named
+#   engineer during an incident = probably benign, confirm in chat.
+kubectl exec -n <namespace> <pod> -- ps -eo pid,ppid,user,args 2>/dev/null | head -30
+```
+
+```bash
+# WHERE: shell with cluster-admin context.
+# WHAT: recent exec calls from the audit log against this pod.
+# READ: the `user` field tells you WHO exec'd. A service account or an
+#   unknown user exec-ing into prod = red flag; a human on-call = confirm.
+grep -E '"verb":"create".*"subresource":"exec".*<pod>' /var/log/kubernetes/audit.log | tail -20
+```
+
+```bash
+# WHERE: Falco logs / Falco sidecar.
+# WHAT: the full Falco event that fired, with proc.cmdline and user.
+# READ: shows the exact command and parent process. A shell whose parent
+#   is the app process (not kubectl-exec) suggests code execution via a
+#   vuln, not an operator — escalate.
+kubectl logs -n falco -l app.kubernetes.io/name=falco --since=15m | grep -i "shell in container"
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Warning | No — chat only | 15m | Possible hands-on-keyboard intrusion |
+
+Escalate to **critical/page** if the shell's parent is the application
+process (RCE indicator) or the exec came from a service account.
+
+## Diagnostic steps
+
+1. **Attribute** — audit log `user` field: named human vs. service account vs. unknown.
+2. **Confirm intent** — ask in `#mm-incidents` whether an operator is actively debugging.
+3. **Inspect the session** — parent process (Falco event); app-process parent = RCE, not exec.
+4. **Contain if unexplained** — snapshot the pod, revoke the acting identity, isolate with a deny-all NetworkPolicy, open IR.
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| On-call debugging | Named user in audit | Benign; prefer ephemeral debug containers |
+| SA-initiated exec | Non-human user | Revoke SA, investigate as compromise |
+| Shell parented to app | Falco proc tree | Treat as RCE; isolate + IR |
+
+## Escalation
+
+1. **Primary** — `@sre-oncall` in `#mm-incidents`.
+2. **Security** — `@security-oncall` for any unexplained or SA-initiated shell.
+
+## Required Prometheus labels
+
+Diagnostics use `k8s_ns_name`, `k8s_pod_name` (Falco label names differ
+from kube-state-metrics). **Requires Falco** shipping `falco_events`.
+
+## Related runbooks
+
+- [Privileged / Root Container Started](privileged-container-started.md) — escalation path after a shell.
+- [RBAC Privilege Escalation](rbac-privilege-escalation.md) — what the shell reaches for next.
diff --git a/runbooks/pods-unschedulable.md b/runbooks/pods-unschedulable.md
new file mode 100644
index 0000000..486b26a
--- /dev/null
+++ b/runbooks/pods-unschedulable.md
@@ -0,0 +1,80 @@
+# Pods Unschedulable
+
+!!! danger "Severity: warning"
+    **Target response: 20m.** One or more pods are stuck `Pending` —
+    the scheduler can't place them. Capacity, taints, affinity, or
+    resource requests are blocking placement.
+
+## What this alert means
+
+The scheduler evaluated every node and none fit. Common blockers:
+insufficient CPU/memory, node selectors/affinity with no matching node,
+taints without tolerations, or unbound PVCs.
+
+```promql
+kube_pod_status_unschedulable == 1
+```
+
+Pending pods = capacity you asked for but didn't get. During a scale-up
+or node loss, this is why the extra replicas never arrive.
+
+## Quick diagnostics
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: the scheduler's own reason for not placing the pod.
+# READ: "Insufficient cpu/memory" = cluster is full. "didn't match node
+#   selector/affinity" = placement rules too strict. "had untolerated
+#   taint" = needs a toleration. "unbound PersistentVolumeClaim" = storage.
+kubectl describe pod -n <namespace> <pod> | grep -A5 -i "Events\|FailedScheduling"
+```
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: node allocatable vs requested — is the cluster actually full?
+# READ: if every node's CPU/memory requests are near allocatable, you need
+#   more nodes (or smaller requests). Free capacity present = it's a
+#   placement-rule problem, not raw capacity.
+kubectl describe nodes | grep -A5 "Allocated resources"
+```
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: the pod's resource requests, nodeSelector, affinity, tolerations.
+# READ: an oversized request (more than any node has), or a nodeSelector
+#   for a label no node carries, pinpoints the blocker.
+kubectl get pod -n <namespace> <pod> -o jsonpath='req={.spec.containers[0].resources.requests} nodeSelector={.spec.nodeSelector} tolerations={.spec.tolerations}{"\n"}'
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Warning | No — chat only | 20m | Requested capacity not delivered |
+
+Escalate if the unschedulable pods are for a critical service scaling up
+during an incident.
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| Insufficient cpu/memory | node allocation full | Add nodes / lower requests |
+| No matching node | nodeSelector/affinity | Relax the rule or label a node |
+| Untolerated taint | describe event | Add toleration or untaint |
+| Unbound PVC | pending PVC | Fix storage class / provisioner |
+
+## Escalation
+
+1. **Primary** — `@sre-oncall` in `#mm-incidents`.
+2. **Cloud/platform** — for cluster-autoscaler / node-pool capacity.
+
+## Required Prometheus labels
+
+Diagnostics use `namespace`, `pod`. From `kube_pod_status_unschedulable`
+(kube-state-metrics).
+
+## Related runbooks
+
+- [Node Not Ready](node-not-ready.md) — lost nodes shrink schedulable capacity.
+- [Deployment Replicas Unavailable](deployment-replicas-unavailable.md) — the deployment-level view.
diff --git a/runbooks/postgres-connections-near-max.md b/runbooks/postgres-connections-near-max.md
new file mode 100644
index 0000000..ec94ea5
--- /dev/null
+++ b/runbooks/postgres-connections-near-max.md
@@ -0,0 +1,88 @@
+# Postgres Connections Near Max
+
+!!! danger "Severity: warning"
+    **Target response: 20m.** Postgres is approaching `max_connections`.
+    When it hits the ceiling, new connections are refused and the app
+    starts throwing "too many clients" — a self-inflicted outage.
+
+## What this alert means
+
+Every Postgres connection costs a backend process and memory. When
+in-use connections approach `max_connections`, the next client (including
+Mattermost) gets `FATAL: sorry, too many clients already`.
+
+```promql
+# Fraction of max_connections currently in use.
+sum(pg_stat_activity_count) / on() pg_settings_max_connections > 0.8
+```
+
+At >80% you're one traffic spike or connection leak away from refusal.
+Mattermost without a working DB connection fails writes immediately.
+
+## Quick diagnostics
+
+```sql
+-- WHERE: psql against the affected DB (use $DATABASE_URL or the admin DSN).
+-- WHAT: connection count by state and application, plus the configured max.
+-- READ: many 'idle in transaction' = a leak (app holding txns open); many
+--   'idle' from one app = an oversized pool; 'active' near max = genuine load.
+SELECT state, application_name, count(*)
+FROM pg_stat_activity GROUP BY state, application_name ORDER BY count(*) DESC;
+```
+
+```sql
+-- WHERE: psql against the affected DB.
+-- WHAT: longest-running / oldest transactions holding connections.
+-- READ: an 'idle in transaction' row with a large age is a leaked
+--   connection — the app opened a txn and never committed. That's the
+--   usual culprit behind creeping connection counts.
+SELECT pid, state, now()-xact_start AS xact_age, left(query,80)
+FROM pg_stat_activity WHERE state <> 'idle' ORDER BY xact_start LIMIT 15;
+```
+
+```promql
+# WHERE: Grafana → Explore or Prometheus /graph.
+# WHAT: connection count over time vs the max.
+# READ: a steady climb that never drops = a leak (won't self-heal, will
+#   hit the ceiling). A spike tracking traffic = capacity; scale the pooler
+#   or raise max_connections with memory headroom.
+sum(pg_stat_activity_count)
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Warning | No — chat only | 20m | Imminent connection refusals → write failures |
+
+Escalate to **critical/page** at >95% or once refusals appear in app logs.
+
+## Diagnostic steps
+
+1. **Confirm headroom** (ratio query) — how close to the ceiling.
+2. **Classify** — leak (`idle in transaction`, climbing) vs. load (`active`, tracks traffic).
+3. **Relieve** — terminate leaked backends (`pg_terminate_backend(pid)`); for load, add/adjust a connection pooler (PgBouncer).
+4. **Right-size** — raise `max_connections` only with RAM to back it (~10 MB/conn); prefer pooling.
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| Growing `idle in transaction` | App leaks txns | Fix app; kill leaked pids as stopgap |
+| One app with huge idle pool | Oversized client pool | Lower pool size / front with PgBouncer |
+| `active` near max under load | Genuine capacity | Add PgBouncer; scale read replicas |
+
+## Escalation
+
+1. **Primary** — `@dba-oncall` in `#mm-incidents`.
+2. **Service owner** — if a specific app is leaking connections.
+
+## Required Prometheus labels
+
+Diagnostics are cluster-level (no per-pod labels). Requires the Postgres
+exporter (`pg_stat_activity_count`, `pg_settings_max_connections`).
+
+## Related runbooks
+
+- [Database Connectivity Loss](database-connectivity-loss.md) — what happens when the ceiling is hit.
+- [Database High Latency](database-high-latency.md) — connection saturation and latency often co-fire.
diff --git a/runbooks/privileged-container-started.md b/runbooks/privileged-container-started.md
new file mode 100644
index 0000000..fdc1886
--- /dev/null
+++ b/runbooks/privileged-container-started.md
@@ -0,0 +1,92 @@
+# Privileged / Root Container Started
+
+!!! danger "Severity: critical"
+    **Target response: 15m.** A container started `privileged`, as UID 0,
+    or with a hostPath mount — the standard setup for a container escape
+    to the node.
+
+## What this alert means
+
+A privileged container shares the host's kernel capabilities; root +
+hostPath lets a process reach the node filesystem. Legitimate uses exist
+(CNI, CSI, node-exporter) — those should be allowlisted. Anything else
+starting privileged is a policy violation and a potential escape.
+
+```promql
+# Requires a policy engine (Kyverno/Gatekeeper) OR kube-state-metrics
+# security-context series. Example against a Kyverno violation counter:
+sum by (namespace, policy, rule) (
+  increase(kyverno_policy_results_total{rule=~".*privileged.*|.*run-as-non-root.*", policy_result="fail"}[10m])
+) > 0
+```
+
+**Dependency:** vanilla kube-state-metrics does not expose
+`securityContext.privileged` by default. This alert needs a policy engine
+(Kyverno/Gatekeeper) or a custom exporter. Without one, it will never
+fire — see Required labels.
+
+## Quick diagnostics
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: dump the security context of every container in the pod.
+# READ: privileged=true, runAsUser=0/absent, allowPrivilegeEscalation=true,
+#   or capabilities.add including SYS_ADMIN/NET_ADMIN = the risky settings.
+kubectl get pod -n <namespace> <pod> -o jsonpath='{range .spec.containers[*]}{.name}: {.securityContext}{"\n"}{end}'
+```
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: check for hostPath / host namespace access on the pod.
+# READ: hostPath mounts, hostNetwork/hostPID/hostIPC = true widen the blast
+#   radius from container to node. Combined with privileged, treat as an
+#   active escape risk.
+kubectl get pod -n <namespace> <pod> -o jsonpath='hostNetwork={.spec.hostNetwork} hostPID={.spec.hostPID} volumes={.spec.volumes}{"\n"}'
+```
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: who owns this pod, so you know if it's an allowlisted system
+#   workload or a user workload that shouldn't be privileged.
+# READ: kube-system CNI/CSI DaemonSets = expected; a Deployment in an app
+#   namespace running privileged = policy violation to remediate.
+kubectl get pod -n <namespace> <pod> -o jsonpath='{.metadata.ownerReferences}{"\n"}'
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Critical | Yes — page on-call | 15m | Container-to-node escape path is open |
+
+## Diagnostic steps
+
+1. **Confirm the settings** (security-context query above).
+2. **Classify** — allowlisted system workload vs. user workload.
+3. **Attribute** — owner ref + who applied it (GitOps history / audit).
+4. **Contain** — if not allowlisted, delete the pod and scale its owner to 0; the admission policy should block recreation once enforced.
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| App pod privileged | securityContext.privileged=true | Remove it; grant only the specific capability needed |
+| Runs as root | runAsUser 0 / unset | Set runAsNonRoot + a non-zero UID |
+| hostPath to /var/run/docker.sock | volumes list | Remove; this is a full node takeover primitive |
+
+## Escalation
+
+1. **Primary** — `@sre-oncall` in `#mm-incidents`.
+2. **Security** — `@security-oncall` immediately if the workload is
+   unrecognized or on production — assume escape attempt.
+
+## Required Prometheus labels
+
+Diagnostics use `namespace`, `policy`, `rule` (Kyverno) or your policy
+engine's equivalents. **Requires Kyverno/Gatekeeper/OPA** — not available
+from stock kube-state-metrics.
+
+## Related runbooks
+
+- [Unexpected Container Image](unexpected-container-image.md) — rogue images often ask for privilege.
+- [Interactive Shell in Container](interactive-shell-in-container.md) — what an attacker does once privileged.
diff --git a/runbooks/rbac-privilege-escalation.md b/runbooks/rbac-privilege-escalation.md
new file mode 100644
index 0000000..0c46935
--- /dev/null
+++ b/runbooks/rbac-privilege-escalation.md
@@ -0,0 +1,88 @@
+# RBAC Privilege Escalation
+
+!!! danger "Severity: critical"
+    **Target response: 15m.** A high-privilege RBAC object was created or
+    modified — e.g. a ClusterRoleBinding granting `cluster-admin`, or a
+    binding to a service account that shouldn't have it.
+
+## What this alert means
+
+RBAC changes that grant broad power are rare and deliberate in a healthy
+cluster. A binding to `cluster-admin`, or a new ClusterRole with wildcard
+verbs, is either a reviewed platform change or an attacker escalating
+after landing a foothold.
+
+```promql
+# Requires Falco (k8s audit plugin) or an audit-log → metrics pipeline.
+sum by (k8s_ns_name) (
+  rate(falco_events{rule=~"Create ClusterRoleBinding.*|Attach to cluster-admin.*|K8s Role.*wildcard.*"}[5m])
+) > 0
+```
+
+**Dependency:** needs Kubernetes **audit logs** flowing into Falco (k8s
+audit plugin) or a log→metric pipeline. Vanilla Prometheus/kube-state-metrics
+cannot observe RBAC *mutations*. See Required labels.
+
+## Quick diagnostics
+
+```bash
+# WHERE: shell with cluster-admin kubectl context.
+# WHAT: list every binding that grants cluster-admin, newest first.
+# READ: a binding you don't recognize — especially to a ServiceAccount or
+#   a user that isn't a platform admin — is the finding. Note its subjects.
+kubectl get clusterrolebindings -o json | jq -r '.items[] | select(.roleRef.name=="cluster-admin") | "\(.metadata.creationTimestamp)  \(.metadata.name)  subjects=\(.subjects)"' | sort
+```
+
+```bash
+# WHERE: shell with cluster-admin context.
+# WHAT: the audit-log entries for RBAC create/update in the last window.
+# READ: the `user.username` that made the change. A CI/GitOps identity =
+#   likely legitimate (confirm the PR). A workload SA or unknown user =
+#   escalation attempt.
+grep -E '"resource":"(clusterrolebindings|rolebindings|clusterroles)".*"verb":"(create|update|patch)"' /var/log/kubernetes/audit.log | tail -20
+```
+
+```bash
+# WHERE: shell with cluster-admin context.
+# WHAT: what a suspect subject can now do cluster-wide.
+# READ: if `can-i --list` for the subject shows * on * , it has full
+#   control — contain immediately.
+kubectl auth can-i --list --as=system:serviceaccount:<namespace>:<suspect-sa> 2>/dev/null | head
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Critical | Yes — page on-call | 15m | Cluster-wide compromise if attacker-driven |
+
+## Diagnostic steps
+
+1. **Identify the object** — which binding/role, granting what, to whom (queries above).
+2. **Attribute the change** — audit-log `user.username`; match against a reviewed PR/GitOps commit.
+3. **Decide** — legitimate platform change (confirmed PR) vs. unexplained.
+4. **Contain if unexplained** — delete the binding, revoke the subject's tokens, rotate any credentials it could have read, open IR.
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| New cluster-admin binding via GitOps | Matches a merged PR | Benign; confirm reviewer |
+| Binding created by a workload SA | Audit user = SA | Delete, revoke SA, IR |
+| Wildcard ClusterRole appears | roleRef verbs `*` | Scope it down; investigate origin |
+
+## Escalation
+
+1. **Security** — `@security-oncall` in `#mm-incidents` **first** — RBAC
+   escalation is a compromise indicator, not a config issue.
+2. **Platform** — `@sre-oncall` to confirm whether it maps to a reviewed change.
+
+## Required Prometheus labels
+
+Diagnostics use `k8s_ns_name` (Falco). **Requires audit logs + Falco k8s
+audit plugin** (or equivalent). Not observable from kube-state-metrics.
+
+## Related runbooks
+
+- [API Server Auth Failure Spike](apiserver-auth-failure-spike.md) — probing that precedes escalation.
+- [Interactive Shell in Container](interactive-shell-in-container.md) — the foothold escalation follows from.
diff --git a/runbooks/security-tooling-down.md b/runbooks/security-tooling-down.md
new file mode 100644
index 0000000..8766dca
--- /dev/null
+++ b/runbooks/security-tooling-down.md
@@ -0,0 +1,80 @@
+# Security Tooling Down
+
+!!! danger "Severity: critical"
+    **Target response: 15m.** A security sensor (Falco, Kyverno, or the
+    audit pipeline) stopped reporting. Your runtime/policy/audit-based
+    alerts are now blind — threats can occur without firing anything.
+
+## What this alert means
+
+Several security alerts depend on tooling that emits its own metrics:
+Falco (`interactive-shell`, `rbac-escalation`), Kyverno/Gatekeeper
+(`privileged-container`). If that tooling's scrape target goes down, those
+detections silently stop — the absence of alerts starts meaning nothing.
+
+```promql
+up{job=~"falco|kyverno.*|gatekeeper.*"} == 0
+```
+
+This is the meta-alert that watches the watchers. Without it, a Falco
+crash looks exactly like "all clear."
+
+## Quick diagnostics
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: state of the security tooling pods (Falco daemonset, Kyverno).
+# READ: CrashLoopBackOff / not Ready = the sensor is down. Fewer Falco
+#   pods than nodes = some nodes are unmonitored (per-node blind spots).
+kubectl get pods -n falco; kubectl get pods -n kyverno 2>/dev/null
+```
+
+```promql
+# WHERE: Grafana → Explore or Prometheus /graph.
+# WHAT: which security scrape targets are down, by job/instance.
+# READ: the job label tells you which sensor (falco vs kyverno). instance
+#   tells you which node/replica. All instances of a job down = full blind
+#   spot for that detection class.
+up{job=~"falco|kyverno.*|gatekeeper.*"} == 0
+```
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: recent logs from the down sensor to see why it stopped.
+# READ: OOMKilled = raise limits. Config/parse error = a bad rule/policy
+#   push. Crash on start = version/kernel mismatch (Falco eBPF).
+kubectl logs -n falco -l app.kubernetes.io/name=falco --tail=40 --previous 2>/dev/null
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Critical | Yes — page on-call | 15m | Security detections dark; blind to intrusions |
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| Falco pods OOMKilled | logs / OOM events | Raise memory limits; check ruleset size |
+| Kyverno not Ready | pod status | Restart; check webhook cert validity |
+| Bad rule/policy push | parse error in logs | Roll back the offending rule/policy |
+| Fewer Falco pods than nodes | daemonset desired vs ready | Fix node taints/scheduling for the DS |
+
+## Escalation
+
+1. **Security** — `@security-oncall` in `#mm-incidents` — you are operating
+   without detections until this is restored; treat as a security incident
+   if the outage overlaps any suspicious activity.
+2. **Platform** — `@sre-oncall` to restore the workload.
+
+## Required Prometheus labels
+
+Diagnostics use `job`, `instance` from `up`. Requires the security tooling
+(Falco/Kyverno) to expose Prometheus scrape targets.
+
+## Related runbooks
+
+- [Interactive Shell in Container](interactive-shell-in-container.md) — dark while Falco is down.
+- [RBAC Privilege Escalation](rbac-privilege-escalation.md) — dark while the audit pipeline is down.
+- [Privileged / Root Container Started](privileged-container-started.md) — dark while Kyverno is down.
diff --git a/runbooks/unexpected-container-image.md b/runbooks/unexpected-container-image.md
new file mode 100644
index 0000000..4739afe
--- /dev/null
+++ b/runbooks/unexpected-container-image.md
@@ -0,0 +1,96 @@
+# Unexpected Container Image
+
+!!! danger "Severity: warning"
+    **Target response: 30m.** A pod is running an image from outside the
+    approved registry allowlist — possible supply-chain compromise, a
+    mis-tagged deploy, or a workload that bypassed CI.
+
+## What this alert means
+
+Every image your workloads run should come from a known registry. This
+alert fires when `kube_pod_container_info` reports an image whose
+reference does not match the allowlist regex. Tighten the regex to your
+real registries before enabling.
+
+```promql
+# Any running container whose image is NOT from an approved registry.
+count by (namespace, pod, container, image) (
+  kube_pod_container_info{image!~"^(mattermost/|registry\\.mattermost\\.com/|ghcr\\.io/mattermost/).*"}
+) > 0
+```
+
+An image from an unexpected registry means code you didn't vet is
+running with your service accounts and network access — the classic
+foothold for lateral movement or data exfil.
+
+## Quick diagnostics
+
+```bash
+# WHERE: shell with kubectl context set to the affected cluster.
+# WHAT: show the exact image reference (with digest) the pod is running.
+# READ: compare the registry host against your allowlist. A public
+#   registry (docker.io/library/*, random ghcr users) on a prod workload
+#   is the red flag. Note the digest for the image-provenance check below.
+kubectl get pod -n <namespace> <pod> -o jsonpath='{range .spec.containers[*]}{.name}{" -> "}{.image}{"\n"}{end}'
+```
+
+```bash
+# WHERE: shell with kubectl context set.
+# WHAT: who created this pod and when — owner ref + creation timestamp.
+# READ: a Deployment/Job you recognize + a recent rollout = probably a
+#   mis-tagged deploy. No owner, or an unfamiliar creator = investigate
+#   as a rogue workload immediately.
+kubectl get pod -n <namespace> <pod> -o jsonpath='{.metadata.ownerReferences}{"\n"}{.metadata.creationTimestamp}{"\n"}'
+```
+
+```bash
+# WHERE: shell with cosign installed (if you sign images).
+# WHAT: verify the image signature against your public key.
+# READ: "no matching signatures" on a supposedly-internal image means it
+#   was NOT built by your pipeline. Treat as compromise until disproven.
+cosign verify --key cosign.pub $(kubectl get pod -n <namespace> <pod> -o jsonpath='{.spec.containers[0].image}')
+```
+
+## Severity & urgency
+
+| Severity | Pager? | Target response | Business impact |
+|---|---|---|---|
+| Warning | No — chat only | 30m | Unvetted code running with cluster access |
+
+Escalate to **critical/page** if the image is on a production namespace,
+signature verification fails, or the pod has no recognizable owner.
+
+## Diagnostic steps
+
+1. **Confirm it's still running** — `kubectl get pod -n <namespace> <pod>`.
+   If gone, capture the image ref from logs/audit before it's GC'd.
+2. **Identify the source** — owner ref → Deployment/Job → who applied it
+   (`kubectl get <owner> -o yaml`, check `kubectl.kubernetes.io/last-applied` + GitOps history).
+3. **Classify** — mis-tag (image *almost* matches, wrong host) vs. rogue
+   (unknown workload, public registry, failed signature).
+4. **Contain if rogue** — cordon nothing yet; scale the owning workload to
+   0 or delete the pod, revoke its service-account token, snapshot for IR.
+
+## Common causes & fixes
+
+| Symptom | Diagnosis | Fix |
+|---|---|---|
+| Image host is a typo of yours | Compare to allowlist | Fix the deploy manifest / CI tag |
+| Public registry on prod | `image` starts `docker.io/` | Re-pull through your registry; add admission policy |
+| Unknown workload, no owner | No ownerReferences | Contain, revoke SA, open IR ticket |
+
+## Escalation
+
+1. **Primary** — `@sre-oncall` in `#mm-incidents`.
+2. **Security** — `@security-oncall` immediately if signature fails or the
+   workload is unrecognized. This is a potential compromise, not a config bug.
+
+## Required Prometheus labels
+
+Diagnostics use: `namespace`, `pod`, `container`. Provided by
+`kube_pod_container_info` (kube-state-metrics).
+
+## Related runbooks
+
+- [Privileged Container Started](privileged-container-started.md) — often co-fires on a rogue workload.
+- [RBAC Privilege Escalation](rbac-privilege-escalation.md) — the next step an attacker takes after landing a pod.
diff --git a/server/cmd_docs.go b/server/cmd_docs.go
index fdf0715..ea24bde 100644
--- a/server/cmd_docs.go
+++ b/server/cmd_docs.go
@@ -20,6 +20,9 @@ const maxDocPostBytes = 14000
 // the embedded filename. Explicit map so we can rename embedded files
 // without breaking the user-facing argument vocabulary.
 var docTopics = map[string]string{
+	"alerts":         "ALERT_CATALOG.md",
+	"alert-catalog":  "ALERT_CATALOG.md",
+	"requirements":   "ALERT_REQUIREMENTS.md",
 	"architecture":   "ARCHITECTURE.md",
 	"configuration":  "CONFIGURATION.md",
 	"development":    "DEVELOPMENT.md",
diff --git a/server/cmd_scaffold.go b/server/cmd_scaffold.go
index 1ff194b..cc017dd 100644
--- a/server/cmd_scaffold.go
+++ b/server/cmd_scaffold.go
@@ -27,6 +27,9 @@ var scaffoldSets = map[string][]string{
 		"pod-not-ready",
 		"deployment-replicas-unavailable",
 		"node-not-ready",
+		"cpu-throttling-high",
+		"image-pull-backoff",
+		"pods-unschedulable",
 	},
 	"application": {
 		"high-http-error-rate",
@@ -38,6 +41,7 @@ var scaffoldSets = map[string][]string{
 		"database-connectivity-loss",
 		"database-replication-lag",
 		"database-high-latency",
+		"postgres-connections-near-max",
 	},
 	"storage": {
 		"persistent-volume-full",
@@ -52,6 +56,14 @@ var scaffoldSets = map[string][]string{
 		"prometheus-scrape-target-down",
 		"alertmanager-notification-failure",
 	},
+	"security": {
+		"unexpected-container-image",
+		"apiserver-auth-failure-spike",
+		"privileged-container-started",
+		"interactive-shell-in-container",
+		"rbac-privilege-escalation",
+		"security-tooling-down",
+	},
 }
 
 // handleAdd creates Mattermost incoming webhooks for the chosen target
diff --git a/server/commands.go b/server/commands.go
index d51a63f..c23a639 100644
--- a/server/commands.go
+++ b/server/commands.go
@@ -15,10 +15,10 @@ const (
 	helpMsg = "**Alertmanager bridge slash commands** _(channel-scoped — you only see receivers bound to this channel)_\n\n" +
 		"_All commands listed in alphabetical order to match the autocomplete dropdown._\n\n" +
 		"- `/alertmanager about` — plugin build info, configured settings, reconciler health, jump-off links\n" +
-		"- `/alertmanager add <team> <channel> <am-url> [target] [on]` — create receivers for a group set OR an individual runbook slug. Group sets: `all` (default), `application`, `compute`, `database`, `networking`, `observability`, `storage`. Each group share ONE Mattermost webhook; individual-slug adds get their own webhook. Trailing `on` opts these receivers INTO rotation reminders (configured via System Console → WebhookRotationDays).\n" +
+		"- `/alertmanager add <team> <channel> <am-url> [target] [on]` — create receivers for a group set OR an individual runbook slug. Group sets: `all` (default), `application`, `compute`, `database`, `networking`, `observability`, `security`, `storage`. Each group share ONE Mattermost webhook; individual-slug adds get their own webhook. Trailing `on` opts these receivers INTO rotation reminders (configured via System Console → WebhookRotationDays).\n" +
 		"- `/alertmanager alerts` — list currently firing alerts (grouped by Alertmanager URL — one section per backend, not per receiver)\n" +
 		"- `/alertmanager config <name>` — show full detail card + slack_configs YAML for one receiver\n" +
-		"- `/alertmanager docs [topic]` — embedded documentation (tab through topics: architecture, configuration, development, kubernetes, slash_commands)\n" +
+		"- `/alertmanager docs [topic]` — embedded documentation (tab through topics: alerts, requirements, architecture, configuration, development, kubernetes, slash_commands)\n" +
 		"- `/alertmanager expire_silence <name> <silence-id>` — expire a silence\n" +
 		"- `/alertmanager export [--diff-against-loaded]` — DM the assembled receivers.yml + routes.yml for this channel; with `--diff-against-loaded` (sysadmin) diff against AM's currently-loaded config\n" +
 		"- `/alertmanager help` — this message\n" +
@@ -76,12 +76,13 @@ func getAutocompleteData() *model.AutocompleteData {
 	add.AddDynamicListArgument("Mattermost channel URL slug — public channels in the chosen team (or type a new name to auto-create)", channelFetchURL, true)
 	add.AddTextArgument("Alertmanager API base URL (no trailing slash)", "[am-url]", "")
 	add.AddStaticListArgument("Group set OR individual runbook slug (defaults to `all`). Type a slug freely; static list shows group sets only.", false, []model.AutocompleteListItem{
-		{Item: "all", HelpText: "Every embedded runbook (default — 20 receivers in one shared webhook)"},
+		{Item: "all", HelpText: "Every embedded runbook (default — 30 receivers in one shared webhook)"},
 		{Item: "application", HelpText: "HTTP error, latency, endpoint, request-rate alerts (4) — one shared webhook"},
-		{Item: "compute", HelpText: "CPU, memory, pod, deployment, node alerts (6) — one shared webhook"},
-		{Item: "database", HelpText: "Connectivity, replication lag, query latency (3) — one shared webhook"},
+		{Item: "compute", HelpText: "CPU, memory, throttling, pod, deployment, node, image-pull, scheduling alerts (9) — one shared webhook"},
+		{Item: "database", HelpText: "Connectivity, replication lag, query latency, connection saturation (4) — one shared webhook"},
 		{Item: "networking", HelpText: "Ingress 5xx, cert expiry, DNS failure (3) — one shared webhook"},
 		{Item: "observability", HelpText: "Prometheus scrape down, Alertmanager notify fail (2) — one shared webhook"},
+		{Item: "security", HelpText: "Unexpected image, API auth spike, privileged container, shell-in-container, RBAC escalation, security tooling down (6) — one shared webhook"},
 		{Item: "storage", HelpText: "PV full, disk fill rate (2) — one shared webhook"},
 	})
 	add.AddStaticListArgument("Optional: enable webhook rotation reminders for these receivers", false, []model.AutocompleteListItem{
@@ -97,6 +98,8 @@ func getAutocompleteData() *model.AutocompleteData {
 
 	docs := model.NewAutocompleteData("docs", "[topic]", "Show embedded documentation in chat")
 	docs.AddStaticListArgument("Pick a topic", false, []model.AutocompleteListItem{
+		{Item: "alerts", HelpText: "Catalog of every alert type + severity + what fires it, by group"},
+		{Item: "requirements", HelpText: "Per-alert requirements: which metric/exporter/tooling each alert needs to fire"},
 		{Item: "architecture", HelpText: "Plugin design: why slack_configs over webhook_configs"},
 		{Item: "configuration", HelpText: "Plugin settings + alertmanager.yml structure"},
 		{Item: "development", HelpText: "Local build, test, deploy workflow"},
@@ -132,6 +135,7 @@ func getAutocompleteData() *model.AutocompleteData {
 		{Item: "database", HelpText: "All database receivers in this channel (requires --force)"},
 		{Item: "networking", HelpText: "All networking receivers in this channel (requires --force)"},
 		{Item: "observability", HelpText: "All observability receivers in this channel (requires --force)"},
+		{Item: "security", HelpText: "All security receivers in this channel (requires --force)"},
 		{Item: "storage", HelpText: "All storage receivers in this channel (requires --force)"},
 	})
 	remove.AddStaticListArgument("Pass `--force` to confirm bulk delete", false, []model.AutocompleteListItem{
@@ -158,6 +162,7 @@ func getAutocompleteData() *model.AutocompleteData {
 		{Item: "database", HelpText: "Database receivers only"},
 		{Item: "networking", HelpText: "Networking receivers only"},
 		{Item: "observability", HelpText: "Observability receivers only"},
+		{Item: "security", HelpText: "Security receivers only"},
 		{Item: "storage", HelpText: "Storage receivers only"},
 	})
 	validate.AddStaticListArgument("Optional mode flag — pick how validate runs", false, []model.AutocompleteListItem{

From 5bf1c1f76e9e830592ccb04320807e6c88fc2be7 Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Mon, 13 Jul 2026 13:21:47 -0500
Subject: [PATCH 02/15] docs: reflect 30-runbook catalog, security set, and new
 docs topics

README (20->30 runbooks, +security category), SLASH_COMMANDS (security group set + alerts/requirements docs topics + seven-set count), QUICKSTART (security set), CONFIGURATION (security label profile + docs requirements pointer).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 README.md              |  8 ++++----
 docs/CONFIGURATION.md  |  6 ++++--
 docs/QUICKSTART.md     |  2 +-
 docs/SLASH_COMMANDS.md | 15 ++++++++-------
 4 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index 8a1ecc5..bb67e0b 100644
--- a/README.md
+++ b/README.md
@@ -66,9 +66,9 @@ The diagnostic content above is parsed from the embedded runbook file
 [`runbooks/high-cpu-usage.md`](runbooks/high-cpu-usage.md) at YAML-render
 time. Placeholders like `<namespace>` and `<pod>` are substituted by
 Alertmanager at alert delivery time using the alert's actual labels.
-Each of the 20 shipped runbooks (compute / application / database /
-storage / networking / observability) follows the same WHERE / WHAT /
-READ structure.
+Each of the 30 shipped runbooks (compute / application / database /
+storage / networking / observability / security) follows the same
+WHERE / WHAT / READ structure.
 
 ## Two-minute setup
 
@@ -147,7 +147,7 @@ Alertmanager templates the post body — including the inlined Quick
 diagnostics from the runbook — and POSTs to the Mattermost webhook
 URL. The plugin doesn't intercept alert payloads at runtime; it
 only owns the YAML generation, webhook lifecycle, and the inventory
-page. The 20 runbook markdown files ship embedded in the plugin
+page. The 30 runbook markdown files ship embedded in the plugin
 binary via Go's `embed.FS`.
 
 For deeper context: [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md)
diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md
index 7e09c16..2a94209 100644
--- a/docs/CONFIGURATION.md
+++ b/docs/CONFIGURATION.md
@@ -239,8 +239,10 @@ Each shipped runbook documents the labels its Quick diagnostics
 section expects in a "Required Prometheus labels" footer. Most
 expect at least `namespace` and `pod` (for compute, application,
 storage runbooks) plus the runbook-specific label like `instance` /
-`job` / `service`. See the corresponding `runbooks/*.md` file for
-the exact list.
+`job` / `service`. Security runbooks differ — several use apiserver /
+Falco labels (e.g. `k8s_ns_name`, `job`) instead. See the
+corresponding `runbooks/*.md` file, or run `/alertmanager docs
+requirements` for the full per-alert metric / label / tooling matrix.
 
 If a label is missing on an incoming alert, the template falls back
 to leaving the placeholder text in place — operators see
diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md
index 57e7283..d0e88c1 100644
--- a/docs/QUICKSTART.md
+++ b/docs/QUICKSTART.md
@@ -102,7 +102,7 @@ In the channel where you want alerts, run:
 /alertmanager add <team> <channel> <alertmanager-url> <target>
 ```
 - `<target>` is a **runbook set** (`all`, `compute`, `application`, `database`,
-  `storage`, `networking`, `observability`) or a **single runbook slug**
+  `storage`, `networking`, `observability`, `security`) or a **single runbook slug**
   (e.g. `pod-crashloopbackoff`).
 - Add a trailing `on` to opt these receivers into rotation reminders.
 
diff --git a/docs/SLASH_COMMANDS.md b/docs/SLASH_COMMANDS.md
index d446fa0..cd6fc37 100644
--- a/docs/SLASH_COMMANDS.md
+++ b/docs/SLASH_COMMANDS.md
@@ -12,7 +12,7 @@ the channel where you ran the command.
 | `add` | `<team> <channel> <am-url> [target] [on]` | Create receivers for a group set OR individual runbook slug. One shared Mattermost webhook per add invocation. DMs assembled `receivers.yml` + `routes.yml`. Trailing `on` opts in to rotation reminders. | sysadmin / team_admin |
 | `alerts` | _(none)_ | Currently firing alerts, grouped by Alertmanager URL | any user |
 | `config` | `<name>` | Detail card + `slack_configs` YAML for one receiver | sysadmin / team_admin |
-| `docs` | `[topic]` | List or print embedded docs (architecture / configuration / development / kubernetes / rotation / slash_commands) | any user |
+| `docs` | `[topic]` | List or print embedded docs (alerts / architecture / configuration / development / kubernetes / requirements / rotation / slash_commands) | any user |
 | `expire_silence` | `<name> <silence-id>` | Expire an active Alertmanager silence | any user |
 | `export` | `[--diff-against-loaded]` | DM a fresh `receivers.yml` + `routes.yml` for this channel. With `--diff-against-loaded`, diff against AM's loaded config + schema-validate. | sysadmin / team_admin |
 | `help` | _(none)_ | Slash-command reference | any user |
@@ -93,8 +93,8 @@ To create just one category:
 ```
 
 Available group sets: `all` (default), `application`, `compute`,
-`database`, `networking`, `observability`, `storage`. Each set add
-creates one shared webhook named `Alertmanager: <set>--<channel>`.
+`database`, `networking`, `observability`, `security`, `storage`. Each
+set add creates one shared webhook named `Alertmanager: <set>--<channel>`.
 
 ### Add one specific runbook (individual)
 
@@ -319,9 +319,9 @@ dry-run preview of what would be deleted, then re-run with `--force`
 to confirm. Single-receiver remove doesn't require `--force` because
 the name is explicit (low blast radius).
 
-The `remove` autocomplete dropdown shows `all` plus the six category
+The `remove` autocomplete dropdown shows `all` plus the seven category
 sets (compute, application, database, storage, networking,
-observability) — pick one or type a receiver name freely.
+observability, security) — pick one or type a receiver name freely.
 
 ### Find orphaned receivers
 
@@ -343,8 +343,9 @@ Reports how many orphans (if any) it removed.
 /alertmanager docs rotation
 ```
 
-Available topics: `architecture`, `configuration`, `development`,
-`kubernetes`, `rotation`, `slash_commands`. Content is rendered
+Available topics: `alerts`, `architecture`, `configuration`,
+`development`, `kubernetes`, `requirements`, `rotation`,
+`slash_commands`. Content is rendered
 inline in chat — for the full version, hit
 `/plugins/com.mattermost.alertmanager/public/help/home.html` in a
 browser.

From de94964b6165c06184b0e753139ffa396f43beab Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Mon, 13 Jul 2026 13:23:19 -0500
Subject: [PATCH 03/15] docs: fix remaining 20->30 runbook/receiver count
 references

Sweep of README + docs (DEVELOPMENT smoke-test steps, KUBERNETES route comment, CONFIGURATION, SLASH_COMMANDS worked examples) missed in the first pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 README.md              | 2 +-
 docs/CONFIGURATION.md  | 2 +-
 docs/DEVELOPMENT.md    | 6 +++---
 docs/KUBERNETES.md     | 2 +-
 docs/SLASH_COMMANDS.md | 4 ++--
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index bb67e0b..8e6f721 100644
--- a/README.md
+++ b/README.md
@@ -81,7 +81,7 @@ WHERE / WHAT / READ structure.
    /alertmanager add ops alerts http://alertmanager:9093 all
    ```
 
-   The plugin creates 20 Mattermost incoming webhooks (one per
+   The plugin creates 30 Mattermost incoming webhooks (one per
    runbook) bound to this channel and DMs you the assembled
    `receivers.yml` + `routes.yml`.
 
diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md
index 2a94209..161896e 100644
--- a/docs/CONFIGURATION.md
+++ b/docs/CONFIGURATION.md
@@ -256,7 +256,7 @@ That walks AM's route tree against your label set without firing a
 real alert.
 
 `samples/prometheus-rules.yaml` ships a complete rule set covering
-all 20 runbooks with the correct label patterns. Use it as a
+all 30 runbooks with the correct label patterns. Use it as a
 starting point.
 
 ## Receiver naming convention
diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md
index d2aae16..e788845 100644
--- a/docs/DEVELOPMENT.md
+++ b/docs/DEVELOPMENT.md
@@ -88,9 +88,9 @@ Useful surrounding targets: `make disable`, `make enable`, `make reset`, `make c
 Once the plugin is loaded:
 
 1. **Plugin alive:** `/alertmanager about` returns version info.
-2. **Add the canonical set:** `/alertmanager add testing alerts-smoke http://localhost:9093`. Plugin creates 20 receivers, DMs you the assembled YAML files.
+2. **Add the canonical set:** `/alertmanager add testing alerts-smoke http://localhost:9093`. Plugin creates 30 receivers, DMs you the assembled YAML files.
 3. **Verify webhooks created:** Open System Console → Integrations → Incoming Webhooks → see entries named `Alertmanager: <slug>--alerts-smoke`.
-4. **List receivers in the channel:** `/alertmanager list` from `~alerts-smoke`. Should show all 20 in a table.
+4. **List receivers in the channel:** `/alertmanager list` from `~alerts-smoke`. Should show all 30 in a table.
 5. **Detail card for one:** `/alertmanager config high-cpu-usage` from `~alerts-smoke`. Should show metadata + slack_configs YAML.
 6. **Test the post path:**
    ```bash
@@ -101,7 +101,7 @@ Once the plugin is loaded:
    ```
    Post lands in `~alerts-smoke` as `@alertmanagerbot`.
 7. **Re-export:** `/alertmanager export` should produce the same receivers + routes YAML as the original add.
-8. **Bulk cleanup:** `/alertmanager remove all --force` deletes all 20 receivers and their webhooks.
+8. **Bulk cleanup:** `/alertmanager remove all --force` deletes all 30 receivers and their webhooks.
 
 ## Verifying persisted state
 
diff --git a/docs/KUBERNETES.md b/docs/KUBERNETES.md
index ab0df26..c7e322c 100644
--- a/docs/KUBERNETES.md
+++ b/docs/KUBERNETES.md
@@ -81,7 +81,7 @@ route:
     - matchers: [runbook="pod-crashloopbackoff"]
       receiver: pod-crashloopbackoff--alert-slo-channel
       continue: true
-    # ... one route per receiver, 20 total for the standard set
+    # ... one route per receiver, 30 total for the standard set
 ```
 
 `continue: true` on every plugin-generated route is what makes fan-out
diff --git a/docs/SLASH_COMMANDS.md b/docs/SLASH_COMMANDS.md
index cd6fc37..c9f59d0 100644
--- a/docs/SLASH_COMMANDS.md
+++ b/docs/SLASH_COMMANDS.md
@@ -72,13 +72,13 @@ routes with `continue: true` so AM doesn't stop at the first match.
 
 ## Worked examples
 
-### First-time setup — all 20 canonical receivers in one channel
+### First-time setup — all 30 canonical receivers in one channel
 
 ```
 /alertmanager add testing alert-slo-channel http://alertmanager:9093
 ```
 
-Creates every embedded runbook (the `all` set — 20 receivers) bound
+Creates every embedded runbook (the `all` set — 30 receivers) bound
 to `~alert-slo-channel` **behind one shared Mattermost webhook**.
 The summary lands in the channel; the assembled
 `alertmanager-receivers.yml` + `alertmanager-routes.yml` land in

From 43dd3f3dd37ba2607c9a6109916dc5871b37d390 Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 08:35:03 -0500
Subject: [PATCH 04/15] test: verify every scaffold runbook renders valid
 Alertmanager config

For each slug in scaffoldSets, asserts renderReceiverYAML produces config Alertmanager's own parser accepts, with Quick Diagnostics injected and no unresolved plugin placeholders. Closes the gap where a malformed runbook only surfaced at /alertmanager add time.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 server/runbook_render_test.go | 78 +++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 server/runbook_render_test.go

diff --git a/server/runbook_render_test.go b/server/runbook_render_test.go
new file mode 100644
index 0000000..ea3a111
--- /dev/null
+++ b/server/runbook_render_test.go
@@ -0,0 +1,78 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+// TestScaffoldRunbooksRender guards the render path for every runbook the
+// scaffold can offer. For each slug it asserts the generated slack_configs
+// receiver:
+//
+//	(a) leaves no plugin placeholder ({{NAME}}, {{QUICK_DIAGNOSTICS}}, ...)
+//	    unresolved,
+//	(b) has its inline Quick Diagnostics block injected — which also proves
+//	    the runbook .md exists and has a "## Quick diagnostics" section, and
+//	(c) parses as valid Alertmanager config (via AM's own loader).
+//
+// Without this, a malformed fence or an odd <placeholder> in a new runbook
+// only surfaces at `/alertmanager add` time, never in CI. This exercises the
+// otherwise-untested renderReceiverYAML path across all shipped runbooks.
+func TestScaffoldRunbooksRender(t *testing.T) {
+	// Unique slugs across every category set. `all` is nil (resolved at
+	// runtime from the embedded FS), so skip it — the category lists
+	// enumerate the same slugs explicitly.
+	seen := map[string]bool{}
+	var slugs []string
+	for set, list := range scaffoldSets {
+		if set == "all" {
+			continue
+		}
+		for _, s := range list {
+			if !seen[s] {
+				seen[s] = true
+				slugs = append(slugs, s)
+			}
+		}
+	}
+	if len(slugs) == 0 {
+		t.Fatal("scaffoldSets resolved to zero slugs — map misconfigured")
+	}
+
+	placeholders := []string{
+		"{{NAME}}", "{{URL}}", "{{CHANNEL}}",
+		"{{RUNBOOK_DEFAULT}}", "{{ICON_URL}}", "{{QUICK_DIAGNOSTICS}}",
+	}
+
+	for _, slug := range slugs {
+		t.Run(slug, func(t *testing.T) {
+			block := renderReceiverYAML(
+				slug,
+				"http://localhost:8065/hooks/aaaaaaaaaaaaaaaaaaaaaaaaaa",
+				"alerts",
+				"http://localhost:8065/plugins/com.mattermost.alertmanager/public/runbooks/"+slug+".html",
+				"http://localhost:8065/plugins/com.mattermost.alertmanager/public/alertmanager-logo.png",
+			)
+
+			// (a) every plugin placeholder was substituted.
+			for _, ph := range placeholders {
+				if strings.Contains(block, ph) {
+					t.Errorf("unresolved placeholder %s in rendered receiver", ph)
+				}
+			}
+
+			// (b) inline diagnostics were injected.
+			if !strings.Contains(block, "**Quick diagnostics:**") {
+				t.Errorf("no Quick diagnostics rendered — missing runbook file or section?")
+			}
+
+			// (c) Alertmanager can parse it. Wrap the receiver (which
+			// begins with a column-0 `- name:` sequence, valid directly
+			// under `receivers:`) in a minimal config that references it.
+			cfg := "route:\n  receiver: " + slug + "\nreceivers:\n" + block
+			if err := validateMergedConfig(cfg); err != nil {
+				t.Errorf("rendered receiver is not valid Alertmanager config: %v\n---\n%s", err, cfg)
+			}
+		})
+	}
+}

From 63b821b4c425f19ad2101dd7023bdaa26469ba45 Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 08:43:21 -0500
Subject: [PATCH 05/15] test: add catalog consistency + placeholder +
 docs-topic guards

Tier-1 pure-Go guards (run in make test, no infra): (1) scaffoldSets <-> runbook files match both ways (drift guard); (2) every Quick-Diagnostics <placeholder> is an allowlisted label or a known literal (catches typo'd labels that render literally); (3) every /alertmanager docs topic resolves to an embedded file; (4) every scaffold category is surfaced in help + autocomplete.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 server/catalog_consistency_test.go | 162 +++++++++++++++++++++++++++++
 1 file changed, 162 insertions(+)
 create mode 100644 server/catalog_consistency_test.go

diff --git a/server/catalog_consistency_test.go b/server/catalog_consistency_test.go
new file mode 100644
index 0000000..eb3f456
--- /dev/null
+++ b/server/catalog_consistency_test.go
@@ -0,0 +1,162 @@
+package main
+
+import (
+	"io/fs"
+	"strings"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+
+	root "github.com/mattermost/mattermost-plugin-alertmanager"
+)
+
+// runbookSlugsFromFS returns the slug of every embedded runbook, excluding
+// the non-runbook TEMPLATE.md and INDEX.md.
+func runbookSlugsFromFS(t *testing.T) []string {
+	t.Helper()
+	paths, err := fs.Glob(root.RunbooksFS, "runbooks/*.md")
+	if err != nil {
+		t.Fatalf("glob embedded runbooks: %v", err)
+	}
+	var slugs []string
+	for _, p := range paths {
+		base := strings.TrimSuffix(strings.TrimPrefix(p, "runbooks/"), ".md")
+		if base == "TEMPLATE" || base == "INDEX" {
+			continue
+		}
+		slugs = append(slugs, base)
+	}
+	if len(slugs) == 0 {
+		t.Fatal("no embedded runbooks found — embed pattern broken?")
+	}
+	return slugs
+}
+
+// scaffoldSlugSet is the union of every category set's slugs ("all" excluded —
+// it's resolved at runtime from the embedded FS, not enumerated here).
+func scaffoldSlugSet() map[string]bool {
+	set := map[string]bool{}
+	for name, list := range scaffoldSets {
+		if name == "all" {
+			continue
+		}
+		for _, s := range list {
+			set[s] = true
+		}
+	}
+	return set
+}
+
+// TestScaffoldMatchesRunbookFiles is the drift guard: the set of runbooks the
+// scaffold offers must equal the set of runbooks that actually ship. Catches
+// both directions — a scaffoldSets entry with no runbook file (add would fail
+// / link 404s), and a shipped runbook missing from every category (only
+// reachable via `all`, invisible to per-category add/remove/validate).
+func TestScaffoldMatchesRunbookFiles(t *testing.T) {
+	files := map[string]bool{}
+	for _, s := range runbookSlugsFromFS(t) {
+		files[s] = true
+	}
+	scaffold := scaffoldSlugSet()
+
+	for slug := range scaffold {
+		if !files[slug] {
+			t.Errorf("scaffoldSets references %q but runbooks/%s.md does not exist", slug, slug)
+		}
+	}
+	for slug := range files {
+		if !scaffold[slug] {
+			t.Errorf("runbook %q ships but is not in any scaffoldSets category (unreachable via per-category commands)", slug)
+		}
+	}
+}
+
+// knownLiteralPlaceholders are `<tag>`-shaped strings that legitimately appear
+// as literal command text, not as label placeholders. kubectl prints "<none>"
+// for empty fields, so a runbook may grep for it. Anything matching the
+// placeholder regex that is neither an allowlisted label nor a known literal is
+// flagged — it's almost certainly a typo'd label (e.g. <podname> for <pod>)
+// that would render literally instead of substituting. A genuinely new literal
+// gets added here as a reviewed, one-line decision.
+var knownLiteralPlaceholders = map[string]bool{
+	"none": true, // kubectl's empty-field marker
+}
+
+// TestRunbookPlaceholdersAllowlisted scans the Quick Diagnostics of every
+// runbook for `<label>` placeholders on executable (non-comment) lines and
+// fails if any is neither an allowlisted label nor a known literal. An
+// un-allowlisted tag like <podname> isn't substituted — it renders as the
+// literal text "<podname>" in the alert post instead of the value. This is the
+// only check that catches that silent typo.
+func TestRunbookPlaceholdersAllowlisted(t *testing.T) {
+	for _, slug := range runbookSlugsFromFS(t) {
+		blocks := loadQuickDiagnosticsForSlug(slug)
+		for _, blk := range blocks {
+			for _, line := range strings.Split(blk.Code, "\n") {
+				// Mirror substituteLabelPlaceholders: comment lines are
+				// skipped (they reference labels in prose, not commands).
+				trim := strings.TrimLeft(line, " \t")
+				if strings.HasPrefix(trim, "#") || strings.HasPrefix(trim, "--") {
+					continue
+				}
+				for _, m := range labelPlaceholderRegex.FindAllStringSubmatch(line, -1) {
+					name := m[1]
+					if !labelPlaceholderAllowlist[name] && !knownLiteralPlaceholders[name] {
+						t.Errorf("runbook %q: placeholder <%s> is neither an allowlisted label nor a known literal — it will render literally, not substituted (line: %q). If it's a real label, add it to labelPlaceholderAllowlist; if intentional literal text, add it to knownLiteralPlaceholders.", slug, name, strings.TrimSpace(line))
+					}
+				}
+			}
+		}
+	}
+}
+
+// TestDocTopicsResolve asserts every registered `/alertmanager docs <topic>`
+// maps to a real embedded doc file. Catches a topic pointing at a renamed or
+// missing file (which would 500 the command at runtime).
+func TestDocTopicsResolve(t *testing.T) {
+	for topic, filename := range docTopics {
+		if _, err := root.DocsFS.ReadFile("docs/" + filename); err != nil {
+			t.Errorf("docs topic %q -> %q is not an embedded doc: %v", topic, filename, err)
+		}
+	}
+}
+
+// collectStaticListItems walks an AutocompleteData tree and returns every
+// static-list item value it finds (across all subcommands and args).
+func collectStaticListItems(data *model.AutocompleteData) []string {
+	var items []string
+	for _, arg := range data.Arguments {
+		if sl, ok := arg.Data.(*model.AutocompleteStaticListArg); ok {
+			for _, it := range sl.PossibleArguments {
+				items = append(items, it.Item)
+			}
+		}
+	}
+	for _, sub := range data.SubCommands {
+		items = append(items, collectStaticListItems(sub)...)
+	}
+	return items
+}
+
+// TestScaffoldCategoriesDocumented guards against the count/label drift that
+// staled the docs repeatedly: every scaffoldSets category must be surfaced in
+// both the help text and the command autocomplete, so adding a category can't
+// silently leave it undiscoverable.
+func TestScaffoldCategoriesDocumented(t *testing.T) {
+	autocompleteItems := map[string]bool{}
+	for _, it := range collectStaticListItems(getAutocompleteData()) {
+		autocompleteItems[it] = true
+	}
+
+	for name := range scaffoldSets {
+		if name == "all" {
+			continue
+		}
+		if !strings.Contains(helpMsg, "`"+name+"`") {
+			t.Errorf("scaffold category %q is not mentioned in helpMsg", name)
+		}
+		if !autocompleteItems[name] {
+			t.Errorf("scaffold category %q is not offered in any command autocomplete static list", name)
+		}
+	}
+}

From fd3841e6b540d5831a1d7e68b6317f3a96debb28 Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 08:46:59 -0500
Subject: [PATCH 06/15] docs(readme): fix badge URLs, add make test + docs
 discoverability

Badges pointed at the old christopherfickess fork (broken) -> mattermost-community. Add 'make test' to the build block + a DEVELOPMENT.md link, and a 'Docs & help' section pointing to /alertmanager docs topics (alerts/requirements/...), the runbook INDEX, and the setup/ops docs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 README.md | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 8e6f721..de1619e 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,12 @@
 # Mattermost Alertmanager Plugin
 
-[![Lint](https://github.com/christopherfickess/mattermost-plugin-alertmanager/actions/workflows/lint.yml/badge.svg?branch=main)](https://github.com/christopherfickess/mattermost-plugin-alertmanager/actions/workflows/lint.yml)
-[![Test](https://github.com/christopherfickess/mattermost-plugin-alertmanager/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/christopherfickess/mattermost-plugin-alertmanager/actions/workflows/test.yml)
-[![Build](https://github.com/christopherfickess/mattermost-plugin-alertmanager/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/christopherfickess/mattermost-plugin-alertmanager/actions/workflows/build.yml)
-[![Security](https://github.com/christopherfickess/mattermost-plugin-alertmanager/actions/workflows/security.yml/badge.svg?branch=main)](https://github.com/christopherfickess/mattermost-plugin-alertmanager/actions/workflows/security.yml)
-[![Latest release](https://img.shields.io/github/v/release/christopherfickess/mattermost-plugin-alertmanager?sort=semver)](https://github.com/christopherfickess/mattermost-plugin-alertmanager/releases/latest)
-[![Go version](https://img.shields.io/github/go-mod/go-version/christopherfickess/mattermost-plugin-alertmanager)](https://github.com/christopherfickess/mattermost-plugin-alertmanager/blob/main/go.mod)
-[![License](https://img.shields.io/github/license/christopherfickess/mattermost-plugin-alertmanager)](./LICENSE)
+[![Lint](https://github.com/mattermost-community/mattermost-plugin-alertmanager/actions/workflows/lint.yml/badge.svg?branch=main)](https://github.com/mattermost-community/mattermost-plugin-alertmanager/actions/workflows/lint.yml)
+[![Test](https://github.com/mattermost-community/mattermost-plugin-alertmanager/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/mattermost-community/mattermost-plugin-alertmanager/actions/workflows/test.yml)
+[![Build](https://github.com/mattermost-community/mattermost-plugin-alertmanager/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/mattermost-community/mattermost-plugin-alertmanager/actions/workflows/build.yml)
+[![Security](https://github.com/mattermost-community/mattermost-plugin-alertmanager/actions/workflows/security.yml/badge.svg?branch=main)](https://github.com/mattermost-community/mattermost-plugin-alertmanager/actions/workflows/security.yml)
+[![Latest release](https://img.shields.io/github/v/release/mattermost-community/mattermost-plugin-alertmanager?sort=semver)](https://github.com/mattermost-community/mattermost-plugin-alertmanager/releases/latest)
+[![Go version](https://img.shields.io/github/go-mod/go-version/mattermost-community/mattermost-plugin-alertmanager)](https://github.com/mattermost-community/mattermost-plugin-alertmanager/blob/main/go.mod)
+[![License](https://img.shields.io/github/license/mattermost-community/mattermost-plugin-alertmanager)](./LICENSE)
 
 **When `PostgresReplicationLagHigh` fires into your channel at 3am,
 the runbook's first three diagnostic commands appear inline with the
@@ -118,6 +118,14 @@ only) provides an org-wide view including AM reachability badges,
 inverse drift detection (receivers in AM that the plugin doesn't
 track), and a route simulator form.
 
+## Docs & help — finding your way around
+
+Everything is discoverable from chat and the repo:
+
+- **In chat:** `/alertmanager help` (command reference) and `/alertmanager docs <topic>` — embedded docs rendered inline. Topics: `alerts` (full alert catalog by group), `requirements` (per-alert metric / exporter / tooling), `slash_commands`, `architecture`, `configuration`, `kubernetes`, `rotation`.
+- **Runbook library:** [`runbooks/INDEX.md`](runbooks/INDEX.md) — all 30 runbooks by category; each renders to a page linked from its alert.
+- **Setup & ops docs:** [`docs/QUICKSTART.md`](docs/QUICKSTART.md) (first alert end-to-end), [`docs/CONFIGURATION.md`](docs/CONFIGURATION.md) (settings + Prometheus wiring), [`docs/KUBERNETES.md`](docs/KUBERNETES.md) (HA deploy), [`docs/DEVELOPMENT.md`](docs/DEVELOPMENT.md) (build/test/deploy), [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md).
+
 ## Key features for daily use
 
 - **Inline diagnostics** — every alert post includes 3 runbook-specific
@@ -159,12 +167,14 @@ settings.
 ## Building from source
 
 ```bash
+make test          # run the Go test suite — no server needed, sub-second
 make dist          # cross-compile all platforms + assemble tarball
 make dist-host     # current host's arch only (faster for dev iter)
 make deploy-local  # build + upload to localhost:8065
 ```
 
 Requires export `MM_ADMIN_TOKEN` and `MM_SERVICESETTINGS_SITEURL` for deploy targets.
+Full build / test / deploy / iterate workflow: [`docs/DEVELOPMENT.md`](docs/DEVELOPMENT.md).
 
 ## Contributing
 

From 14969275900e6efdff1b65d71174f816ca709408 Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 08:49:42 -0500
Subject: [PATCH 07/15] test: use strings.SplitSeq to satisfy modernize linter

golangci-lint (modernize) flagged strings.Split in a range; match the repo convention (SplitSeq, as in runbook_diagnostics.go).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 server/catalog_consistency_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/catalog_consistency_test.go b/server/catalog_consistency_test.go
index eb3f456..95d3cad 100644
--- a/server/catalog_consistency_test.go
+++ b/server/catalog_consistency_test.go
@@ -92,7 +92,7 @@ func TestRunbookPlaceholdersAllowlisted(t *testing.T) {
 	for _, slug := range runbookSlugsFromFS(t) {
 		blocks := loadQuickDiagnosticsForSlug(slug)
 		for _, blk := range blocks {
-			for _, line := range strings.Split(blk.Code, "\n") {
+			for line := range strings.SplitSeq(blk.Code, "\n") {
 				// Mirror substituteLabelPlaceholders: comment lines are
 				// skipped (they reference labels in prose, not commands).
 				trim := strings.TrimLeft(line, " \t")

From caf692e3f0466192bb6a64c92c7d9c1528de7cf3 Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 08:53:48 -0500
Subject: [PATCH 08/15] docs(readme): fix nested code fences in the alert
 example

The 'What an alert looks like in chat' block nested ```bash/```promql fences inside a 3-backtick fence, so markdown closed the outer block early and the Quick Diagnostics rendered outside it. Bump the outer fence to 4 backticks so the inner fences render literally and the full example (incl. inline diagnostics) shows correctly.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index de1619e..b752e8d 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ but the runbook-at-fire-time experience is why your on-call wants it.
 
 ## What an alert looks like in chat
 
-```
+````
 [WARNING:HighCPUUsage] (namespace=billing, pod=api-7d9-2xfgs)
 
 **Alert:** HighCPUUsage - warning
@@ -60,7 +60,7 @@ rate(container_cpu_cfs_throttled_seconds_total[5m])
 #   CPU alert started after, you've found the cause...
 kubectl rollout history deployment -n billing --limit 5
 ```
-```
+````
 
 The diagnostic content above is parsed from the embedded runbook file
 [`runbooks/high-cpu-usage.md`](runbooks/high-cpu-usage.md) at YAML-render

From 93fd0449432eee523c8e23293183babe99e3f85a Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 09:21:17 -0500
Subject: [PATCH 09/15] feat: complete sample rules to 31 rules (all 30
 runbooks) + CI validation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bring samples/prometheus-rules.yaml up to the full catalog: add the 11
rules missing since the security category + SRE gap-fill runbooks landed
(4 compute: cpu-throttling-high, container-oom, image-pull-backoff,
pods-unschedulable; 1 database: postgres-connections-near-max; 6 new
security: unexpected-container-image, apiserver-auth-failure-spike,
privileged-container-started, interactive-shell-in-container,
rbac-privilege-escalation, security-tooling-down).

31 rules cover 30 runbooks — ContainerOOMKilled reuses the
high-memory-usage runbook (same remediation).

Add a validate-rules job to test.yml running `promtool check rules` on
the shipped sample on every PR, so a broken PromQL expr or malformed
YAML in the file users copy can never merge.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .github/workflows/test.yml    |  19 ++++
 samples/prometheus-rules.yaml | 198 +++++++++++++++++++++++++++++++++-
 2 files changed, 215 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0b44563..8f79b9a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -31,3 +31,22 @@ jobs:
 
       - name: Run unit tests
         run: go test -race -count=1 ./...
+
+  # Validates the shipped sample rules (samples/prometheus-rules.yaml) with
+  # promtool. Users copy this file into their own Prometheus, so a broken
+  # PromQL expr or malformed YAML must never merge. promtool ships inside the
+  # prom/prometheus image (its entrypoint is /bin/prometheus, hence the
+  # --entrypoint override); the tag is pinned to match the version the
+  # deploy-local verify-rules.sh gate uses.
+  validate-rules:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: promtool check rules
+        run: |
+          docker run --rm --entrypoint promtool \
+            -v "${{ github.workspace }}/samples":/w -w /w \
+            prom/prometheus:v3.5.0 \
+            check rules /w/prometheus-rules.yaml
diff --git a/samples/prometheus-rules.yaml b/samples/prometheus-rules.yaml
index ce9d1ea..a77c467 100644
--- a/samples/prometheus-rules.yaml
+++ b/samples/prometheus-rules.yaml
@@ -1,5 +1,5 @@
-# Sample Prometheus alert rules covering all 20 canonical runbooks
-# shipped by the mattermost-plugin-alertmanager plugin.
+# Sample Prometheus alert rules covering all 30 canonical runbooks
+# (including the security category) shipped by the mattermost-plugin-alertmanager plugin.
 #
 # WHERE THIS FILE LIVES
 # ---------------------
@@ -191,6 +191,73 @@ groups:
           summary: "Node {{ $labels.node }} not Ready for 5m"
           description: "A NotReady node is a candidate for cordon + drain. Investigate kubelet, networking, and disk pressure."
 
+      # CPUThrottlingHigh — labels required: namespace, pod, container.
+      # CFS throttling as a fraction of scheduler periods. Catches
+      # latency from hitting the CPU *limit* even when average usage
+      # looks modest — invisible to HighCPUUsage.
+      - alert: CPUThrottlingHigh
+        expr: |
+          sum by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total[5m]))
+          /
+          sum by (namespace, pod, container) (rate(container_cpu_cfs_periods_total[5m])) > 0.25
+        for: 10m
+        labels:
+          severity: warning
+          category: compute
+          runbook: cpu-throttling-high
+        annotations:
+          summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) CPU-throttled > 25% for 10m"
+          description: "Throttled by the CFS scheduler — raise the CPU limit or the request/limit ratio."
+
+      # ContainerOOMKilled — labels required: namespace, pod, container.
+      # Distinct from a crashloop: the kernel OOM-killer reaped the
+      # container for exceeding its memory limit. Reuses the
+      # high-memory-usage runbook (same remediation).
+      - alert: ContainerOOMKilled
+        expr: |
+          increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) > 0
+        for: 0m
+        labels:
+          severity: critical
+          category: compute
+          runbook: high-memory-usage
+        annotations:
+          summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) was OOMKilled"
+          description: "Container hit its memory limit and was killed. Raise the limit or fix the leak."
+
+      # PodImagePullBackOff — labels required: namespace, pod.
+      # Container can't pull its image (bad tag, missing/expired pull
+      # secret, registry issue). The pod never starts — vs a crashloop
+      # where it starts then dies.
+      - alert: PodImagePullBackOff
+        expr: |
+          max by (namespace, pod) (
+            kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}
+          ) > 0
+        for: 5m
+        labels:
+          severity: critical
+          category: compute
+          runbook: image-pull-backoff
+        annotations:
+          summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"
+          description: "Bad tag, missing/expired imagePullSecret, or registry outage — the rollout is blocked."
+
+      # PodsUnschedulable — labels required: namespace, pod.
+      # Scheduler can't place the pod: capacity, taints, affinity, or an
+      # unbound PVC. Requested capacity that never arrives.
+      - alert: PodsUnschedulable
+        expr: |
+          max by (namespace, pod) (kube_pod_status_unschedulable) > 0
+        for: 10m
+        labels:
+          severity: warning
+          category: compute
+          runbook: pods-unschedulable
+        annotations:
+          summary: "{{ $labels.namespace }}/{{ $labels.pod }} is unschedulable for 10m"
+          description: "No node fits — check requests vs capacity, node selectors/affinity, taints, and PVC binding."
+
   # =========================================================
   # APPLICATION — request-path SLO indicators.
   # =========================================================
@@ -338,6 +405,21 @@ groups:
           summary: "Postgres {{ $labels.instance }} p95 query time > 500ms for 10m"
           description: "Slow queries usually mean missing indexes, table bloat, or a noisy-neighbor workload."
 
+      # PostgresConnectionsNearMax — cluster-level (aggregate, no labels).
+      # In-use connections as a fraction of max_connections. New clients
+      # get refused at 100% — Mattermost writes fail. Critical at 95%.
+      - alert: PostgresConnectionsNearMax
+        expr: |
+          sum(pg_stat_activity_count) / on() pg_settings_max_connections > 0.8
+        for: 5m
+        labels:
+          severity: warning
+          category: database
+          runbook: postgres-connections-near-max
+        annotations:
+          summary: "Postgres connections > 80% of max_connections"
+          description: "Approaching the connection ceiling. Add a pooler (PgBouncer) or raise max_connections with RAM to back it."
+
   # =========================================================
   # STORAGE — volume fill, disk fill rate.
   # =========================================================
@@ -469,3 +551,115 @@ groups:
         annotations:
           summary: "Alertmanager notification failures for receiver {{ $labels.receiver }}"
           description: "Alertmanager cannot deliver. If THIS fires, the whole alerting pipeline is degraded — investigate the receiver target (Mattermost webhook, PagerDuty, etc.)."
+
+  # =========================================================
+  # SECURITY — supply-chain, auth, runtime, and policy signals.
+  # Several of these depend on tooling BEYOND the stack above and
+  # are valid-but-silent until it's present:
+  #   - PrivilegedContainerStarted  -> Kyverno / Gatekeeper policy metrics
+  #   - InteractiveShellInContainer -> Falco (falco_events)
+  #   - RBACPrivilegeEscalation     -> Falco k8s-audit plugin
+  #   - SecurityToolingDown         -> those sensors' own /metrics targets
+  # Adjust the allowlist regex and job names to YOUR environment.
+  # =========================================================
+  - name: security
+    interval: 30s
+    rules:
+      # UnexpectedContainerImage — labels required: namespace, pod.
+      # Any running container whose image is OUTSIDE the approved
+      # registry allowlist. The regex below is an example — replace it
+      # with your own registries before trusting this rule.
+      - alert: UnexpectedContainerImage
+        expr: |
+          count by (namespace, pod, container, image) (
+            kube_pod_container_info{image!~"^(mattermost/|registry\\.mattermost\\.com/|ghcr\\.io/mattermost/).*"}
+          ) > 0
+        for: 5m
+        labels:
+          severity: warning
+          category: security
+          runbook: unexpected-container-image
+        annotations:
+          summary: "{{ $labels.namespace }}/{{ $labels.pod }} runs unapproved image {{ $labels.image }}"
+          description: "Image from outside the approved registry allowlist — possible supply-chain issue or rogue workload."
+
+      # APIServerAuthFailureSpike — cluster-level (no per-pod labels).
+      # A surge of 401/403 at the API server: credential abuse, a
+      # stolen/expired token, or a broken RBAC change hammering the API.
+      - alert: APIServerAuthFailureSpike
+        expr: |
+          sum(rate(apiserver_request_total{code=~"401|403"}[5m])) > 1
+        for: 10m
+        labels:
+          severity: warning
+          category: security
+          runbook: apiserver-auth-failure-spike
+        annotations:
+          summary: "kube-apiserver auth failures (401/403) elevated for 10m"
+          description: "Sustained authn/authz denials — investigate the source identity and any recent RBAC changes."
+
+      # PrivilegedContainerStarted — requires Kyverno or Gatekeeper.
+      # Stock kube-state-metrics can't see securityContext.privileged;
+      # this keys off a policy engine's violation counter instead.
+      - alert: PrivilegedContainerStarted
+        expr: |
+          sum by (namespace, policy, rule) (
+            increase(kyverno_policy_results_total{rule=~".*privileged.*|.*run-as-non-root.*", policy_result="fail"}[10m])
+          ) > 0
+        for: 0m
+        labels:
+          severity: critical
+          category: security
+          runbook: privileged-container-started
+        annotations:
+          summary: "Privileged/root container policy violation in {{ $labels.namespace }}"
+          description: "A container started privileged / root / hostPath — container-to-node escape risk."
+
+      # InteractiveShellInContainer — requires Falco (falco_events).
+      # Falco's built-in "Terminal shell in container" rule. Label names
+      # (k8s_ns_name / k8s_pod_name) follow Falco's metric conventions.
+      - alert: InteractiveShellInContainer
+        expr: |
+          sum by (k8s_ns_name, k8s_pod_name) (
+            rate(falco_events{rule="Terminal shell in container"}[5m])
+          ) > 0
+        for: 0m
+        labels:
+          severity: warning
+          category: security
+          runbook: interactive-shell-in-container
+        annotations:
+          summary: "Shell spawned in {{ $labels.k8s_ns_name }}/{{ $labels.k8s_pod_name }}"
+          description: "Interactive shell inside a running container — hands-on-keyboard activity worth confirming."
+
+      # RBACPrivilegeEscalation — requires Kubernetes audit logs via Falco.
+      # High-privilege RBAC changes: cluster-admin bindings, wildcard
+      # roles. A classic post-compromise persistence step.
+      - alert: RBACPrivilegeEscalation
+        expr: |
+          sum by (k8s_ns_name) (
+            rate(falco_events{rule=~"Create ClusterRoleBinding.*|Attach to cluster-admin.*|K8s Role.*wildcard.*"}[5m])
+          ) > 0
+        for: 0m
+        labels:
+          severity: critical
+          category: security
+          runbook: rbac-privilege-escalation
+        annotations:
+          summary: "High-privilege RBAC change detected"
+          description: "cluster-admin binding or wildcard role created/modified — treat as a compromise indicator until cleared."
+
+      # SecurityToolingDown — the meta-alert. If Falco / Kyverno / Gatekeeper
+      # stop reporting, every tooling-dependent rule above goes silently
+      # blind. This is the rule that tells you the others can't fire.
+      - alert: SecurityToolingDown
+        expr: |
+          up{job=~"falco|kyverno.*|gatekeeper.*"} == 0
+        for: 5m
+        labels:
+          severity: critical
+          category: security
+          runbook: security-tooling-down
+        annotations:
+          summary: "Security sensor {{ $labels.job }}/{{ $labels.instance }} is down for 5m"
+          description: "A runtime/policy/audit sensor stopped reporting — tooling-dependent security alerts are now blind."

From ea2f522df4b2252ffca140e3cb25e8445c666938 Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 09:57:41 -0500
Subject: [PATCH 10/15] feat: ship sample Prometheus rules in-plugin + trim
 System Console settings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Users shouldn't have to clone the repo to get the Prometheus rules — the
plugin only owns the Alertmanager side, but their Prometheus has to emit
the `runbook:`-labelled alerts. So surface the shipped sample rules from
inside the plugin:

- render-docs now renders samples/prometheus-rules.yaml into a browsable
  help page (public/help/prometheus-rules.html) AND copies the raw file
  alongside it for download. Single source of truth: the page is built
  live from the yaml, so it can't drift from what users download.
- New `/alertmanager rules` command DMs both links (view + download) and
  explains the runbook-label join contract. Registered in help text and
  autocomplete.
- System Console header gets a "Sample Prometheus rules" link.

System Console cleanup (was a wall of text, now scannable):
- All 5 settings' help_text cut to one sentence each; the removed detail
  (k8s DNS example, openssl command) already lives in the placeholders.
- Header and footer trimmed; footer command table shortened.

Stale-count fixes (catalog is 30 runbooks / 31 rules, not 20):
- plugin.json footer, CONTRIBUTING.md, render-docs runbook index (+ the
  missing security category and new compute/database runbooks), and
  three Go comments (admin_inventory.go, cmd_query.go). CHANGELOG.md left
  as-is (historical release notes).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 CONTRIBUTING.md           |   4 +-
 build/render-docs/main.go | 113 +++++++++++++++++++++++++++++++-------
 plugin.json               |  14 ++---
 server/admin_inventory.go |   4 +-
 server/cmd_query.go       |   2 +-
 server/cmd_rules.go       |  32 +++++++++++
 server/commands.go        |   6 ++
 7 files changed, 144 insertions(+), 31 deletions(-)
 create mode 100644 server/cmd_rules.go

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 556d990..42674e3 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -16,8 +16,8 @@ The contribution surface is small on purpose.
 │   ├── reconciler.go      Cluster-leader background job
 │   └── *_test.go          Unit tests
 ├── docs/                  Markdown sources (rendered to HTML at bundle time)
-├── runbooks/              The 20 canonical SRE runbook docs
-├── samples/               Sample Prometheus rules for the 20 runbooks
+├── runbooks/              The 30 canonical SRE runbook docs
+├── samples/               Sample Prometheus rules for the 30 runbooks
 ├── public/                Static assets bundled into the plugin tarball
 └── build/                 Build tooling (manifest, render-docs)
 ```
diff --git a/build/render-docs/main.go b/build/render-docs/main.go
index c67fa2c..c348567 100644
--- a/build/render-docs/main.go
+++ b/build/render-docs/main.go
@@ -63,6 +63,7 @@ type siteSection struct {
 	SiteName    string
 	LandingBody string
 	SkipFiles   map[string]bool // filenames to skip (e.g., INDEX.md, TEMPLATE.md)
+	ExtraPages  []page          // pre-rendered pages with no .md source (e.g. the sample-rules page)
 }
 
 var sections = []siteSection{
@@ -84,11 +85,12 @@ var sections = []siteSection{
 }
 
 type page struct {
-	Slug    string // filename without extension, lowercased — drives the link
-	Title   string // taken from the first H1 of the markdown
-	HTML    template.HTML
-	Active  bool // set per-render so the nav link gets the .active class
-	Source  string // original filename (e.g., MIGRATION.md) for footer reference
+	Slug      string // filename without extension, lowercased — drives the link
+	Title     string // taken from the first H1 of the markdown
+	HTML      template.HTML
+	Active    bool   // set per-render so the nav link gets the .active class
+	Source    string // original filename (e.g., MIGRATION.md) for footer reference
+	Generated bool   // true when HTML is pre-built (no .md source to read)
 }
 
 // pageTemplate wraps each rendered doc in a shared shell. Inlining the
@@ -283,7 +285,7 @@ site, so reading either form gets you to the same place.
 // runbookIndexBody is the landing page for the runbook library.
 const runbookIndexBody = `# SRE Runbooks
 
-20 runbooks covering the most common alert categories an SRE
+30 runbooks covering the most common alert categories an SRE
 encounters. Each follows the same structure: severity → what this
 means → diagnostic steps → common causes & fixes → escalation →
 post-incident.
@@ -295,15 +297,16 @@ you land on the runbook, you work the alert.
 
 ## Categories
 
-**Compute & containers** — High CPU Usage, High Memory Usage, Pod
-CrashLoopBackOff, Pod Not Ready, Deployment Replicas Unavailable, Node
-Not Ready.
+**Compute & containers** — High CPU Usage, High Memory Usage, CPU
+Throttling High, Pod CrashLoopBackOff, Pod Not Ready, Image Pull
+BackOff, Pods Unschedulable, Deployment Replicas Unavailable, Node Not
+Ready.
 
 **Application** — High HTTP 5xx Error Rate, High API Latency, Service
 Endpoint Down, Request Rate Anomaly.
 
 **Database** — Database Connectivity Loss, Database Replication Lag,
-Database High Latency.
+Database High Latency, Postgres Connections Near Max.
 
 **Storage** — Persistent Volume Full, Disk Fill Rate High.
 
@@ -313,6 +316,10 @@ Resolution Failure.
 **Observability** — Prometheus Scrape Target Down, Alertmanager
 Notification Failure.
 
+**Security** — Unexpected Container Image, API Server Auth Failure
+Spike, Privileged Container Started, Interactive Shell in Container,
+RBAC Privilege Escalation, Security Tooling Down.
+
 Use the sidebar on the left to navigate. Use the search box up top to
 jump straight to a runbook by name or by error string.
 
@@ -325,8 +332,24 @@ commands with real namespaces, not ` + "`<NAMESPACE>`" + `
 placeholders. The on-call wants to copy-paste.
 `
 
+// rulesSrcPath / rulesOutYAML — the shipped sample rules and where the raw
+// copy lands in the help site. Relative to build/render-docs/'s working dir.
+const (
+	rulesSrcPath = "../../samples/prometheus-rules.yaml"
+	rulesOutYAML = "../../public/help/prometheus-rules.yaml"
+)
 
 func main() {
+	// Attach the generated "Sample Prometheus rules" page to the help site
+	// (sections[0]). Built live from the sample yaml so the rendered page
+	// can't drift from the file users actually download.
+	rulesPage, err := buildRulesPage()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "render-docs:", err)
+		os.Exit(1)
+	}
+	sections[0].ExtraPages = append(sections[0].ExtraPages, rulesPage)
+
 	totalPages := 0
 	for _, s := range sections {
 		n, err := renderSection(s)
@@ -336,7 +359,51 @@ func main() {
 		}
 		totalPages += n
 	}
-	fmt.Printf("rendered %d total pages across %d sites\n", totalPages, len(sections))
+
+	// Ship the raw yaml next to the rendered page so the download link (and
+	// any curl) resolves to the real file. Done after render so the help
+	// OutDir already exists.
+	if err := copyRawRules(); err != nil {
+		fmt.Fprintln(os.Stderr, "render-docs:", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("rendered %d total pages across %d sites (+ raw prometheus-rules.yaml)\n", totalPages, len(sections))
+}
+
+// buildRulesPage renders the shipped sample Prometheus rules into a help-site
+// page — the yaml verbatim (HTML-escaped) in a code block, with an intro and a
+// download link. Single source of truth: the same samples/prometheus-rules.yaml
+// users download, so there's no markdown copy to drift out of sync.
+func buildRulesPage() (page, error) {
+	raw, err := os.ReadFile(rulesSrcPath)
+	if err != nil {
+		return page{}, fmt.Errorf("read %s: %w", rulesSrcPath, err)
+	}
+	intro := `<p>Sample Prometheus alerting rules shipped with the plugin — one working rule for every runbook (31 rules across 30 runbooks). Copy them into your Prometheus, tune the thresholds, and alerts route straight to the matching runbook in chat.</p>` +
+		`<p><a href="prometheus-rules.yaml" download><strong>⬇ Download prometheus-rules.yaml</strong></a> — add it to your Prometheus <code>rule_files:</code> glob, or wrap the <code>groups:</code> block in a <code>PrometheusRule</code> CRD if you run the Prometheus Operator.</p>` +
+		`<blockquote>Every threshold and <code>for:</code> duration below is a demo-cluster placeholder — tune to your traffic and SLOs before trusting them in production. The <code>runbook:</code> label on each rule is the join key that routes it to a runbook.</blockquote>`
+	body := intro + "<pre><code>" + template.HTMLEscapeString(string(raw)) + "</code></pre>"
+	return page{
+		Slug:      "prometheus-rules",
+		Title:     "Sample Prometheus rules",
+		HTML:      template.HTML(body),
+		Source:    "samples/prometheus-rules.yaml",
+		Generated: true,
+	}, nil
+}
+
+// copyRawRules writes the sample rules verbatim into the help site so the
+// page's download link and any `curl` resolve to the actual file.
+func copyRawRules() error {
+	raw, err := os.ReadFile(rulesSrcPath)
+	if err != nil {
+		return fmt.Errorf("read %s: %w", rulesSrcPath, err)
+	}
+	if err := os.WriteFile(rulesOutYAML, raw, 0644); err != nil {
+		return fmt.Errorf("write %s: %w", rulesOutYAML, err)
+	}
+	return nil
 }
 
 // renderSection processes one site's markdown → HTML transform. Reads
@@ -382,6 +449,10 @@ func renderSection(section siteSection) (int, error) {
 		})
 	}
 
+	// Pre-rendered pages (no .md source, e.g. the sample-rules page) join
+	// the nav alongside the discovered markdown pages.
+	pages = append(pages, section.ExtraPages...)
+
 	// Stable nav order — alphabetical. Predictable across builds.
 	sort.Slice(pages, func(i, j int) bool { return pages[i].Slug < pages[j].Slug })
 
@@ -397,15 +468,19 @@ func renderSection(section siteSection) (int, error) {
 	}
 
 	for i, p := range pages {
-		body, err := os.ReadFile(filepath.Join(section.SrcDir, p.Source))
-		if err != nil {
-			return 0, fmt.Errorf("read %s: %w", p.Source, err)
-		}
-		var buf bytes.Buffer
-		if err := md.Convert(body, &buf); err != nil {
-			return 0, fmt.Errorf("render %s: %w", p.Source, err)
+		// Generated pages already carry their HTML — there's no markdown
+		// source to read. Everything else renders from its .md file.
+		if !p.Generated {
+			body, err := os.ReadFile(filepath.Join(section.SrcDir, p.Source))
+			if err != nil {
+				return 0, fmt.Errorf("read %s: %w", p.Source, err)
+			}
+			var buf bytes.Buffer
+			if err := md.Convert(body, &buf); err != nil {
+				return 0, fmt.Errorf("render %s: %w", p.Source, err)
+			}
+			p.HTML = template.HTML(rewriteMDLinks(buf.String()))
 		}
-		p.HTML = template.HTML(rewriteMDLinks(buf.String()))
 
 		if err := writePage(tmpl, section, pages, p, i, isRunbook); err != nil {
 			return 0, err
diff --git a/plugin.json b/plugin.json
index 09b8975..8a8ab8d 100644
--- a/plugin.json
+++ b/plugin.json
@@ -14,14 +14,14 @@
         }
     },
     "settings_schema": {
-        "header": "## Alertmanager Plugin\n\nBrings Prometheus Alertmanager alerts into Mattermost via native incoming webhooks.\n\n- 📚 [Plugin documentation](/plugins/com.mattermost.alertmanager/public/help/home.html)\n- 📖 [SRE Runbook library](/plugins/com.mattermost.alertmanager/public/runbooks/home.html)\n- 📊 [Org-wide inventory (sysadmin)](/plugins/com.mattermost.alertmanager/admin/inventory)\n\nConfigure the settings below if needed, then run `/alertmanager help` in any channel for the full command list.\n\n---\n*Plugin version: `__PLUGIN_VERSION__` — [release notes](__PLUGIN_RELEASE_URL__)*",
-        "footer": "### Most-used commands\n\n| Command | Use it when |\n|---|---|\n| `/alertmanager add <team> <channel> <am-url>` | First-time setup — creates the 20 canonical receivers in the named channel and DMs you the assembled `receivers:` block |\n| `/alertmanager list` | See what receivers are bound to the current channel |\n| `/alertmanager config <name>` | View one receiver's `slack_configs` YAML block + metadata |\n| `/alertmanager export` | DM yourself a fresh `receivers.yml` for the current channel (use after template changes or to back up state) |\n\nRun `/alertmanager help` in chat for the full list. Created webhooks appear at System Console → Integrations → Incoming Webhooks as `Alertmanager: <name>`.",
+        "header": "## Alertmanager Plugin\n\nPrometheus Alertmanager alerts in Mattermost, with the runbook attached.\n\n- 📚 [Documentation](/plugins/com.mattermost.alertmanager/public/help/home.html)\n- 📖 [Runbook library](/plugins/com.mattermost.alertmanager/public/runbooks/home.html)\n- 📈 [Sample Prometheus rules](/plugins/com.mattermost.alertmanager/public/help/prometheus-rules.html)\n- 📊 [Org-wide inventory (sysadmin)](/plugins/com.mattermost.alertmanager/admin/inventory)\n\nRun `/alertmanager help` in any channel for commands. Version `__PLUGIN_VERSION__` — [release notes](__PLUGIN_RELEASE_URL__).",
+        "footer": "### Most-used commands\n\n| Command | When |\n|---|---|\n| `/alertmanager add <team> <channel> <am-url>` | First-time setup — creates the 30 canonical receivers in a channel |\n| `/alertmanager rules` | Get the sample Prometheus rules (view or download) |\n| `/alertmanager list` | See receivers bound to this channel |\n\nRun `/alertmanager help` in chat for the full command list.",
         "settings": [
             {
                 "key": "AssembledYAMLTTLHours",
                 "display_name": "Auto-delete DM'd YAML files after (hours)",
                 "type": "number",
-                "help_text": "How many hours to keep the assembled receivers.yml + routes.yml files in the bot's DM with the calling sysadmin before auto-deleting them. These files contain webhook URLs (channel-bound bearer tokens), so a finite retention period limits long-term exposure if a user account is later compromised.\n\nDefaults to 72 hours. Set to 0 to disable auto-delete (files persist forever — not recommended).",
+                "help_text": "Hours to keep the DM'd receivers.yml/routes.yml (they contain webhook tokens) before auto-deleting; 0 disables auto-delete.",
                 "placeholder": "72",
                 "default": 72
             },
@@ -29,7 +29,7 @@
                 "key": "AlertManagerCABundle",
                 "display_name": "Alertmanager CA bundle (PEM)",
                 "type": "longtext",
-                "help_text": "Optional. Paste PEM-encoded CA certificates here to trust self-signed or internal CAs when the plugin queries Alertmanager (used by `/alertmanager status`, `alerts`, `silences`, `expire_silence`). Multiple certs OK — concatenate them. Leave empty to trust only the system CAs.\n\nThis only affects the plugin's outbound calls TO Alertmanager. The webhook URLs Alertmanager POSTs to are Mattermost-side and use whatever TLS Mattermost is configured with.",
+                "help_text": "Optional PEM CA bundle so the plugin trusts self-signed or internal Alertmanager certs on its outbound calls; empty uses the system CAs.",
                 "placeholder": "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----",
                 "default": ""
             },
@@ -37,7 +37,7 @@
                 "key": "MetricsToken",
                 "display_name": "Metrics endpoint bearer token",
                 "type": "text",
-                "help_text": "Optional. When set, the plugin exposes Prometheus-format metrics at `/plugins/com.mattermost.alertmanager/metrics`. Prometheus scrapes the endpoint using this token in the `Authorization: Bearer <token>` header. Leave empty to disable the metrics endpoint entirely (returns 404).\n\nGenerate a random token (e.g., `openssl rand -hex 32`) and configure Prometheus to use it in its scrape_configs.",
+                "help_text": "Optional bearer token that turns on the Prometheus metrics endpoint at `/plugins/com.mattermost.alertmanager/metrics`; empty disables it (404).",
                 "placeholder": "(random 64-character hex string)",
                 "default": ""
             },
@@ -45,7 +45,7 @@
                 "key": "WebhookHost",
                 "display_name": "Webhook host override (api_url base)",
                 "type": "text",
-                "help_text": "Optional. Overrides the host:port portion of the Mattermost webhook URL that gets rendered into alertmanager.yml's api_url field. Leave empty to use the Mattermost SiteURL.\n\nSet this when Alertmanager reaches Mattermost over a different network path than end users:\n- **Docker compose**: http://host.docker.internal:8065 (AM container can't reach localhost)\n- **Kubernetes**: http://mattermost.<namespace>.svc.cluster.local:8065 (cluster-internal service DNS; avoids Ingress round-trip)\n- **Bare metal / shared host**: leave empty\n\nThe runbook URLs in the slack_configs text template continue using SiteURL because those are clicked by users in browsers.",
+                "help_text": "Optional host:port for the `api_url` written into alertmanager.yml, for when Alertmanager reaches Mattermost on a different network path than users (see placeholder); empty uses the SiteURL.",
                 "placeholder": "http://mattermost.mattermost.svc.cluster.local:8065",
                 "default": ""
             },
@@ -53,7 +53,7 @@
                 "key": "WebhookRotationDays",
                 "display_name": "Webhook rotation reminder period (days)",
                 "type": "number",
-                "help_text": "Optional. When set to a value > 0, the plugin DMs sysadmins when a receiver's webhook hasn't been rotated for this many days. The DM includes the exact `/alertmanager rotate all --overdue` command to run in the affected channel.\n\nIMPORTANT: this is a REMINDER only — the plugin does NOT automatically rotate webhooks. Webhook delivery continues to work until an admin runs the rotation command. After rotation, the admin must paste the new YAML into alertmanager.yml (Alertmanager has no write API; the plugin can't push the new URL to AM directly).\n\nCommon values:\n- 90 (quarterly) — recommended baseline for security hygiene\n- 180 (semi-annual)\n- 365 (annual)\n\nLower than 30 generates noise unless you have a specific secret-rotation policy. Default 0 = feature disabled.\n\nReminders repeat at most once every 7 days per receiver until the rotation happens.",
+                "help_text": "Days before the plugin DMs sysadmins to rotate a receiver's webhook (reminder only — no auto-rotation); 0 disables, 90 is a good baseline.",
                 "placeholder": "0",
                 "default": 0
             }
diff --git a/server/admin_inventory.go b/server/admin_inventory.go
index 973015d..5243724 100644
--- a/server/admin_inventory.go
+++ b/server/admin_inventory.go
@@ -584,7 +584,7 @@ func sortedKeys(m map[string]bool) []string {
 // decodeTargetSelection turns the (Type, Value) dropdown pair into
 // a flat list of runbook slugs:
 //
-//	type=all                → all 20 slugs (value ignored)
+//	type=all                → all 30 slugs (value ignored)
 //	type=group, value=X     → scaffoldSets[X]
 //	type=individual, value=X → [X]
 //	anything else           → nil (caller falls back to label-only sim)
@@ -630,7 +630,7 @@ func filterConfigsByChannel(configs []alertConfig, channel string) []alertConfig
 }
 
 // runInventorySimulationMatrixForSlugs runs a simulation per slug
-// from the supplied list (instead of all 20). Used when the admin
+// from the supplied list (instead of all 30). Used when the admin
 // picks a Group target — we get a coverage table scoped to just
 // that group.
 func (p *Plugin) runInventorySimulationMatrixForSlugs(slugs []string, severity, extra string, configs []alertConfig, amStatus map[string]amReachabilityEntry) []inventorySimResult {
diff --git a/server/cmd_query.go b/server/cmd_query.go
index 76388d2..e8a86f0 100644
--- a/server/cmd_query.go
+++ b/server/cmd_query.go
@@ -80,7 +80,7 @@ func emptyScopeMessage(verb string) string {
 // each URL in the input so output is deterministic across calls.
 //
 // Why this matters: in the common case where one channel hosts N
-// receivers all pointing at the same Alertmanager (e.g. all 20 canonical
+// receivers all pointing at the same Alertmanager (e.g. all 30 canonical
 // runbook receivers in #alerts), the older per-receiver loop hit AM N
 // times and printed N copies of the same "firing alerts" or "loaded
 // silences" block. Querying once per distinct AM URL collapses that to
diff --git a/server/cmd_rules.go b/server/cmd_rules.go
new file mode 100644
index 0000000..9e16395
--- /dev/null
+++ b/server/cmd_rules.go
@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+// handleRules points users at the shipped sample Prometheus alerting rules —
+// a browsable HTML view and the raw file to download. This is the upstream
+// half of the pipeline the plugin does NOT own: the plugin creates the
+// Alertmanager receivers and routes, but SOMETHING in the user's Prometheus
+// has to emit alerts carrying the `runbook:` label. This command hands them a
+// ready starting set so they don't have to go dig it out of the repo.
+//
+// The files are served from the plugin's public/ assets (generated by
+// build/render-docs from samples/prometheus-rules.yaml), so there's no server
+// state to read — just the two URLs derived from SiteURL.
+func (p *Plugin) handleRules(args *model.CommandArgs) (string, error) {
+	base := fmt.Sprintf("%s/plugins/%s/public/help", p.siteURL(), Manifest.Id)
+	viewURL := base + "/prometheus-rules.html"
+	rawURL := base + "/prometheus-rules.yaml"
+
+	return fmt.Sprintf(
+		"**Sample Prometheus alerting rules** — the upstream half of the pipeline.\n\n"+
+			"The plugin creates the Alertmanager receivers and routes, but your **Prometheus** has to emit the alerts that carry the `runbook:` label. These samples give you one working rule for every runbook (31 rules across 30 runbooks) so you don't start from a blank page.\n\n"+
+			"- 📖 [View the rules in your browser](%s)\n"+
+			"- ⬇ [Download `prometheus-rules.yaml`](%s) — add it to your Prometheus `rule_files:` glob, or wrap the `groups:` block in a `PrometheusRule` CRD if you run the Prometheus Operator\n\n"+
+			"⚠️ Every threshold is a demo-cluster placeholder — tune to your traffic and SLOs before production. The `runbook:` label on each rule is the join key that routes it to the matching runbook in chat.",
+		viewURL, rawURL,
+	), nil
+}
diff --git a/server/commands.go b/server/commands.go
index c23a639..b56ac9c 100644
--- a/server/commands.go
+++ b/server/commands.go
@@ -26,6 +26,7 @@ const (
 		"- `/alertmanager reconcile` — prune receivers whose Mattermost webhook has been deleted out-of-band (sysadmin; runs automatically every 5 min)\n" +
 		"- `/alertmanager remove <name|set|all> [--force]` — delete a receiver, a runbook set, or everything in this channel\n" +
 		"- `/alertmanager rotate <name>` — delete + recreate the webhook (new hook-id, new URL)\n" +
+		"- `/alertmanager rules` — links to the shipped sample Prometheus alerting rules (browsable HTML + downloadable YAML) so you can wire up the Prometheus side without cloning the repo\n" +
 		"- `/alertmanager silences` — list active silences (grouped by Alertmanager URL)\n" +
 		"- `/alertmanager status` — Alertmanager version + uptime per backend (grouped by Alertmanager URL)\n" +
 		"- `/alertmanager validate [name|set] [--webhook-test|--end-to-end|--simulate <labels>] [--severity=<value>]` — validate pipeline configuration. Without flags: AM reach + receiver-loaded-in-AM. `--webhook-test` / `--end-to-end` fire side-effect tests. `--simulate runbook=<slug> severity=<level>` walks AM's loaded route tree against the labels and reports which receiver(s) an alert with those labels would dispatch to (read-only, no synthetic alert). `--severity=<warning|critical|info|all>` controls which severities `--end-to-end` fires; `all` fires four alerts per receiver (warning + critical + info + resolved) so you can visually verify every render path in one shot."
@@ -150,6 +151,8 @@ func getAutocompleteData() *model.AutocompleteData {
 	})
 	root.AddCommand(rotate)
 
+	root.AddCommand(model.NewAutocompleteData("rules", "", "Links to the shipped sample Prometheus alerting rules (view or download)"))
+
 	root.AddCommand(model.NewAutocompleteData("silences", "", "List active Alertmanager silences (grouped by Alertmanager URL)"))
 
 	root.AddCommand(model.NewAutocompleteData("status", "", "Alertmanager version + uptime per backend (grouped by Alertmanager URL)"))
@@ -244,6 +247,9 @@ func (p *Plugin) executeCommand(args *model.CommandArgs) string {
 	case "rotate":
 		msg, err := p.handleRotate(args)
 		return joinErr(msg, err)
+	case "rules":
+		msg, err := p.handleRules(args)
+		return joinErr(msg, err)
 	case "silences":
 		msg, err := p.handleListSilences(args)
 		return joinErr(msg, err)

From 72f7131557ed3b32ad91da0f55979e28e8ac838e Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 10:35:56 -0500
Subject: [PATCH 11/15] feat: WebhookHost preset dropdown + richer am-url
 autocomplete hint
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two discoverability fixes for the URLs users have to get right:

am-url autocomplete hint — the `/alertmanager add` URL argument now spells
out the container-networking gotcha at type time: a containerized
Mattermost reaches Alertmanager via http://host.docker.internal:9093 (not
localhost, which is the MM container itself), or a service name on a shared
Docker network. Previously just "API base URL (no trailing slash)", which
let people enter localhost and hit a confusing connection error later.

WebhookHost preset dropdown — new System Console `WebhookHostPreset`
dropdown offers known-good hosts for common setups (Docker Desktop,
default-namespace K8s) so admins don't have to know the magic
host.docker.internal string. Kept the free-text WebhookHost field for
custom URLs (non-default namespace, bare-metal FQDN). resolveWebhookHost
collapses the two into one effective value with custom-wins precedence, so
existing installs (text field only) behave identically — the dropdown
defaults to None. Resolution happens at config-load time into the existing
configuration.WebhookHost, so no consumer changed. Validation now runs on
the resolved value. Added TestResolveWebhookHost pinning the precedence.

Mattermost plugin settings can't do a native select-or-type combo and this
plugin is server-only (no webapp), so a two-field dropdown+text is the
achievable shape.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 plugin.json                  | 27 ++++++++++++++++++++++++---
 server/commands.go           |  2 +-
 server/configuration.go      | 32 ++++++++++++++++++++++++++++++--
 server/configuration_test.go | 28 ++++++++++++++++++++++++++++
 4 files changed, 83 insertions(+), 6 deletions(-)

diff --git a/plugin.json b/plugin.json
index 8a8ab8d..dddb8ab 100644
--- a/plugin.json
+++ b/plugin.json
@@ -41,12 +41,33 @@
                 "placeholder": "(random 64-character hex string)",
                 "default": ""
             },
+            {
+                "key": "WebhookHostPreset",
+                "display_name": "Webhook host — preset",
+                "type": "dropdown",
+                "help_text": "Quick-fill for how Alertmanager reaches Mattermost to POST alerts. Pick a common setup, or leave 'None' and type a custom value below (a custom value overrides this preset).",
+                "default": "",
+                "options": [
+                    {
+                        "display_name": "None — use the custom field below (blank there = Mattermost SiteURL)",
+                        "value": ""
+                    },
+                    {
+                        "display_name": "Docker Desktop / Compose — host.docker.internal:8065",
+                        "value": "http://host.docker.internal:8065"
+                    },
+                    {
+                        "display_name": "Kubernetes service DNS — namespace 'mattermost' (edit custom field if yours differs)",
+                        "value": "http://mattermost.mattermost.svc.cluster.local:8065"
+                    }
+                ]
+            },
             {
                 "key": "WebhookHost",
-                "display_name": "Webhook host override (api_url base)",
+                "display_name": "Webhook host — custom (overrides preset)",
                 "type": "text",
-                "help_text": "Optional host:port for the `api_url` written into alertmanager.yml, for when Alertmanager reaches Mattermost on a different network path than users (see placeholder); empty uses the SiteURL.",
-                "placeholder": "http://mattermost.mattermost.svc.cluster.local:8065",
+                "help_text": "Optional custom host:port for the `api_url` written into alertmanager.yml; overrides the preset above. Use for a non-default K8s namespace or a bare-metal FQDN. Blank = use the preset (or SiteURL if that's None too).",
+                "placeholder": "http://mattermost.<your-namespace>.svc.cluster.local:8065",
                 "default": ""
             },
             {
diff --git a/server/commands.go b/server/commands.go
index b56ac9c..fa8cfff 100644
--- a/server/commands.go
+++ b/server/commands.go
@@ -75,7 +75,7 @@ func getAutocompleteData() *model.AutocompleteData {
 	add := model.NewAutocompleteData("add", "[team] [channel] [am-url] [target] [on]", "Create receivers for a group set OR individual runbook slug. One shared webhook per group; individual slugs get their own. Trailing `on` opts in to rotation reminders. (sysadmin/team_admin)")
 	add.AddDynamicListArgument("Mattermost team URL slug — tab through your teams", teamFetchURL, true)
 	add.AddDynamicListArgument("Mattermost channel URL slug — public channels in the chosen team (or type a new name to auto-create)", channelFetchURL, true)
-	add.AddTextArgument("Alertmanager API base URL (no trailing slash)", "[am-url]", "")
+	add.AddTextArgument("Alertmanager base URL reachable FROM the Mattermost server, no trailing slash. Containerized MM (Docker Desktop): http://host.docker.internal:9093 — NOT localhost, that's the MM container itself. Same Docker network: http://<am-service>:9093.", "[am-url]", "")
 	add.AddStaticListArgument("Group set OR individual runbook slug (defaults to `all`). Type a slug freely; static list shows group sets only.", false, []model.AutocompleteListItem{
 		{Item: "all", HelpText: "Every embedded runbook (default — 30 receivers in one shared webhook)"},
 		{Item: "application", HelpText: "HTTP error, latency, endpoint, request-rate alerts (4) — one shared webhook"},
diff --git a/server/configuration.go b/server/configuration.go
index 45ff1f7..6a0067d 100644
--- a/server/configuration.go
+++ b/server/configuration.go
@@ -19,9 +19,16 @@ import (
 // Mattermost webhook URL when rendered into alertmanager.yml. See
 // plugin.json settings_schema for the full rationale. Empty = fall back
 // to SiteURL.
+//
+// WebhookHostPreset is the System Console dropdown of known-good hosts for
+// common setups (Docker Desktop, default-namespace K8s). It and WebhookHost
+// are collapsed into a single effective value by resolveWebhookHost — a
+// free-text WebhookHost always wins, so existing installs and custom URLs
+// are unaffected.
 type rawConfiguration struct {
 	AlertConfigsJSON      string
 	WebhookHost           string
+	WebhookHostPreset     string
 	AssembledYAMLTTLHours int
 	AlertManagerCABundle  string
 	MetricsToken          string
@@ -213,6 +220,23 @@ func validateWebhookHost(raw string) error {
 	return nil
 }
 
+// resolveWebhookHost collapses the two System Console fields that can set the
+// global webhook host into the single effective value the rest of the plugin
+// uses (stored in configuration.WebhookHost, read by renderReceiverAPIURL).
+// Precedence, most specific first:
+//  1. WebhookHost (free text) — a value here always wins, so an existing
+//     install's setting keeps working and a custom URL (e.g. a K8s service
+//     DNS with a non-default namespace) overrides any preset.
+//  2. WebhookHostPreset (dropdown) — a known-good constant for a common
+//     setup (Docker Desktop, default-namespace K8s).
+//  3. "" — neither set; callers fall back to SiteURL.
+func resolveWebhookHost(custom, preset string) string {
+	if c := strings.TrimSpace(custom); c != "" {
+		return c
+	}
+	return strings.TrimSpace(preset)
+}
+
 // parseAlertConfigs decodes and validates the JSON blob. Surfaces byte
 // offsets on syntax errors and entry indices on validation errors so an
 // admin staring at a multi-screen JSON blob can find the typo.
@@ -296,7 +320,11 @@ func (p *Plugin) OnConfigurationChange() error {
 		return fmt.Errorf("load plugin configuration: %w", err)
 	}
 
-	if err := validateWebhookHost(raw.WebhookHost); err != nil {
+	// Collapse the preset dropdown + free-text field into one effective host,
+	// then validate that — a preset is known-good, but a free-text override
+	// still needs the sanity check.
+	effectiveHost := resolveWebhookHost(raw.WebhookHost, raw.WebhookHostPreset)
+	if err := validateWebhookHost(effectiveHost); err != nil {
 		return err
 	}
 
@@ -320,7 +348,7 @@ func (p *Plugin) OnConfigurationChange() error {
 		}
 	}
 
-	p.setConfiguration(newConfiguration(entries, raw.WebhookHost, raw.AssembledYAMLTTLHours, raw.AlertManagerCABundle, raw.MetricsToken, raw.WebhookRotationDays))
+	p.setConfiguration(newConfiguration(entries, effectiveHost, raw.AssembledYAMLTTLHours, raw.AlertManagerCABundle, raw.MetricsToken, raw.WebhookRotationDays))
 	// Refresh the alertmanager package's HTTP client to use the new
 	// CA bundle (if set). Applied on every config change so admins
 	// can rotate certificates without a plugin restart.
diff --git a/server/configuration_test.go b/server/configuration_test.go
index 6087ba7..1d727fc 100644
--- a/server/configuration_test.go
+++ b/server/configuration_test.go
@@ -47,6 +47,34 @@ func TestValidateWebhookHost(t *testing.T) {
 	}
 }
 
+// TestResolveWebhookHost pins the precedence between the WebhookHost free-text
+// field and the WebhookHostPreset dropdown. The custom-wins rule is what keeps
+// existing installs (which only ever had the text field) behaving identically
+// after the dropdown was added — a regression here would silently repoint every
+// webhook URL.
+func TestResolveWebhookHost(t *testing.T) {
+	cases := []struct {
+		name   string
+		custom string
+		preset string
+		want   string
+	}{
+		{"both empty falls through to SiteURL", "", "", ""},
+		{"preset only", "", "http://host.docker.internal:8065", "http://host.docker.internal:8065"},
+		{"custom only", "https://mm.example.com", "", "https://mm.example.com"},
+		{"custom wins over preset", "https://mm.example.com", "http://host.docker.internal:8065", "https://mm.example.com"},
+		{"whitespace custom is ignored, preset used", "   ", "http://host.docker.internal:8065", "http://host.docker.internal:8065"},
+		{"custom is trimmed", "  https://mm.example.com  ", "", "https://mm.example.com"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := resolveWebhookHost(tc.custom, tc.preset); got != tc.want {
+				t.Fatalf("resolveWebhookHost(%q, %q) = %q, want %q", tc.custom, tc.preset, got, tc.want)
+			}
+		})
+	}
+}
+
 // TestAlertConfigIsValid covers the per-entry invariant rules. Each
 // branch in IsValid corresponds to a misconfiguration mode an admin
 // could ship by hand-editing the JSON blob in System Console — these

From 2c6edc0dcdb4de8adae4e6b9fd06ce32d2bcb84b Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 10:41:13 -0500
Subject: [PATCH 12/15] feat: split am-url hint into three hover-able
 suggestions

The single dense hint line was hard to read in the autocomplete dropdown.
Replace it with a static list of the three real patterns, each with its
own per-item hover: Docker Desktop (host.docker.internal), Kubernetes
(service DNS), and a custom URL template. Non-enforcing, so users can
still type any URL - matches the existing target/remove arg pattern.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 server/commands.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/server/commands.go b/server/commands.go
index fa8cfff..6e0cb7b 100644
--- a/server/commands.go
+++ b/server/commands.go
@@ -75,7 +75,16 @@ func getAutocompleteData() *model.AutocompleteData {
 	add := model.NewAutocompleteData("add", "[team] [channel] [am-url] [target] [on]", "Create receivers for a group set OR individual runbook slug. One shared webhook per group; individual slugs get their own. Trailing `on` opts in to rotation reminders. (sysadmin/team_admin)")
 	add.AddDynamicListArgument("Mattermost team URL slug — tab through your teams", teamFetchURL, true)
 	add.AddDynamicListArgument("Mattermost channel URL slug — public channels in the chosen team (or type a new name to auto-create)", channelFetchURL, true)
-	add.AddTextArgument("Alertmanager base URL reachable FROM the Mattermost server, no trailing slash. Containerized MM (Docker Desktop): http://host.docker.internal:9093 — NOT localhost, that's the MM container itself. Same Docker network: http://<am-service>:9093.", "[am-url]", "")
+	// am-url as a static list of the three real patterns, each with its own
+	// hover, instead of one cramped hint line. Non-enforcing — users can
+	// still type any URL; these are labelled suggestions that surface the
+	// container-networking gotcha (localhost inside the MM container is the
+	// container, not the host) at the moment of typing.
+	add.AddStaticListArgument("Alertmanager base URL, reachable FROM the Mattermost server (no trailing slash). Pick a pattern or type your own.", false, []model.AutocompleteListItem{
+		{Item: "http://host.docker.internal:9093", HelpText: "Docker Desktop / Compose — MM runs in a container and reaches Alertmanager via the host gateway. Use this, NOT localhost (localhost is the MM container itself)."},
+		{Item: "http://alertmanager.monitoring.svc.cluster.local:9093", HelpText: "Kubernetes — cluster-internal service DNS. Change 'monitoring' to the namespace your Alertmanager actually runs in."},
+		{Item: "https://alertmanager.example.com", HelpText: "Custom / anything else — your Alertmanager base URL as reachable from the MM server. Replace the host; keep it scheme://host[:port], no trailing slash."},
+	})
 	add.AddStaticListArgument("Group set OR individual runbook slug (defaults to `all`). Type a slug freely; static list shows group sets only.", false, []model.AutocompleteListItem{
 		{Item: "all", HelpText: "Every embedded runbook (default — 30 receivers in one shared webhook)"},
 		{Item: "application", HelpText: "HTTP error, latency, endpoint, request-rate alerts (4) — one shared webhook"},

From ccacf303246443ef7aa923dd48702e2516a21475 Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 11:13:12 -0500
Subject: [PATCH 13/15] fix: team-qualify receiver names to prevent cross-team
 channel collisions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Receiver names were <slug>--<channel>, but channel names are only unique
per team (town-square exists in every team). Two same-named channels in
different teams produced identical receiver names, so:
  - `/alertmanager add` in the second team silently skipped every receiver
    as "already exists" (dedup matched on channel name alone), creating
    nothing and misrouting that team's alerts to the first team's webhook;
  - list / remove / export / validate (all via configsForCurrentChannel)
    matched on channel name too, so they'd show — and `remove all` could
    delete — another team's receivers.

Fix (forward-only; no migration, nobody has data yet):
  - receiverNameForChannel now takes team: <slug>--<team>-<channel>.
    Alertmanager requires globally-unique receiver names, so team must be
    in the name, not just the stored fields.
  - add-dedup, configsForCurrentChannel, and resolveReceiverName all
    team-scope now (configsForCurrentChannel resolves team via
    channel.TeamId -> GetTeam). That one function backs 13 command sites.
  - name length cap raised 64 -> 190 to fit <slug>--<team>-<channel>
    (~164 worst case with 64-char team/channel slugs); AM/YAML don't care.
  - the runbook-slug boundary (first `--`) is still unambiguous;
    uniqueness of the team-channel tail is enforced by the existing
    duplicate-name rejection, not the separator.

Updated CONFIGURATION.md / SLASH_COMMANDS.md and the name tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 docs/CONFIGURATION.md        | 27 ++++++++++++---------
 docs/SLASH_COMMANDS.md       | 10 +++++---
 server/cmd_lifecycle.go      | 13 ++++++----
 server/cmd_query.go          | 13 +++++++++-
 server/cmd_scaffold.go       | 47 ++++++++++++++++++++++--------------
 server/cmd_scaffold_test.go  | 28 +++++++++++++--------
 server/configuration.go      | 10 +++++---
 server/configuration_test.go |  2 +-
 8 files changed, 97 insertions(+), 53 deletions(-)

diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md
index 161896e..651e04a 100644
--- a/docs/CONFIGURATION.md
+++ b/docs/CONFIGURATION.md
@@ -137,7 +137,7 @@ When you run `/alertmanager add <team> <channel> <am-url> [set] [on]`:
    create one Mattermost incoming webhook per receiver in the chosen
    set, revokes the PAT.
 3. Stores each receiver in plugin config, named
-   `<runbook-slug>--<channel-slug>`. Stamps `LastRotatedAt = now`.
+   `<runbook-slug>--<team-slug>-<channel-slug>`. Stamps `LastRotatedAt = now`.
    If `on` was passed, sets `RotationRemindersEnabled = true`.
 4. Renders two YAML fragments and DMs them to the calling user:
    - `alertmanager-receivers.yml` — paste under `receivers:`
@@ -261,20 +261,25 @@ starting point.
 
 ## Receiver naming convention
 
-The plugin names every receiver `<runbook-slug>--<channel-slug>`:
+The plugin names every receiver `<runbook-slug>--<team-slug>-<channel-slug>`:
 
 ```
-high-cpu-usage--alert-slo-channel
-high-cpu-usage--oncall-critical
-database-connectivity-loss--dba-team-channel
+high-cpu-usage--sre-alert-slo-channel
+high-cpu-usage--sre-oncall-critical
+database-connectivity-loss--platform-dba-team-channel
 ```
 
-The `--` separator is unambiguous — Mattermost channel slugs and
-runbook filenames use single hyphens, never doubles.
-
-The suffix guarantees uniqueness when the same runbook is delivered
-to multiple channels. Routes are auto-generated to fan out alerts
-to all receivers that share a base slug — see fan-out below.
+The team slug is part of the name because channel names are unique
+only *per team* — `town-square` exists in every team. Without it,
+the same runbook delivered to same-named channels in different teams
+would collide (Alertmanager requires globally-unique receiver names).
+
+The `--` after the runbook slug is the parse boundary — runbook
+filenames never contain doubles, so the runbook slug is always
+recoverable. The `<team>-<channel>` tail uses a single hyphen and is
+identity/display only (team and channel are also stored as separate
+fields); global uniqueness is enforced by the plugin rejecting
+duplicate names on save, not by the separator.
 
 ## Fan-out: one alert, multiple channels
 
diff --git a/docs/SLASH_COMMANDS.md b/docs/SLASH_COMMANDS.md
index c9f59d0..837e506 100644
--- a/docs/SLASH_COMMANDS.md
+++ b/docs/SLASH_COMMANDS.md
@@ -31,10 +31,12 @@ in the channel.
 
 ## How receiver names work
 
-The plugin names receivers `<runbook-slug>--<channel-slug>`, e.g.
-`high-cpu-usage--alert-slo-channel`. The `--` separator is the
-boundary between the runbook (what the alert is about) and the
-channel (where it's delivered).
+The plugin names receivers `<runbook-slug>--<team-slug>-<channel-slug>`,
+e.g. `high-cpu-usage--sre-alert-slo-channel`. The `--` after the
+runbook slug is the boundary between the runbook (what the alert is
+about) and where it's delivered (team + channel). Team is included
+because channel names repeat across teams (`town-square` is in every
+team), so it's needed to keep receiver names globally unique.
 
 When invoking commands from chat, you can use either form:
 
diff --git a/server/cmd_lifecycle.go b/server/cmd_lifecycle.go
index 450e046..c76d509 100644
--- a/server/cmd_lifecycle.go
+++ b/server/cmd_lifecycle.go
@@ -601,12 +601,15 @@ func resolveReceiverName(all []alertConfig, supplied, channelID string, p *Plugi
 			return c.Name
 		}
 	}
-	// 2. Short-name match scoped to current channel
+	// 2. Short-name match scoped to current team + channel. Team is part of
+	// the receiver name now, so the candidate needs the team slug too.
 	if ch, appErr := p.API.GetChannel(channelID); appErr == nil {
-		candidate := receiverNameForChannel(supplied, ch.Name)
-		for _, c := range all {
-			if c.Name == candidate {
-				return c.Name
+		if team, teamErr := p.API.GetTeam(ch.TeamId); teamErr == nil {
+			candidate := receiverNameForChannel(supplied, team.Name, ch.Name)
+			for _, c := range all {
+				if c.Name == candidate {
+					return c.Name
+				}
 			}
 		}
 	}
diff --git a/server/cmd_query.go b/server/cmd_query.go
index e8a86f0..fc0da8d 100644
--- a/server/cmd_query.go
+++ b/server/cmd_query.go
@@ -52,10 +52,21 @@ func (p *Plugin) configsForCurrentChannel(args *model.CommandArgs) []alertConfig
 		return nil
 	}
 
+	// Team-scope as well as channel-scope: channel names are unique only per
+	// team (`town-square` exists in every team), so matching on channel name
+	// alone would leak — and let commands like `remove all` act on — another
+	// team's receivers that happen to share this channel's name.
+	team, appErr := p.API.GetTeam(channel.TeamId)
+	if appErr != nil {
+		p.API.LogWarn("could not resolve current team for scoping",
+			"teamID", channel.TeamId, "err", appErr.Error())
+		return nil
+	}
+
 	all := p.getConfiguration().AlertConfigs
 	matched := make([]alertConfig, 0, len(all))
 	for _, c := range all {
-		if c.Channel == channel.Name {
+		if c.Team == team.Name && c.Channel == channel.Name {
 			matched = append(matched, c)
 		}
 	}
diff --git a/server/cmd_scaffold.go b/server/cmd_scaffold.go
index cc017dd..1b8b44c 100644
--- a/server/cmd_scaffold.go
+++ b/server/cmd_scaffold.go
@@ -160,26 +160,27 @@ func (p *Plugin) handleAdd(args *model.CommandArgs) (string, error) {
 	p.configWriteMu.Lock()
 	defer p.configWriteMu.Unlock()
 
-	// Skip-check is scoped to the destination channel only. A receiver
-	// named `high-cpu-usage--alert-slo-channel` MUST block creating it
-	// again, but `high-cpu-usage--alert-sre-channel` in another channel
-	// is independent (fan-out pattern). Walk current config once.
+	// Skip-check is scoped to the destination TEAM + channel. Channel names
+	// are unique only per team (`town-square` exists in every team), so a
+	// receiver in team-a's `town-square` must NOT block creating one in
+	// team-b's `town-square` — matching on channel name alone did exactly
+	// that, silently skipping the second team's add. Walk current config once.
 	current := p.getConfiguration().AlertConfigs
 	existingInThisChannel := make(map[string]bool)
 	for _, c := range current {
-		if c.Channel == channel {
+		if c.Team == team && c.Channel == channel {
 			existingInThisChannel[c.Name] = true
 		}
 	}
 
 	// Two-pass: identify slugs that need creation, then create one shared
-	// webhook for the whole batch. Channel-suffix every receiver name
-	// (pattern <slug>--<channel>); the shared webhook itself is named
+	// webhook for the whole batch. Team+channel-qualify every receiver name
+	// (pattern <slug>--<team>-<channel>); the shared webhook itself is named
 	// <group-or-slug>--<channel> in Mattermost.
 	results := make([]scaffoldResult, 0, len(slugs))
 	newSlugs := make([]string, 0, len(slugs))
 	for _, slug := range slugs {
-		receiverName := receiverNameForChannel(slug, channel)
+		receiverName := receiverNameForChannel(slug, team, channel)
 		if existingInThisChannel[receiverName] || existingInThisChannel[slug] {
 			results = append(results, scaffoldResult{receiverName, "skipped", "already exists"})
 			continue
@@ -202,13 +203,13 @@ func (p *Plugin) handleAdd(args *model.CommandArgs) (string, error) {
 			// Existing skipped slugs remain in the results; rendering
 			// below shows the full picture.
 			for _, slug := range newSlugs {
-				results = append(results, scaffoldResult{receiverNameForChannel(slug, channel), "failed", hookErr.Error()})
+				results = append(results, scaffoldResult{receiverNameForChannel(slug, team, channel), "failed", hookErr.Error()})
 			}
 		} else {
 			sharedHookID = hookID
 			now := time.Now().UTC()
 			for _, slug := range newSlugs {
-				receiverName := receiverNameForChannel(slug, channel)
+				receiverName := receiverNameForChannel(slug, team, channel)
 				newEntries = append(newEntries, alertConfig{
 					Name:                     receiverName,
 					Team:                     team,
@@ -486,15 +487,25 @@ func (p *Plugin) assembleReceiversYAML(newEntries []alertConfig, results []scaff
 	return y.String()
 }
 
-// receiverNameForChannel constructs the channel-scoped receiver name
-// from a runbook slug + channel slug. Pattern: <slug>--<channel>.
+// receiverNameForChannel constructs the receiver name from a runbook slug,
+// team slug, and channel slug. Pattern: <slug>--<team>-<channel>.
 //
-// The double-hyphen separator is deliberate: it's a visually obvious
-// boundary that can't be confused with hyphens inside either component.
-// `high-cpu-usage--alert-slo-channel` parses unambiguously as
-// `(high-cpu-usage)--(alert-slo-channel)`.
-func receiverNameForChannel(slug, channelSlug string) string {
-	return slug + "--" + channelSlug
+// Team is part of the name because channel names are only unique PER TEAM —
+// `town-square` exists in every team by default. Without the team segment,
+// receivers for the same runbook in same-named channels across teams would
+// collide (Alertmanager requires globally-unique receiver names), silently
+// misrouting one team's alerts to another's channel.
+//
+// The double-hyphen after the slug is the load-bearing separator:
+// receiverBaseSlug splits on the FIRST `--` to recover the runbook slug, so
+// the slug boundary stays unambiguous. The team-channel tail uses a single
+// `-` and is NOT parsed back out — team and channel live in their own
+// alertConfig fields; the tail is display/uniqueness only. Global uniqueness
+// is enforced by the duplicate-name rejection in parseAlertConfigs, not by
+// the separator, so a (rare) team/channel hyphen ambiguity fails loud at
+// config-save rather than misrouting.
+func receiverNameForChannel(slug, teamSlug, channelSlug string) string {
+	return slug + "--" + teamSlug + "-" + channelSlug
 }
 
 // receiverBaseSlug returns the runbook slug portion of a receiver name.
diff --git a/server/cmd_scaffold_test.go b/server/cmd_scaffold_test.go
index 4fb0ebc..54c30d5 100644
--- a/server/cmd_scaffold_test.go
+++ b/server/cmd_scaffold_test.go
@@ -6,22 +6,27 @@ import (
 	"testing"
 )
 
-// TestReceiverNameForChannel pins the suffixing pattern used to keep
-// receiver names unique across channels. The double-hyphen separator
-// is load-bearing: it has to be unambiguous when parsed back out by
-// receiverBaseSlug.
+// TestReceiverNameForChannel pins the team-qualified suffixing pattern used to
+// keep receiver names globally unique. Team is in the name because channel
+// names are unique only per team — without it, same-named channels in
+// different teams would collide. The double-hyphen after the slug is
+// load-bearing: it stays unambiguous when parsed back out by receiverBaseSlug.
 func TestReceiverNameForChannel(t *testing.T) {
 	cases := []struct {
-		slug, channel, want string
+		slug, team, channel, want string
 	}{
-		{"high-cpu-usage", "alerts", "high-cpu-usage--alerts"},
-		{"high-cpu-usage", "alert-slo-channel", "high-cpu-usage--alert-slo-channel"},
-		{"pod-not-ready", "ops", "pod-not-ready--ops"},
+		{"high-cpu-usage", "ops", "alerts", "high-cpu-usage--ops-alerts"},
+		{"high-cpu-usage", "sre", "alert-slo-channel", "high-cpu-usage--sre-alert-slo-channel"},
+		{"pod-not-ready", "platform", "ops", "pod-not-ready--platform-ops"},
+		// Same channel name, different teams → distinct receiver names.
+		// This is the collision the team segment exists to prevent.
+		{"pod-crashloopbackoff", "team-a", "town-square", "pod-crashloopbackoff--team-a-town-square"},
+		{"pod-crashloopbackoff", "team-b", "town-square", "pod-crashloopbackoff--team-b-town-square"},
 	}
 
 	for _, tc := range cases {
-		t.Run(tc.slug+"/"+tc.channel, func(t *testing.T) {
-			got := receiverNameForChannel(tc.slug, tc.channel)
+		t.Run(tc.slug+"/"+tc.team+"/"+tc.channel, func(t *testing.T) {
+			got := receiverNameForChannel(tc.slug, tc.team, tc.channel)
 			if got != tc.want {
 				t.Fatalf("expected %q, got %q", tc.want, got)
 			}
@@ -43,6 +48,9 @@ func TestReceiverBaseSlug(t *testing.T) {
 		{"high-cpu-usage--alerts", "high-cpu-usage"},
 		{"high-cpu-usage--alert-slo-channel", "high-cpu-usage"},
 		{"pod-not-ready--ops", "pod-not-ready"},
+		// Team-qualified form: slug is still everything before the first `--`,
+		// the team-channel tail is ignored.
+		{"pod-crashloopbackoff--team-a-town-square", "pod-crashloopbackoff"},
 
 		// Unsuffixed names pass through unchanged. Treated as a legacy
 		// shape — receivers created before channel-suffixing existed
diff --git a/server/configuration.go b/server/configuration.go
index 6a0067d..5ca8264 100644
--- a/server/configuration.go
+++ b/server/configuration.go
@@ -132,14 +132,18 @@ type alertConfig struct {
 }
 
 // Names are user-facing identifiers — URL-safe so they can appear in slash
-// command args and YAML output without escaping concerns.
-var alertConfigNameRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,63}$`)
+// command args and YAML output without escaping concerns. The 190-char cap
+// (up from 64) accommodates the team-qualified form <slug>--<team>-<channel>:
+// a long runbook slug (~33) plus a 64-char team slug plus a 64-char channel
+// slug plus separators tops out around 164, so 190 leaves headroom without an
+// unbounded name. Alertmanager and the rendered YAML impose no tighter limit.
+var alertConfigNameRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,189}$`)
 
 // IsValid enforces per-entry invariants. Name validation runs first so
 // downstream errors can reference it.
 func (ac *alertConfig) IsValid() error {
 	if !alertConfigNameRegex.MatchString(ac.Name) {
-		return fmt.Errorf("invalid name %q: must be 1-64 chars, start with [a-z0-9], remainder [a-z0-9_-]", ac.Name)
+		return fmt.Errorf("invalid name %q: must be 1-190 chars, start with [a-z0-9], remainder [a-z0-9_-]", ac.Name)
 	}
 	if ac.Team == "" {
 		return errors.New("must set team")
diff --git a/server/configuration_test.go b/server/configuration_test.go
index 1d727fc..ed75482 100644
--- a/server/configuration_test.go
+++ b/server/configuration_test.go
@@ -101,7 +101,7 @@ func TestAlertConfigIsValid(t *testing.T) {
 		{"name starts with hyphen", func(c *alertConfig) { c.Name = "-bad" }, "invalid name"},
 		{"name has uppercase", func(c *alertConfig) { c.Name = "HighCPU" }, "invalid name"},
 		{"name has spaces", func(c *alertConfig) { c.Name = "high cpu" }, "invalid name"},
-		{"name too long", func(c *alertConfig) { c.Name = strings.Repeat("a", 65) }, "invalid name"},
+		{"name too long", func(c *alertConfig) { c.Name = strings.Repeat("a", 191) }, "invalid name"},
 		{"empty team", func(c *alertConfig) { c.Team = "" }, "team"},
 		{"empty channel", func(c *alertConfig) { c.Channel = "" }, "channel"},
 		{"empty webhookID", func(c *alertConfig) { c.WebhookID = "" }, "webhookID"},

From a327033437149a995b93b2200ec7dee2997256cc Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 11:15:30 -0500
Subject: [PATCH 14/15] docs: rewrite .claude/CLAUDE.md into a real agent guide

Replace the stub with project-specific guidance capturing the invariants
that break silently: catalog count sync (30 runbooks / 31 rules) + drift
guards, tests required in all CI jobs, the three promtool tiers, security
coverage (Grype SARIF gate, the open GO-2026-5932 finding, version pins),
the team-qualified receiver-name contract, the runbook-label join key,
single-source rules file, and auto-update discipline for mirrored counts
and docs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .claude/CLAUDE.md | 131 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 100 insertions(+), 31 deletions(-)

diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md
index ad39538..8e03c00 100644
--- a/.claude/CLAUDE.md
+++ b/.claude/CLAUDE.md
@@ -1,33 +1,102 @@
-# Project Guidelines
+# Project Guidelines — mattermost-plugin-alertmanager
+
+Guidance for any Claude agent working in this repo. Read it before changing
+code, docs, or CI. It captures invariants that are easy to break silently.
 
 ## Overview
-<!-- Describe your project here -->
-
-## Architecture
-<!-- Document your project's architecture and key directories -->
-
-## Coding Conventions
-- Follow existing patterns in the codebase
-- Match the style of surrounding code
-- Use meaningful variable and function names
-- Keep functions focused and small
-
-## Testing
-- Write tests for new functionality
-- Run existing tests before submitting changes
-- Aim for meaningful coverage of critical paths
-
-## Git Workflow
-- Create feature branches for new work
-- Write clear, descriptive commit messages
-- Keep commits focused on a single change
-
-## Error Handling
-- Handle errors explicitly, don't ignore them
-- Provide context when wrapping errors
-- Log errors at appropriate levels
-
-## Dependencies
-- Prefer standard library solutions when available
-- Evaluate dependencies for maintenance and security
-- Keep dependencies up to date
+
+Server-only Mattermost plugin that bridges Prometheus Alertmanager → Mattermost
+via native incoming webhooks. The daily-use value: when an alert fires, the post
+includes the matching runbook's first three diagnostic commands inline,
+pre-filled with the failing host/pod from the alert's labels.
+
+- **Enterprise Edition only.** Deploy/test against `mattermost-enterprise-edition`, never team-edition.
+- **No webapp bundle.** `plugin.json` has a `server` block only. This rules out
+  System Console `type: "custom"` settings and any React UI. Settings are the
+  flat `settings_schema` — there is **no conditional show/hide** between settings.
+
+## Architecture (key paths)
+
+- `server/` — the Go plugin. Table-driven tests, stdlib only (no testify).
+- `runbooks/*.md` — the 30 canonical SRE runbooks, embedded via `//go:embed` (`runbooks_embed.go`).
+- `docs/*.md` — embedded docs, shown in chat (`/alertmanager docs`) and rendered to HTML.
+- `samples/prometheus-rules.yaml` — the shipped example rules. **Single source of truth** for the in-plugin rules page.
+- `build/render-docs/main.go` — renders `docs/*.md` + `runbooks/*.md` **and** `samples/prometheus-rules.yaml` → `public/*.html` at bundle time. Its own Go module.
+- `public/` — **generated** (`make render-docs`, run by `make bundle`). Gitignored. Never hand-edit; edit the source + regenerate.
+- `plugin.json` — manifest + System Console `settings_schema` (header/footer/settings).
+- `.github/workflows/` — `lint`, `test`, `build`, `security`, `release-please`, `pages`, `release`.
+
+## The catalog and the "keep every count in sync" rule
+
+The catalog is **30 runbooks / 31 rules** (`ContainerOOMKilled` reuses the
+`high-memory-usage` runbook, so one extra rule than runbooks). A recurring bug in
+this repo has been stale hardcoded counts ("20 runbooks") left behind when the
+catalog grew. **When the catalog changes, update every mirror in the same PR:**
+
+- `plugin.json` (header, footer, `add`/`remove`/`validate` autocomplete help)
+- `README.md`
+- `docs/` (ALERT_CATALOG, ALERT_REQUIREMENTS, etc.) and the `runbookIndexBody` in `build/render-docs/main.go`
+- `samples/prometheus-rules.yaml` (one rule per runbook) and deploy-local's copy
+- Go comments that name a count (`admin_inventory.go`, `cmd_query.go`)
+- `scaffoldSets` in `server/cmd_scaffold.go` (category → slugs)
+
+**Drift guards already enforce most of this** — run `go test ./server/...`:
+- `TestScaffoldMatchesRunbookFiles` — every scaffold slug has a runbook file and vice versa.
+- `TestScaffoldCategoriesDocumented` — every category appears in help text + autocomplete.
+- `TestRunbookPlaceholdersAllowlisted` — runbook `<label>` placeholders are real labels.
+- `TestDocTopicsResolve` — every `docs` topic maps to a real embedded file.
+
+Adding a runbook = add `runbooks/<slug>.md` + a `scaffoldSets` entry + a rule in
+`samples/prometheus-rules.yaml`. The embed glob and tests handle the rest.
+
+## Testing — required in all CI jobs
+
+Every PR must pass **Test, Lint, Build, and Security**. Add/adjust tests with
+every change; don't merge red.
+
+- **Go:** `go test -race -count=1 ./...`. Table-driven, stdlib. New behavior gets a test.
+- **Lint:** `golangci-lint` pinned at **v2.12.2** in CI. The `modernize` linter is on — e.g. prefer `strings.SplitSeq` over `strings.Split` in range loops. `gofmt` must be clean.
+- **Prometheus rules — three tiers, don't confuse them:**
+  1. `promtool check rules` (syntax/PromQL parse) — CI `validate-rules` job in `test.yml`, runs on `samples/prometheus-rules.yaml`.
+  2. `promtool test rules` (fire/no-fire fixtures) — lives in deploy-local, **not** in this repo's CI yet.
+  3. End-to-end (real AM + `/alertmanager validate --end-to-end`) — manual.
+  `check rules` proves the file loads; it does **not** prove a rule fires or uses a real metric. Say so; don't overstate "validated."
+- **Run promtool locally** without the stack: `docker run --rm --entrypoint promtool -v "$PWD":/w -w /w prom/prometheus:v3.5.0 check rules /w/samples/prometheus-rules.yaml` (the image entrypoint is `/bin/prometheus`, hence the override).
+- **Regenerate docs after editing** `docs/`, `runbooks/`, or `samples/`: `make render-docs`.
+
+## Security — what's covered, and what must stay covered
+
+- **Grype SBOM scan** runs in `security.yml` and uploads SARIF to Code Scanning. Code Scanning rejects empty `artifactLocation.uri`; a jq step anchors it (to `go.sum` / `webapp/package-lock.json`). If you touch that workflow, keep the anchoring or the Security job breaks silently.
+- **Standing finding `GO-2026-5932` is intentionally left OPEN** for a security review. Do **not** suppress, dismiss, or add an ignore rule for it.
+- **No automated CVE gate yet** — target policy is block on CRITICAL/HIGH. When recommending or bumping an image/chart, include the scan command (`trivy image <ref>` or `grype <ref>`) and flag known HIGH/CRITICAL. This is an open hardening item.
+- **Version policy:** exact pins everywhere, never `latest`, never floating ranges. Verify a version live before recommending it (training data is stale). Current pins in use: `prom/prometheus:v3.5.0`, `prom/alertmanager:v0.30.1`, `golangci-lint v2.12.2`.
+- **Secrets:** webhook URLs are channel-bound bearer tokens. `AssembledYAMLTTLHours` auto-deletes the DM'd YAML; `WebhookRotationDays` drives rotation reminders. Don't log webhook URLs.
+- **Dev stacks** (`docker-compose.dev.yml`) bind `0.0.0.0` with `admin`/`password` — local-only convenience, never for a reachable instance.
+
+## Invariants / contracts — break these and things misroute silently
+
+- **Receiver name = `<slug>--<team>-<channel>`** (`receiverNameForChannel`). Team is in the name because channel names are unique only per team (`town-square` is in every team) and Alertmanager requires globally-unique receiver names. The first `--` is the slug boundary (`receiverBaseSlug`); the `<team>-<channel>` tail is identity/display only. Global uniqueness is enforced by duplicate-name rejection in `parseAlertConfigs`, not the separator. Name length cap is 190.
+- **`configsForCurrentChannel` is the scoping chokepoint** for ~13 commands (list/remove/export/validate/about/alerts/…). It scopes by **team + channel** (resolves team via `channel.TeamId` → `GetTeam`). Keep any new channel-scoped command routed through it — scoping by channel name alone leaks across teams.
+- **The `runbook:` label is the join key.** Prometheus rule emits `runbook: <slug>` → Alertmanager route matches it → receiver → runbook rendered in chat. **The plugin owns only the Alertmanager side** (receivers/routes/rendering). The *user's Prometheus* must emit the labelled alerts — `samples/prometheus-rules.yaml` + `/alertmanager rules` give them a starting set.
+- **`samples/prometheus-rules.yaml` is single-source.** `render-docs` reads it live to build the HTML rules page; never duplicate the ruleset into a markdown doc.
+- **WebhookHost:** `WebhookHostPreset` (dropdown of known-good hosts) + `WebhookHost` (custom text) are collapsed by `resolveWebhookHost` — **custom text wins**, then preset, then SiteURL. Resolved into `configuration.WebhookHost` at load so consumers are unchanged.
+
+## Auto-update discipline (do this without being asked)
+
+When you change something with mirrors, update the mirrors in the same PR:
+- A **count/catalog** change → every count site above (and let the drift-guard tests catch misses).
+- The **receiver-name format** → `CONFIGURATION.md`, `SLASH_COMMANDS.md`, and the name tests.
+- A **new command** → dispatch switch + `helpMsg` + autocomplete in `commands.go` (alphabetical order) + a `docs`/help mention.
+- Anything under `docs/`, `runbooks/`, `samples/` → `make render-docs` (never edit `public/` by hand).
+- A **URL the user has to get right** → surface it as an autocomplete hover/suggestion (containerized MM reaches services via `host.docker.internal`, not `localhost`).
+
+## Git / PR workflow
+
+- Conventional commits: `feat:` → minor, `fix:` → patch (release-please). Breaking changes noted in the body.
+- Branch off `main`; never commit straight to it. CI must be green before merge.
+- End commit messages with the `Co-Authored-By` trailer.
+
+## Build / deploy
+
+- `make test` (sub-second, no server) · `make dist` / `make dist-host` · `make deploy-local` (needs `MM_ADMIN_TOKEN`, `MM_SERVICESETTINGS_SITEURL`).
+- Local dev: `docker-compose.dev.yml` + `make docker-setup` (creates `admin`/`password`, Test team; runs in local mode so `mmctl --local` needs no token).

From 0d205ad072fd9077e688be9f0e1869d67d099c06 Mon Sep 17 00:00:00 2001
From: "christopher.fickess" <Christopher.Fickess@MattermostFed.com>
Date: Tue, 14 Jul 2026 11:59:00 -0500
Subject: [PATCH 15/15] =?UTF-8?q?feat:=20admin=20route-tester=20=E2=80=94?=
 =?UTF-8?q?=20severity=20only=20for=20end-to-end=20+=20by-team=20scope?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Route-tester form cleanups on the inventory page:

- Severity is now shown only in end-to-end mode (hidden + disabled for
  simulate and webhook-test). In simulate the routes key on the `runbook`
  label, so a severity-only sim just fell through to the default route and
  confused people; webhook-test ignores severity entirely. Server drops the
  dedicated severity outside end-to-end; a `change` listener on the mode
  dropdown toggles the field. Admins who genuinely have severity-based
  routes still simulate them via the extra-labels box (placeholder now
  hints `severity=critical`).

- Added a Team scope dropdown. Channel names repeat across teams
  (town-square is in every team), so filtering the tester by channel name
  alone targeted the wrong team's receivers for webhook-test/end-to-end.
  filterConfigsByChannel now takes team; the channel dropdown cascades to
  the selected team's channels (teamChannels map + JS, mirroring the
  existing type/value cascade).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 server/admin_inventory.go | 111 ++++++++++++++++++++++++++++++++++----
 1 file changed, 100 insertions(+), 11 deletions(-)

diff --git a/server/admin_inventory.go b/server/admin_inventory.go
index 5243724..e48567e 100644
--- a/server/admin_inventory.go
+++ b/server/admin_inventory.go
@@ -279,10 +279,21 @@ func (p *Plugin) renderInventoryHTML(w http.ResponseWriter, r *http.Request, con
 	simType := strings.TrimSpace(r.URL.Query().Get("simulate_type"))
 	simValue := strings.TrimSpace(r.URL.Query().Get("simulate_value"))
 	simChannel := strings.TrimSpace(r.URL.Query().Get("simulate_channel"))
+	simTeam := strings.TrimSpace(r.URL.Query().Get("simulate_team"))
 	simSeverity := strings.TrimSpace(r.URL.Query().Get("simulate_severity"))
 	simExtra := strings.TrimSpace(r.URL.Query().Get("simulate_extra"))
 
-	formSubmitted := simMode != "" || simType != "" || simValue != "" || simChannel != "" || simSeverity != "" || simExtra != ""
+	// Severity is a knob only for end-to-end (it sets the severity the
+	// synthetic alert fires at). Webhook-test ignores it, and simulate routes
+	// on the `runbook` label, not severity — a severity-only simulate just
+	// falls through to the default route and confuses people. So drop the
+	// dedicated severity outside end-to-end; an admin who genuinely has
+	// severity-based routes still simulates them via the extra-labels box.
+	if simMode != "end-to-end" {
+		simSeverity = ""
+	}
+
+	formSubmitted := simMode != "" || simType != "" || simValue != "" || simChannel != "" || simTeam != "" || simSeverity != "" || simExtra != ""
 
 	var simResult inventorySimResult
 	var simMatrix []inventorySimResult
@@ -291,7 +302,7 @@ func (p *Plugin) renderInventoryHTML(w http.ResponseWriter, r *http.Request, con
 	if formSubmitted {
 		// Decode (type, value) into a list of slugs.
 		targetSlugs := decodeTargetSelection(simType, simValue)
-		filteredConfigs := filterConfigsByChannel(configs, simChannel)
+		filteredConfigs := filterConfigsByChannel(configs, simTeam, simChannel)
 
 		switch simMode {
 		case "webhook-test":
@@ -325,6 +336,34 @@ func (p *Plugin) renderInventoryHTML(w http.ResponseWriter, r *http.Request, con
 	}
 	sort.Strings(channelOptions)
 
+	// Teams that have receivers — the by-team scope dropdown. Channel names
+	// repeat across teams, so scoping by team disambiguates the channel
+	// dropdown and targets webhook-test/end-to-end at the right team.
+	teamOptions := make([]string, 0, len(teamCounts))
+	for t := range teamCounts {
+		teamOptions = append(teamOptions, t)
+	}
+	sort.Strings(teamOptions)
+
+	// team → its channels, for the JS cascade (pick a team → narrow channels).
+	teamChannelSet := make(map[string]map[string]bool)
+	for _, c := range configs {
+		if teamChannelSet[c.Team] == nil {
+			teamChannelSet[c.Team] = make(map[string]bool)
+		}
+		teamChannelSet[c.Team][c.Channel] = true
+	}
+	teamChannels := make(map[string][]string, len(teamChannelSet))
+	for t, chset := range teamChannelSet {
+		chs := make([]string, 0, len(chset))
+		for ch := range chset {
+			chs = append(chs, ch)
+		}
+		sort.Strings(chs)
+		teamChannels[t] = chs
+	}
+	teamChannelsJSON, _ := json.Marshal(teamChannels)
+
 	// Enumerate group names from scaffoldSets (excluding the "all"
 	// alias so we don't confuse it with the matrix-mode target type).
 	groupOptions := make([]string, 0, len(scaffoldSets))
@@ -366,6 +405,7 @@ func (p *Plugin) renderInventoryHTML(w http.ResponseWriter, r *http.Request, con
 		SimulateType       string
 		SimulateValue      string
 		SimulateChannel    string
+		SimulateTeam       string
 		SimulateSeverity   string
 		SimulateExtra      string
 		SimulateResult     inventorySimResult
@@ -374,9 +414,11 @@ func (p *Plugin) renderInventoryHTML(w http.ResponseWriter, r *http.Request, con
 		RunbookSlugs       []string
 		ChannelOptions     []string
 		GroupOptions       []string
+		TeamOptions        []string
 		TargetChannelsJSON template.JS
 		GroupOptionsJSON   template.JS
 		RunbookSlugsJSON   template.JS
+		TeamChannelsJSON   template.JS
 	}{
 		Total:             len(configs),
 		Channels:          countDistinctChannels(configs),
@@ -391,6 +433,7 @@ func (p *Plugin) renderInventoryHTML(w http.ResponseWriter, r *http.Request, con
 		SimulateType:      simType,
 		SimulateValue:     simValue,
 		SimulateChannel:   simChannel,
+		SimulateTeam:      simTeam,
 		SimulateSeverity:  simSeverity,
 		SimulateExtra:     simExtra,
 		SimulateResult:    simResult,
@@ -399,6 +442,7 @@ func (p *Plugin) renderInventoryHTML(w http.ResponseWriter, r *http.Request, con
 		RunbookSlugs:      runbookSlugs(),
 		ChannelOptions:    channelOptions,
 		GroupOptions:      groupOptions,
+		TeamOptions:       teamOptions,
 		// G203 false positives: targetChannelsJSON/groupsJSON/slugsJSON are
 		// already json.Marshal output of server-controlled data (channel
 		// list, group options, embedded runbook slugs — never user input).
@@ -410,6 +454,7 @@ func (p *Plugin) renderInventoryHTML(w http.ResponseWriter, r *http.Request, con
 		TargetChannelsJSON: template.JS(targetChannelsJSON), //nolint:gosec // G203: pre-escaped by json.Marshal (HTML escape default)
 		GroupOptionsJSON:   template.JS(groupsJSON),         //nolint:gosec // G203: pre-escaped by json.Marshal (HTML escape default)
 		RunbookSlugsJSON:   template.JS(slugsJSON),          //nolint:gosec // G203: pre-escaped by json.Marshal (HTML escape default)
+		TeamChannelsJSON:   template.JS(teamChannelsJSON),   //nolint:gosec // G203: pre-escaped by json.Marshal (HTML escape default)
 		// Relative URL: browser resolves against the current URL,
 		// which is /plugins/<id>/admin/inventory. Using `?format=csv`
 		// gives the right target. Earlier bug: using r.URL.Path
@@ -614,17 +659,23 @@ func decodeTargetSelection(typ, value string) []string {
 	return nil
 }
 
-// filterConfigsByChannel scopes the receiver list to those bound to
-// a specific channel. Empty channel = no filtering.
-func filterConfigsByChannel(configs []alertConfig, channel string) []alertConfig {
-	if channel == "" {
+// filterConfigsByChannel scopes the receiver list by team and/or channel.
+// Both empty = no filtering. Team matters because channel names repeat across
+// teams (`town-square` is in every team) — filtering on channel name alone
+// would target the wrong team's receivers for webhook-test / end-to-end.
+func filterConfigsByChannel(configs []alertConfig, team, channel string) []alertConfig {
+	if team == "" && channel == "" {
 		return configs
 	}
 	out := make([]alertConfig, 0, len(configs))
 	for _, c := range configs {
-		if c.Channel == channel {
-			out = append(out, c)
+		if team != "" && c.Team != team {
+			continue
 		}
+		if channel != "" && c.Channel != channel {
+			continue
+		}
+		out = append(out, c)
 	}
 	return out
 }
@@ -1075,6 +1126,13 @@ var inventoryTemplate = template.Must(template.New("inventory").Parse(`<!DOCTYPE
                         <option value="">(pick type first)</option>
                     </select>
                 </div>
+                <div style="display: flex; flex-direction: column; gap: 4px;">
+                    <span style="font-size: 11px; font-weight: 600; color: #666; text-transform: uppercase; letter-spacing: 0.04em;">Team</span>
+                    <select name="simulate_team" style="padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-size: 13px; min-width: 160px;">
+                        <option value="">(any team)</option>
+                        {{range .TeamOptions}}<option value="{{.}}" {{if eq . $.SimulateTeam}}selected{{end}}>{{.}}</option>{{end}}
+                    </select>
+                </div>
                 <div style="display: flex; flex-direction: column; gap: 4px;">
                     <span style="font-size: 11px; font-weight: 600; color: #666; text-transform: uppercase; letter-spacing: 0.04em;">Channel</span>
                     <select name="simulate_channel" style="padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-size: 13px; min-width: 200px;">
@@ -1082,7 +1140,7 @@ var inventoryTemplate = template.Must(template.New("inventory").Parse(`<!DOCTYPE
                         {{range .ChannelOptions}}<option value="{{.}}" {{if eq . $.SimulateChannel}}selected{{end}}>{{.}}</option>{{end}}
                     </select>
                 </div>
-                <div style="display: flex; flex-direction: column; gap: 4px;">
+                <div id="sim-severity-field" style="display: flex; flex-direction: column; gap: 4px;">
                     <span style="font-size: 11px; font-weight: 600; color: #666; text-transform: uppercase; letter-spacing: 0.04em;">Severity</span>
                     <select name="simulate_severity" style="padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-size: 13px;">
                         <option value="">(none)</option>
@@ -1093,7 +1151,7 @@ var inventoryTemplate = template.Must(template.New("inventory").Parse(`<!DOCTYPE
                     </select>
                 </div>
             </div>
-            <input type="text" name="simulate_extra" value="{{.SimulateExtra}}" placeholder="Additional labels (optional): namespace=billing pod=api-7d9 service=checkout" style="padding: 8px 12px; border: 1px solid #ccc; border-radius: 4px; font-size: 13px; font-family: SFMono-Regular, Menlo, Monaco, monospace;">
+            <input type="text" name="simulate_extra" value="{{.SimulateExtra}}" placeholder="Additional labels (optional): severity=critical namespace=billing pod=api-7d9 service=checkout" style="padding: 8px 12px; border: 1px solid #ccc; border-radius: 4px; font-size: 13px; font-family: SFMono-Regular, Menlo, Monaco, monospace;">
             <div>
                 <button type="submit" style="padding: 8px 16px; background: #1d3a8c; color: white; border: none; border-radius: 4px; font-size: 13px; cursor: pointer;">Run</button>
                 <a href="?group={{.GroupMode}}" style="margin-left: 8px; padding: 8px 12px; font-size: 13px; color: #666; text-decoration: none;">Reset</a>
@@ -1299,15 +1357,18 @@ var inventoryTemplate = template.Must(template.New("inventory").Parse(`<!DOCTYPE
         // ?simulate_type=...&simulate_value=...&simulate_channel=...)
         // loads with the picks intact.
         const targetChannels = {{.TargetChannelsJSON}};
+        const teamChannels = {{.TeamChannelsJSON}};
         const allGroups = {{.GroupOptionsJSON}};
         const allSlugs = {{.RunbookSlugsJSON}};
         const initialType = {{.SimulateType | printf "%q"}};
         const initialValue = {{.SimulateValue | printf "%q"}};
         const initialChannel = {{.SimulateChannel | printf "%q"}};
+        const initialTeam = {{.SimulateTeam | printf "%q"}};
 
         const typeSel = document.querySelector('select[name="simulate_type"]');
         const valueSel = document.querySelector('select[name="simulate_value"]');
         const channelSel = document.querySelector('select[name="simulate_channel"]');
+        const teamSel = document.querySelector('select[name="simulate_team"]');
 
         function refreshValueOptions() {
             if (!typeSel || !valueSel) return;
@@ -1350,7 +1411,14 @@ var inventoryTemplate = template.Must(template.New("inventory").Parse(`<!DOCTYPE
         function refreshChannelOptions() {
             if (!channelSel) return;
             const key = computeTargetKey();
-            const channels = (targetChannels[key] || targetChannels[""] || []);
+            let channels = (targetChannels[key] || targetChannels[""] || []);
+            // Narrow to the selected team's channels — channel names repeat
+            // across teams, so without this the dropdown mixes them.
+            const team = teamSel ? teamSel.value : "";
+            if (team && teamChannels[team]) {
+                const allowed = teamChannels[team];
+                channels = channels.filter(function(c) { return allowed.indexOf(c) !== -1; });
+            }
             const desired = channelSel.value || initialChannel;
             let opts = '<option value="">(any channel)</option>';
             for (const c of channels) {
@@ -1366,12 +1434,33 @@ var inventoryTemplate = template.Must(template.New("inventory").Parse(`<!DOCTYPE
                 refreshChannelOptions();
             });
             valueSel.addEventListener('change', refreshChannelOptions);
+            if (teamSel) teamSel.addEventListener('change', refreshChannelOptions);
             // Initial render — restore server-rendered values, then
             // populate downstream dropdowns based on them.
             if (initialType) typeSel.value = initialType;
+            if (teamSel && initialTeam) teamSel.value = initialTeam;
             refreshValueOptions();
             refreshChannelOptions();
         }
+
+        // Severity is an end-to-end-only knob (see server comment). Show the
+        // field only in that mode; disable it otherwise so it never submits —
+        // a severity in simulate/webhook-test is misleading (routes key on the
+        // runbook label). Admins who really want severity in a simulate put it
+        // in the extra-labels box.
+        const modeSel = document.querySelector('select[name="simulate_mode"]');
+        const sevField = document.getElementById('sim-severity-field');
+        const sevSel = document.querySelector('select[name="simulate_severity"]');
+        function refreshSeverityVisibility() {
+            if (!modeSel || !sevField || !sevSel) return;
+            const show = modeSel.value === 'end-to-end';
+            sevField.style.display = show ? '' : 'none';
+            sevSel.disabled = !show;
+        }
+        if (modeSel) {
+            modeSel.addEventListener('change', refreshSeverityVisibility);
+            refreshSeverityVisibility();
+        }
     </script>
 </body>
 </html>