github-actions · Nov 8, 2026 · Nov 7, 2026 · Nov 7, 2026 · Nov 7, 2026
@@ -104,6 +104,9 @@ jobs:
       - name: unseal-openbao-tls
         run: |
           bash tests/unseal-openbao-tls.sh
+      - name: upgrade-in-place-guard
+        run: |
+          bash tests/upgrade-in-place-guard.sh
 
   review-comments:
     # Comments state why, not which review prompted them

@@ -32,6 +32,7 @@
 # Usage:
 #   bin/unseal-openbao.sh <ENV_NAME>                # interactive 3-share unseal
 #   bin/unseal-openbao.sh <ENV_NAME> --status       # read-only seal status (no secrets touched)
+#   bin/unseal-openbao.sh <ENV_NAME> --status-all   # seal state of every OpenBao pod, one line each
 #   bin/unseal-openbao.sh <ENV_NAME> --force-rerun  # feed shares even if already unsealed
 #   bin/unseal-openbao.sh <ENV_NAME> --yes          # skip the interactive confirm prompt
 #
@@ -62,6 +63,9 @@
 #   5   unseal failed mid-stream
 #   6   still sealed after feeding 3 shares
 #   7   TLS verification against OpenBao failed (never bypassed)
+#
+# --status-all exits 0 when every pod is unsealed, 6 when any pod is sealed, and
+# 4 when no pod is found or a pod cannot be read.
 
 set -euo pipefail
 
@@ -293,6 +297,7 @@ SKIP_CONFIRM=0
 for arg in "$@"; do
     case "$arg" in
         --status)        MODE="status" ;;
+        --status-all)    MODE="status-all" ;;
         --force-rerun)   FORCE_RERUN=1 ;;
         --yes|-y)        SKIP_CONFIRM=1 ;;
         -h|--help)
@@ -342,8 +347,10 @@ elif [ "$IS_SANDBOX" -eq 1 ] && [ -r "$KEYS_JSON" ] && jq -e '(.unseal_keys_hex
     SHARE_SOURCE="keys-json"
 fi
 
-# macOS Keychain CLI is only needed for non-sandbox (3-of-5 Shamir) profiles.
-if [ "$IS_SANDBOX" -ne 1 ]; then
+# macOS Keychain CLI is only needed to unseal a non-sandbox (3-of-5 Shamir)
+# profile. The status modes read seal state over ssh and never touch it, so
+# they also run on a host without `security`.
+if [ "$IS_SANDBOX" -ne 1 ] && [ "$MODE" = "unseal" ]; then
     require security    # macOS Keychain CLI
 fi
 
@@ -354,6 +361,35 @@ info "openbao pod:  $OPENBAO_NAMESPACE/$OPENBAO_POD (bao in-pod, TLS verified)"
 info "share dir:    $SHARE_DIR"
 info "keychain svc: $SHARE_KEYCHAIN_NAME"
 
+# ── Status-all mode ──────────────────────────────────────────────────────
+# One line per pod: "<pod>: unsealed|sealed|unreadable". A single-replica site
+# has one pod, so the result is that pod's seal state. A Raft cluster has one
+# line per pod, and one sealed pod fails the check.
+if [ "$MODE" = "status-all" ]; then
+    section "OpenBao seal status, every pod"
+    pods="$(ssh "${ssh_opts[@]}" "$SSH_TARGET" \
+        sudo -n kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml \
+        -n "$OPENBAO_NAMESPACE" get pods -l app.kubernetes.io/name=openbao \
+        -o name 2>/dev/null </dev/null | sed 's#^pod/##' | sort || true)"
+    if [ -z "$pods" ]; then
+        err "no OpenBao pod found in $OPENBAO_NAMESPACE on $SSH_TARGET"
+        exit 4
+    fi
+    rc=0
+    while IFS= read -r OPENBAO_POD; do
+        if ! fetch_status; then
+            info "$OPENBAO_POD: unreadable"
+            [ "$rc" -eq 6 ] || rc=4
+        elif [ "$(printf '%s' "$STATUS_JSON" | jq -r '.sealed')" = "true" ]; then
+            info "$OPENBAO_POD: sealed"
+            rc=6
+        else
+            info "$OPENBAO_POD: unsealed"
+        fi
+    done <<< "$pods"
+    exit "$rc"
+fi
+
 # ── Status mode ──────────────────────────────────────────────────────────
 if [ "$MODE" = "status" ]; then
     section "OpenBao seal status"

@@ -2,7 +2,8 @@
 # upgrade-in-place.sh — idempotent upgrade of an existing DMF env to current main.
 #
 # Canonical upgrade sequence (6 stages):
-#   Stage 1 — unseal           — verify OpenBao is unsealed
+#   Stage 1 — unseal           — verify every OpenBao pod is unsealed, then run
+#                                the rolling-restart guard (102-openbao-ha-guard.yml)
 #   Stage 2 — seed-bao         — backfill platform+app secrets into OpenBao
 #   Stage 3 — pre-seed         — prepare cluster state for apps
 #   Stage 4 — post-seed        — deploy apps (dependent on seed-bao secrets)
@@ -21,16 +22,23 @@
 # the new CCM/platform on its OLD k3s (acceptable in experiment phase; a true
 # k3s migration requires a separate sequential plan).
 #
+# The playbooks of stages 3-5 can restart an OpenBao pod, and a restarted pod is
+# sealed. Stage 1 therefore refuses while any pod is sealed. On a Raft cluster the
+# guard also refuses while any pod is not a healthy voter, and it needs an ops-admin
+# token in OPENBAO_GUARD_TOKEN. On a single replica the guard passes when that pod
+# is unsealed. No flag skips the guard. Unseal each pod with
+# bin/unseal-openbao.sh (OPENBAO_POD=<pod>) before you run the upgrade again.
+#
 # Usage:
 #   bin/upgrade-in-place.sh <ENV_NAME>
 #   bin/upgrade-in-place.sh <ENV_NAME> --yes (skip confirm prompt)
-#   bin/upgrade-in-place.sh <ENV_NAME> --skip-unseal-check (skip sealed check)
+#   bin/upgrade-in-place.sh <ENV_NAME> --skip-unseal-check (skip the stage 1 seal check; the guard still runs)
 #   bin/upgrade-in-place.sh <ENV_NAME> --dry-run (plan without mutating seed-bao)
 #
 # Exit codes:
 #   0   upgraded successfully, converged
 #   1   bad arg / config
-#   2   OpenBao unreachable or sealed
+#   2   OpenBao unreachable, a pod sealed, or the restart guard refused
 #   3   seed-bao failed
 #   4   playbook stage failed
 #   5   convergence gate failed
@@ -114,13 +122,16 @@ echo ""
 if [ "$SKIP_UNSEAL_CHECK" -ne 1 ]; then
   section "Stage 1: OpenBao unsealed precheck"
 
-  status_output=$("$SCRIPT_DIR/unseal-openbao.sh" "$ENV_NAME" --status 2>&1 || true)
+  status_rc=0
+  status_output=$("$SCRIPT_DIR/unseal-openbao.sh" "$ENV_NAME" --status-all 2>&1) || status_rc=$?
 
-  if echo "$status_output" | grep -q '"sealed": false'; then
-    info "OpenBao is unsealed — proceeding"
-  elif echo "$status_output" | grep -q '"sealed": true'; then
-    err "OpenBao is sealed — unseal first:"
-    err "  bin/unseal-openbao.sh $ENV_NAME"
+  if [ "$status_rc" -eq 0 ]; then
+    printf '%s\n' "$status_output" | grep -E ': (un)?sealed$' || true
+    info "every OpenBao pod is unsealed — proceeding"
+  elif [ "$status_rc" -eq 6 ]; then
+    printf '%s\n' "$status_output" | grep -E ': (un)?sealed$' >&2 || true
+    err "at least one OpenBao pod is sealed — unseal each sealed pod first:"
+    err "  OPENBAO_POD=<pod> bin/unseal-openbao.sh $ENV_NAME"
     err "  (or re-run vertical-security/100-openbao.yml)"
     err "then re-run this upgrade."
     exit 2
@@ -134,6 +145,22 @@ else
   section "Stage 1: OpenBao unsealed precheck (SKIPPED)"
 fi
 
+# ─────────────────────────────────────────────────────────────────────────────
+# Stage 1b: restart guard. No flag skips it.
+# ─────────────────────────────────────────────────────────────────────────────
+
+# Guard before any step that can restart an OpenBao pod. It is read-only, so it
+# runs without --check even in a dry run: ansible skips command tasks under
+# --check, and a skipped guard would pass without looking. --skip-unseal-check
+# skips only the seal precheck of stage 1. It never skips this guard, because the
+# guard also reads the Raft state, which the seal precheck does not.
+section "Stage 1b: OpenBao restart guard"
+if ! "$SCRIPT_DIR/run-playbook.sh" "$ENV_NAME" "../dmf-infra/k3s-lab-bootstrap/playbooks/vertical-security/102-openbao-ha-guard.yml"; then
+  err "the restart guard refused — no OpenBao pod was restarted and nothing changed"
+  err "fix what it reports (sealed pod, a pod that is not a healthy voter, no OPENBAO_GUARD_TOKEN on a cluster)"
+  exit 2
+fi
+
 # ─────────────────────────────────────────────────────────────────────────────
 # Stage 2: seed-bao (secret backfill)
 # ─────────────────────────────────────────────────────────────────────────────

@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+# upgrade-in-place-guard.sh — prove the upgrade wrapper checks every OpenBao pod
+# and calls the restart guard before any stage that can restart a pod.
+#
+# Stubs ssh with a fake cluster (pod list and sealed pods come from the
+# environment) and replaces run-playbook.sh and bootstrap-secrets.sh with loggers,
+# so no playbook runs. The real upgrade-in-place.sh and unseal-openbao.sh run.
+#
+# Cases:
+#   A  one pod, unsealed → exit 0; the guard playbook runs first, then the stages
+#   B  three pods, one sealed → exit 2 before any playbook or seed-bao step
+#   C  three pods unsealed, the guard refuses → exit 2, only the guard ran
+#   D  --skip-unseal-check skips only the stage 1 seal check: the guard still runs, and
+#      a refusing guard still stops the run before any other step
+#   G  no flag skips the guard (--skip-unseal-check, --dry-run, --yes)
+#   E  --status-all exit codes: 0 all unsealed, 6 one sealed, 4 no pod
+#   F  --status-all reports each pod on its own line
+
+set -euo pipefail
+
+echo "=== upgrade-in-place-guard test ==="
+
+for cmd in bash jq; do
+  if ! command -v "$cmd" >/dev/null 2>&1; then
+    echo "SKIP: $cmd not on PATH"
+    exit 0
+  fi
+done
+
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+WORK_DIR="$(mktemp -d)"
+trap 'rm -rf "$WORK_DIR"' EXIT
+
+# A private copy of bin/ so the two helper scripts can be replaced by stubs.
+mkdir -p "${WORK_DIR}/repo"
+cp -R "${REPO_DIR}/bin" "${WORK_DIR}/repo/bin"
+BIN="${WORK_DIR}/repo/bin"
+
+export DMF_DATA_ROOT="${WORK_DIR}/dmf-data"
+ENV_ID="test-upgrade-guard"
+ENV_ROOT="${DMF_DATA_ROOT}/envs/${ENV_ID}"
+GROUP_VARS_DIR="${ENV_ROOT}/inventory/group_vars/all"
+mkdir -p "${GROUP_VARS_DIR}" "${WORK_DIR}/stub" "${WORK_DIR}/state"
+cat > "${ENV_ROOT}/inventory/hosts.ini" <<'EOF'
+[k3s_control]
+control-1
+
+[k3s]
+control-1 ansible_host=127.0.0.1 ansible_user=lima
+EOF
+cat > "${GROUP_VARS_DIR}/main.yml" <<'EOF'
+openbao_key_path: /tmp/openbao/breakglass
+EOF
+touch "${ENV_ROOT}/bundle.sops.yaml"
+
+# Fake cluster behind ssh. FAKE_PODS and FAKE_SEALED are space-separated pod names.
+cat > "${WORK_DIR}/stub/ssh" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+args="$*"
+case "$args" in
+  *" get pods "*"-o name"*)
+    for p in ${FAKE_PODS:-}; do printf 'pod/%s\n' "$p"; done
+    exit 0
+    ;;
+  *" bao status"*)
+    pod="$(printf '%s' "$args" | sed -E 's/.* exec -i ([^ ]+) .*/\1/')"
+    case " ${FAKE_SEALED:-} " in
+      *" $pod "*) printf '{"sealed":true,"version":"2.5.2","t":3,"n":5,"progress":0}'; exit 2 ;;
+    esac
+    printf '{"sealed":false,"version":"2.5.2","t":3,"n":5,"progress":0}'
+    exit 0
+    ;;
+esac
+echo "unexpected ssh invocation: $args" >&2
+exit 1
+EOF
+chmod +x "${WORK_DIR}/stub/ssh"
+
+# Stubs for the two helpers. They log the call and succeed, except for the guard
+# playbook when FAKE_GUARD_FAIL=1.
+cat > "${BIN}/run-playbook.sh" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' "$(basename "$2")" >> "$STATE_DIR/playbooks.log"
+if [ "${FAKE_GUARD_FAIL:-0}" = 1 ] && [ "$(basename "$2")" = "102-openbao-ha-guard.yml" ]; then
+  echo "Refusing to restart or drain an OpenBao pod: fake refusal" >&2
+  exit 2
+fi
+exit 0
+EOF
+cat > "${BIN}/bootstrap-secrets.sh" <<'EOF'
+#!/usr/bin/env bash
+printf 'seed-bao\n' >> "$STATE_DIR/playbooks.log"
+exit 0
+EOF
+chmod +x "${BIN}/run-playbook.sh" "${BIN}/bootstrap-secrets.sh"
+
+export PATH="${WORK_DIR}/stub:${PATH}"
+export STATE_DIR="${WORK_DIR}/state"
+
+FAILS=0
+pass() { echo "  PASS: $*"; }
+fail() { echo "  FAIL: $*"; FAILS=$((FAILS + 1)); }
+
+# run_case <pods> <sealed> <guard-fail> <args...> — sets RC and OUT.
+run_case() {
+  FAKE_PODS="$1" FAKE_SEALED="$2" FAKE_GUARD_FAIL="$3"
+  export FAKE_PODS FAKE_SEALED FAKE_GUARD_FAIL
+  shift 3
+  rm -f "${STATE_DIR}/playbooks.log"
+  RC=0
+  OUT="$("${BIN}/upgrade-in-place.sh" "$ENV_ID" --yes "$@" 2>&1)" || RC=$?
+}
+logged() { cat "${STATE_DIR}/playbooks.log" 2>/dev/null || true; }
+
+echo "[A] one pod, unsealed"
+run_case "openbao-0" "" 0
+[ "$RC" -eq 0 ] && pass "exit 0" || fail "exit $RC: $OUT"
+[ "$(logged | head -1)" = "102-openbao-ha-guard.yml" ] && pass "guard ran first" || fail "order: $(logged | tr '\n' ' ')"
+logged | grep -q "bootstrap-provision-pre-seed.yml" && pass "stages ran after the guard" || fail "no stage ran"
+
+echo "[B] three pods, openbao-1 sealed"
+run_case "openbao-0 openbao-1 openbao-2" "openbao-1" 0
+[ "$RC" -eq 2 ] && pass "exit 2" || fail "exit $RC"
+[ -z "$(logged)" ] && pass "no playbook and no seed-bao ran" || fail "ran: $(logged | tr '\n' ' ')"
+printf '%s' "$OUT" | grep -q "openbao-1: sealed" && pass "names the sealed pod" || fail "sealed pod not named"
+
+echo "[C] three pods unsealed, guard refuses"
+run_case "openbao-0 openbao-1 openbao-2" "" 1
+[ "$RC" -eq 2 ] && pass "exit 2" || fail "exit $RC"
+[ "$(logged)" = "102-openbao-ha-guard.yml" ] && pass "only the guard ran" || fail "ran: $(logged | tr '\n' ' ')"
+
+echo "[D] --skip-unseal-check skips the seal check, not the guard"
+run_case "openbao-0 openbao-1 openbao-2" "openbao-1" 0 --skip-unseal-check
+[ "$RC" -eq 0 ] && pass "the seal precheck is skipped (exit 0 with a sealed pod, the stub guard passing)" || fail "exit $RC: $OUT"
+[ "$(logged | head -1)" = "102-openbao-ha-guard.yml" ] && pass "guard still ran first" || fail "order: $(logged | tr '\n' ' ')"
+run_case "openbao-0 openbao-1 openbao-2" "" 1 --skip-unseal-check
+[ "$RC" -eq 2 ] && pass "a refusing guard exits 2 under --skip-unseal-check" || fail "exit $RC"
+[ "$(logged)" = "102-openbao-ha-guard.yml" ] && pass "nothing else ran" || fail "ran: $(logged | tr '\n' ' ')"
+
+echo "[G] no flag skips the guard"
+for flag in --skip-unseal-check --dry-run --yes; do
+  run_case "openbao-0" "" 0 "$flag"
+  logged | grep -q "102-openbao-ha-guard.yml" && pass "guard ran with $flag" || fail "guard did not run with $flag"
+done
+
+echo "[E] --status-all exit codes"
+export FAKE_PODS="openbao-0 openbao-1 openbao-2" FAKE_SEALED=""
+RC=0; "${BIN}/unseal-openbao.sh" "$ENV_ID" --status-all >/dev/null 2>&1 || RC=$?
+[ "$RC" -eq 0 ] && pass "all unsealed → 0" || fail "all unsealed → $RC"
+export FAKE_SEALED="openbao-2"
+RC=0; "${BIN}/unseal-openbao.sh" "$ENV_ID" --status-all >/dev/null 2>&1 || RC=$?
+[ "$RC" -eq 6 ] && pass "one sealed → 6" || fail "one sealed → $RC"
+export FAKE_PODS="" FAKE_SEALED=""
+RC=0; "${BIN}/unseal-openbao.sh" "$ENV_ID" --status-all >/dev/null 2>&1 || RC=$?
+[ "$RC" -eq 4 ] && pass "no pod → 4" || fail "no pod → $RC"
+
+echo "[F] --status-all reports each pod"
+export FAKE_PODS="openbao-0 openbao-1 openbao-2" FAKE_SEALED="openbao-2"
+OUT="$("${BIN}/unseal-openbao.sh" "$ENV_ID" --status-all 2>&1 || true)"
+for line in "openbao-0: unsealed" "openbao-1: unsealed" "openbao-2: sealed"; do
+  printf '%s' "$OUT" | grep -q "$line" && pass "$line" || fail "missing: $line"
+done
+
+if [ "$FAILS" -ne 0 ]; then
+  echo "=== upgrade-in-place-guard: $FAILS failure(s) ==="
+  exit 1
+fi
+echo "=== upgrade-in-place-guard: all cases passed ==="