From 7990535fd026701ac1222af1a85b9009bf9d2b03 Mon Sep 17 00:00:00 2001 From: znerol2 <6800371+znerol2@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:42:38 +0100 Subject: [PATCH 1/3] feat(openbao): check every pod and run the HA guard before upgrades WP3 of the Tier 2 operator unlock plan. upgrade-in-place.sh now checks the seal state of every OpenBao pod, not only openbao-0, and runs the rolling-restart guard from dmf-infra before any OpenBao restart. On a single replica the guard passes when that pod is unsealed. `--skip-unseal-check` keeps its old seal-only meaning and never skips the guard. unseal-openbao.sh gains --status-all. Merge after the dmf-infra change that adds the guard playbook. Refs dmfdeploy/dmfdeploy#864 Signed-off-by: znerol2 <6800371+znerol2@users.noreply.github.com> --- bin/unseal-openbao.sh | 34 +++++++ bin/upgrade-in-place.sh | 45 +++++++-- tests/upgrade-in-place-guard.sh | 169 ++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 9 deletions(-) create mode 100755 tests/upgrade-in-place-guard.sh diff --git a/bin/unseal-openbao.sh b/bin/unseal-openbao.sh index cc3415c..19275fe 100755 --- a/bin/unseal-openbao.sh +++ b/bin/unseal-openbao.sh @@ -32,6 +32,7 @@ # Usage: # bin/unseal-openbao.sh # interactive 3-share unseal # bin/unseal-openbao.sh --status # read-only seal status (no secrets touched) +# bin/unseal-openbao.sh --status-all # seal state of every OpenBao pod, one line each # bin/unseal-openbao.sh --force-rerun # feed shares even if already unsealed # bin/unseal-openbao.sh --yes # skip the interactive confirm prompt # @@ -62,6 +63,9 @@ # 5 unseal failed mid-stream # 6 still sealed after feeding 3 shares # 7 TLS verification against OpenBao failed (never bypassed) +# +# --status-all exits 0 when every pod is unsealed, 6 when any pod is sealed, and +# 4 when no pod is found or a pod cannot be read. set -euo pipefail @@ -293,6 +297,7 @@ SKIP_CONFIRM=0 for arg in "$@"; do case "$arg" in --status) MODE="status" ;; + --status-all) MODE="status-all" ;; --force-rerun) FORCE_RERUN=1 ;; --yes|-y) SKIP_CONFIRM=1 ;; -h|--help) @@ -354,6 +359,35 @@ info "openbao pod: $OPENBAO_NAMESPACE/$OPENBAO_POD (bao in-pod, TLS verified)" info "share dir: $SHARE_DIR" info "keychain svc: $SHARE_KEYCHAIN_NAME" +# ── Status-all mode ────────────────────────────────────────────────────── +# One line per pod: ": unsealed|sealed|unreadable". A single-replica site +# has one pod, so the result is that pod's seal state. A Raft cluster has one +# line per pod, and one sealed pod fails the check. +if [ "$MODE" = "status-all" ]; then + section "OpenBao seal status, every pod" + pods="$(ssh "${ssh_opts[@]}" "$SSH_TARGET" \ + sudo -n kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml \ + -n "$OPENBAO_NAMESPACE" get pods -l app.kubernetes.io/name=openbao \ + -o name 2>/dev/null ) before you run the upgrade again. +# # Usage: # bin/upgrade-in-place.sh # bin/upgrade-in-place.sh --yes (skip confirm prompt) -# bin/upgrade-in-place.sh --skip-unseal-check (skip sealed check) +# bin/upgrade-in-place.sh --skip-unseal-check (skip the stage 1 seal check; the guard still runs) # bin/upgrade-in-place.sh --dry-run (plan without mutating seed-bao) # # Exit codes: # 0 upgraded successfully, converged # 1 bad arg / config -# 2 OpenBao unreachable or sealed +# 2 OpenBao unreachable, a pod sealed, or the restart guard refused # 3 seed-bao failed # 4 playbook stage failed # 5 convergence gate failed @@ -114,13 +122,16 @@ echo "" if [ "$SKIP_UNSEAL_CHECK" -ne 1 ]; then section "Stage 1: OpenBao unsealed precheck" - status_output=$("$SCRIPT_DIR/unseal-openbao.sh" "$ENV_NAME" --status 2>&1 || true) + status_rc=0 + status_output=$("$SCRIPT_DIR/unseal-openbao.sh" "$ENV_NAME" --status-all 2>&1) || status_rc=$? - if echo "$status_output" | grep -q '"sealed": false'; then - info "OpenBao is unsealed — proceeding" - elif echo "$status_output" | grep -q '"sealed": true'; then - err "OpenBao is sealed — unseal first:" - err " bin/unseal-openbao.sh $ENV_NAME" + if [ "$status_rc" -eq 0 ]; then + printf '%s\n' "$status_output" | grep -E ': (un)?sealed$' || true + info "every OpenBao pod is unsealed — proceeding" + elif [ "$status_rc" -eq 6 ]; then + printf '%s\n' "$status_output" | grep -E ': (un)?sealed$' >&2 || true + err "at least one OpenBao pod is sealed — unseal each sealed pod first:" + err " OPENBAO_POD= bin/unseal-openbao.sh $ENV_NAME" err " (or re-run vertical-security/100-openbao.yml)" err "then re-run this upgrade." exit 2 @@ -134,6 +145,22 @@ else section "Stage 1: OpenBao unsealed precheck (SKIPPED)" fi +# ───────────────────────────────────────────────────────────────────────────── +# Stage 1b: restart guard. No flag skips it. +# ───────────────────────────────────────────────────────────────────────────── + +# Guard before any step that can restart an OpenBao pod. It is read-only, so it +# runs without --check even in a dry run: ansible skips command tasks under +# --check, and a skipped guard would pass without looking. --skip-unseal-check +# skips only the seal precheck of stage 1. It never skips this guard, because the +# guard also reads the Raft state, which the seal precheck does not. +section "Stage 1b: OpenBao restart guard" +if ! "$SCRIPT_DIR/run-playbook.sh" "$ENV_NAME" "../dmf-infra/k3s-lab-bootstrap/playbooks/vertical-security/102-openbao-ha-guard.yml"; then + err "the restart guard refused — no OpenBao pod was restarted and nothing changed" + err "fix what it reports (sealed pod, a pod that is not a healthy voter, no OPENBAO_GUARD_TOKEN on a cluster)" + exit 2 +fi + # ───────────────────────────────────────────────────────────────────────────── # Stage 2: seed-bao (secret backfill) # ───────────────────────────────────────────────────────────────────────────── diff --git a/tests/upgrade-in-place-guard.sh b/tests/upgrade-in-place-guard.sh new file mode 100755 index 0000000..5bbf79c --- /dev/null +++ b/tests/upgrade-in-place-guard.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# upgrade-in-place-guard.sh — prove the upgrade wrapper checks every OpenBao pod +# and calls the restart guard before any stage that can restart a pod. +# +# Stubs ssh with a fake cluster (pod list and sealed pods come from the +# environment) and replaces run-playbook.sh and bootstrap-secrets.sh with loggers, +# so no playbook runs. The real upgrade-in-place.sh and unseal-openbao.sh run. +# +# Cases: +# A one pod, unsealed → exit 0; the guard playbook runs first, then the stages +# B three pods, one sealed → exit 2 before any playbook or seed-bao step +# C three pods unsealed, the guard refuses → exit 2, only the guard ran +# D --skip-unseal-check skips only the stage 1 seal check: the guard still runs, and +# a refusing guard still stops the run before any other step +# G no flag skips the guard (--skip-unseal-check, --dry-run, --yes) +# E --status-all exit codes: 0 all unsealed, 6 one sealed, 4 no pod +# F --status-all reports each pod on its own line + +set -euo pipefail + +echo "=== upgrade-in-place-guard test ===" + +for cmd in bash jq; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "SKIP: $cmd not on PATH" + exit 0 + fi +done + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +# A private copy of bin/ so the two helper scripts can be replaced by stubs. +mkdir -p "${WORK_DIR}/repo" +cp -R "${REPO_DIR}/bin" "${WORK_DIR}/repo/bin" +BIN="${WORK_DIR}/repo/bin" + +export DMF_DATA_ROOT="${WORK_DIR}/dmf-data" +ENV_ID="test-upgrade-guard" +ENV_ROOT="${DMF_DATA_ROOT}/envs/${ENV_ID}" +GROUP_VARS_DIR="${ENV_ROOT}/inventory/group_vars/all" +mkdir -p "${GROUP_VARS_DIR}" "${WORK_DIR}/stub" "${WORK_DIR}/state" +cat > "${ENV_ROOT}/inventory/hosts.ini" <<'EOF' +[k3s_control] +control-1 + +[k3s] +control-1 ansible_host=127.0.0.1 ansible_user=lima +EOF +cat > "${GROUP_VARS_DIR}/main.yml" <<'EOF' +openbao_key_path: /tmp/openbao/breakglass +EOF +touch "${ENV_ROOT}/bundle.sops.yaml" + +# Fake cluster behind ssh. FAKE_PODS and FAKE_SEALED are space-separated pod names. +cat > "${WORK_DIR}/stub/ssh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +args="$*" +case "$args" in + *" get pods "*"-o name"*) + for p in ${FAKE_PODS:-}; do printf 'pod/%s\n' "$p"; done + exit 0 + ;; + *" bao status"*) + pod="$(printf '%s' "$args" | sed -E 's/.* exec -i ([^ ]+) .*/\1/')" + case " ${FAKE_SEALED:-} " in + *" $pod "*) printf '{"sealed":true,"version":"2.5.2","t":3,"n":5,"progress":0}'; exit 2 ;; + esac + printf '{"sealed":false,"version":"2.5.2","t":3,"n":5,"progress":0}' + exit 0 + ;; +esac +echo "unexpected ssh invocation: $args" >&2 +exit 1 +EOF +chmod +x "${WORK_DIR}/stub/ssh" + +# Stubs for the two helpers. They log the call and succeed, except for the guard +# playbook when FAKE_GUARD_FAIL=1. +cat > "${BIN}/run-playbook.sh" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$(basename "$2")" >> "$STATE_DIR/playbooks.log" +if [ "${FAKE_GUARD_FAIL:-0}" = 1 ] && [ "$(basename "$2")" = "102-openbao-ha-guard.yml" ]; then + echo "Refusing to restart or drain an OpenBao pod: fake refusal" >&2 + exit 2 +fi +exit 0 +EOF +cat > "${BIN}/bootstrap-secrets.sh" <<'EOF' +#!/usr/bin/env bash +printf 'seed-bao\n' >> "$STATE_DIR/playbooks.log" +exit 0 +EOF +chmod +x "${BIN}/run-playbook.sh" "${BIN}/bootstrap-secrets.sh" + +export PATH="${WORK_DIR}/stub:${PATH}" +export STATE_DIR="${WORK_DIR}/state" + +FAILS=0 +pass() { echo " PASS: $*"; } +fail() { echo " FAIL: $*"; FAILS=$((FAILS + 1)); } + +# run_case — sets RC and OUT. +run_case() { + FAKE_PODS="$1" FAKE_SEALED="$2" FAKE_GUARD_FAIL="$3" + export FAKE_PODS FAKE_SEALED FAKE_GUARD_FAIL + shift 3 + rm -f "${STATE_DIR}/playbooks.log" + RC=0 + OUT="$("${BIN}/upgrade-in-place.sh" "$ENV_ID" --yes "$@" 2>&1)" || RC=$? +} +logged() { cat "${STATE_DIR}/playbooks.log" 2>/dev/null || true; } + +echo "[A] one pod, unsealed" +run_case "openbao-0" "" 0 +[ "$RC" -eq 0 ] && pass "exit 0" || fail "exit $RC: $OUT" +[ "$(logged | head -1)" = "102-openbao-ha-guard.yml" ] && pass "guard ran first" || fail "order: $(logged | tr '\n' ' ')" +logged | grep -q "bootstrap-provision-pre-seed.yml" && pass "stages ran after the guard" || fail "no stage ran" + +echo "[B] three pods, openbao-1 sealed" +run_case "openbao-0 openbao-1 openbao-2" "openbao-1" 0 +[ "$RC" -eq 2 ] && pass "exit 2" || fail "exit $RC" +[ -z "$(logged)" ] && pass "no playbook and no seed-bao ran" || fail "ran: $(logged | tr '\n' ' ')" +printf '%s' "$OUT" | grep -q "openbao-1: sealed" && pass "names the sealed pod" || fail "sealed pod not named" + +echo "[C] three pods unsealed, guard refuses" +run_case "openbao-0 openbao-1 openbao-2" "" 1 +[ "$RC" -eq 2 ] && pass "exit 2" || fail "exit $RC" +[ "$(logged)" = "102-openbao-ha-guard.yml" ] && pass "only the guard ran" || fail "ran: $(logged | tr '\n' ' ')" + +echo "[D] --skip-unseal-check skips the seal check, not the guard" +run_case "openbao-0 openbao-1 openbao-2" "openbao-1" 0 --skip-unseal-check +[ "$RC" -eq 0 ] && pass "the seal precheck is skipped (exit 0 with a sealed pod, the stub guard passing)" || fail "exit $RC: $OUT" +[ "$(logged | head -1)" = "102-openbao-ha-guard.yml" ] && pass "guard still ran first" || fail "order: $(logged | tr '\n' ' ')" +run_case "openbao-0 openbao-1 openbao-2" "" 1 --skip-unseal-check +[ "$RC" -eq 2 ] && pass "a refusing guard exits 2 under --skip-unseal-check" || fail "exit $RC" +[ "$(logged)" = "102-openbao-ha-guard.yml" ] && pass "nothing else ran" || fail "ran: $(logged | tr '\n' ' ')" + +echo "[G] no flag skips the guard" +for flag in --skip-unseal-check --dry-run --yes; do + run_case "openbao-0" "" 0 "$flag" + logged | grep -q "102-openbao-ha-guard.yml" && pass "guard ran with $flag" || fail "guard did not run with $flag" +done + +echo "[E] --status-all exit codes" +export FAKE_PODS="openbao-0 openbao-1 openbao-2" FAKE_SEALED="" +RC=0; "${BIN}/unseal-openbao.sh" "$ENV_ID" --status-all >/dev/null 2>&1 || RC=$? +[ "$RC" -eq 0 ] && pass "all unsealed → 0" || fail "all unsealed → $RC" +export FAKE_SEALED="openbao-2" +RC=0; "${BIN}/unseal-openbao.sh" "$ENV_ID" --status-all >/dev/null 2>&1 || RC=$? +[ "$RC" -eq 6 ] && pass "one sealed → 6" || fail "one sealed → $RC" +export FAKE_PODS="" FAKE_SEALED="" +RC=0; "${BIN}/unseal-openbao.sh" "$ENV_ID" --status-all >/dev/null 2>&1 || RC=$? +[ "$RC" -eq 4 ] && pass "no pod → 4" || fail "no pod → $RC" + +echo "[F] --status-all reports each pod" +export FAKE_PODS="openbao-0 openbao-1 openbao-2" FAKE_SEALED="openbao-2" +OUT="$("${BIN}/unseal-openbao.sh" "$ENV_ID" --status-all 2>&1 || true)" +for line in "openbao-0: unsealed" "openbao-1: unsealed" "openbao-2: sealed"; do + printf '%s' "$OUT" | grep -q "$line" && pass "$line" || fail "missing: $line" +done + +if [ "$FAILS" -ne 0 ]; then + echo "=== upgrade-in-place-guard: $FAILS failure(s) ===" + exit 1 +fi +echo "=== upgrade-in-place-guard: all cases passed ===" From dc94135c956eaf9ba9cf1e86c9d18442a0db9094 Mon Sep 17 00:00:00 2001 From: znerol2 <6800371+znerol2@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:42:55 +0100 Subject: [PATCH 2/3] ci: run the upgrade-in-place HA guard test Refs dmfdeploy/dmfdeploy#864 Signed-off-by: znerol2 <6800371+znerol2@users.noreply.github.com> --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bad1f92..434540b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,9 @@ jobs: - name: unseal-openbao-tls run: | bash tests/unseal-openbao-tls.sh + - name: upgrade-in-place-guard + run: | + bash tests/upgrade-in-place-guard.sh review-comments: # Comments state why, not which review prompted them From b34e1643cca56b76c66f0641a029b1c35ab0a904 Mon Sep 17 00:00:00 2001 From: znerol2 <6800371+znerol2@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:20:31 +0100 Subject: [PATCH 3/3] fix(openbao): don't require the Keychain tool for status checks unseal-openbao.sh required macOS's `security` command in every mode on a non-sandbox profile. The status modes read seal state over ssh and never use it, so on a Linux host (the CI runner included) --status and --status-all exited before printing anything. Require it only to unseal. Refs dmfdeploy/dmfdeploy#864 Signed-off-by: znerol2 <6800371+znerol2@users.noreply.github.com> --- bin/unseal-openbao.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/unseal-openbao.sh b/bin/unseal-openbao.sh index 19275fe..3884de1 100755 --- a/bin/unseal-openbao.sh +++ b/bin/unseal-openbao.sh @@ -347,8 +347,10 @@ elif [ "$IS_SANDBOX" -eq 1 ] && [ -r "$KEYS_JSON" ] && jq -e '(.unseal_keys_hex SHARE_SOURCE="keys-json" fi -# macOS Keychain CLI is only needed for non-sandbox (3-of-5 Shamir) profiles. -if [ "$IS_SANDBOX" -ne 1 ]; then +# macOS Keychain CLI is only needed to unseal a non-sandbox (3-of-5 Shamir) +# profile. The status modes read seal state over ssh and never touch it, so +# they also run on a host without `security`. +if [ "$IS_SANDBOX" -ne 1 ] && [ "$MODE" = "unseal" ]; then require security # macOS Keychain CLI fi