From 94744df1be038669e96f59c9c04bea6036d5177b Mon Sep 17 00:00:00 2001 From: alovladi007 <83262803+alovladi007@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:57:07 -0400 Subject: [PATCH] feat(provision): one-command automation for runbook steps 2, 4, 5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three idempotent scripts so go-live is copy-paste once the cluster and domain exist (the only parts that need human accounts): - install_addons.sh : ingress-nginx + cert-manager (+ Let's Encrypt ClusterIssuer, webhook-retry) + sealed-secrets + namespace, waits for readiness, prints the ingress IP for the DNS records. - make_sealed_secrets.sh: generates strong values for all five app secrets and seals them against the live cluster — plaintext never touches disk; writes committable manifests + kustomization under k8s/overlays/production/sealed/. Prints the ghcr pull-secret command (the PAT is the user's to create). - wire_github_and_deploy.sh : confirms the kubectl context, sets the KUBE_CONFIG repo secret via gh, tags, and pushes — firing the full CD run; prints the bootstrap_admin follow-up. Runbook sections 2/5/7 now lead with the script and keep the manual flow as reference. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/deployment/PRODUCTION_RUNBOOK.md | 29 ++++++++- scripts/provision/install_addons.sh | 57 ++++++++++++++++++ scripts/provision/make_sealed_secrets.sh | 66 +++++++++++++++++++++ scripts/provision/wire_github_and_deploy.sh | 29 +++++++++ 4 files changed, 180 insertions(+), 1 deletion(-) create mode 100755 scripts/provision/install_addons.sh create mode 100755 scripts/provision/make_sealed_secrets.sh create mode 100755 scripts/provision/wire_github_and_deploy.sh diff --git a/docs/deployment/PRODUCTION_RUNBOOK.md b/docs/deployment/PRODUCTION_RUNBOOK.md index 7fea85e6..73c1d4ab 100644 --- a/docs/deployment/PRODUCTION_RUNBOOK.md +++ b/docs/deployment/PRODUCTION_RUNBOOK.md @@ -57,7 +57,14 @@ kubectl get nodes ## 2. Install cluster add-ons -Three controllers the manifests assume: +One command (installs all three controllers, the ClusterIssuer, and the +namespace, then prints the ingress IP for step 3): + +```bash +./scripts/provision/install_addons.sh you@yourdomain.com +``` + +What it does, if you prefer it by hand: ```bash # NGINX ingress controller (the Ingress uses nginx.ingress.* annotations) @@ -123,6 +130,17 @@ applies from the repo. ## 5. Provision secrets +Application secrets are one command (generates strong values, seals them +against this cluster, writes committable manifests + kustomization under +`k8s/overlays/production/sealed/`): + +```bash +./scripts/provision/make_sealed_secrets.sh +``` + +Only the ghcr pull secret needs your hands (the PAT is yours to create) — +the script prints the exact command. The manual flow, for reference: + The namespace must exist first: ```bash @@ -174,6 +192,15 @@ live `kubectl apply -k` + tagged-image rollout. ## 7. Deploy +Steps 6+7 in one command (sets KUBE_CONFIG from your current kubectl +context via gh, tags, and pushes): + +```bash +./scripts/provision/wire_github_and_deploy.sh v0.1.1 +``` + +Or by hand: + ```bash git tag -a v0.1.1 -m "first cluster deploy" && git push origin v0.1.1 ``` diff --git a/scripts/provision/install_addons.sh b/scripts/provision/install_addons.sh new file mode 100755 index 00000000..fb0deb6c --- /dev/null +++ b/scripts/provision/install_addons.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Runbook step 2 — install the three cluster add-ons the manifests assume: +# ingress-nginx, cert-manager (+ Let's Encrypt ClusterIssuer), sealed-secrets. +# +# Prereq: kubectl pointed at the target cluster (runbook step 1). +# Usage: ./scripts/provision/install_addons.sh you@example.com +# Idempotent — safe to re-run. +set -euo pipefail + +ACME_EMAIL="${1:?usage: install_addons.sh }" + +echo "==> Cluster: $(kubectl config current-context)" +kubectl get nodes >/dev/null # fail fast if kubectl isn't wired + +echo "==> ingress-nginx" +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/cloud/deploy.yaml +kubectl wait --namespace ingress-nginx --for=condition=Available deployment/ingress-nginx-controller --timeout=5m + +echo "==> cert-manager" +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml +kubectl wait --namespace cert-manager --for=condition=Available deployment/cert-manager --timeout=5m +kubectl wait --namespace cert-manager --for=condition=Available deployment/cert-manager-webhook --timeout=5m + +echo "==> Let's Encrypt ClusterIssuer (email: $ACME_EMAIL)" +# Retry: the webhook can reject the first apply right after it reports Available. +for i in 1 2 3 4 5; do + if kubectl apply -f - < sealed-secrets controller" +kubectl apply -f https://github.com/bitnami-labs/sealed-secrets/releases/latest/download/controller.yaml +kubectl wait --namespace kube-system --for=condition=Available deployment/sealed-secrets-controller --timeout=5m + +echo "==> namespace" +kubectl create namespace spectra-lab --dry-run=client -o yaml | kubectl apply -f - + +echo +echo "✅ Add-ons installed. Ingress external IP (create your DNS A records to it):" +kubectl get svc -n ingress-nginx ingress-nginx-controller \ + -o jsonpath='{.status.loadBalancer.ingress[0].ip}{"\n"}' || \ + echo " (LoadBalancer IP still provisioning — re-run: kubectl get svc -n ingress-nginx)" diff --git a/scripts/provision/make_sealed_secrets.sh b/scripts/provision/make_sealed_secrets.sh new file mode 100755 index 00000000..4582b7b0 --- /dev/null +++ b/scripts/provision/make_sealed_secrets.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Runbook step 4 — generate strong values for every application secret and +# seal them against THIS cluster's sealed-secrets controller. Plaintext +# never touches disk; the sealed manifests written under +# k8s/overlays/production/ are safe to commit (that is the point of +# Sealed Secrets — see docs/deployment/SECRETS.md). +# +# Prereqs: install_addons.sh done; kubeseal CLI (brew install kubeseal). +# Usage: ./scripts/provision/make_sealed_secrets.sh +# Re-running rotates every generated secret — do that deliberately, and +# roll the deployments afterwards. +set -euo pipefail + +NS=spectra-lab +OUT="k8s/overlays/production/sealed" +command -v kubeseal >/dev/null || { echo "kubeseal not found — brew install kubeseal"; exit 1; } +kubectl get deployment sealed-secrets-controller -n kube-system >/dev/null \ + || { echo "sealed-secrets controller missing — run install_addons.sh first"; exit 1; } +mkdir -p "$OUT" + +seal() { # seal + local name="$1"; shift + kubectl create secret generic "$name" --namespace "$NS" "$@" \ + --dry-run=client -o yaml | kubeseal --format yaml > "$OUT/$name-sealed.yaml" + echo " sealed: $OUT/$name-sealed.yaml" +} + +echo "==> generating + sealing application secrets" +PG_PASS=$(openssl rand -base64 32 | tr -d '/+=' | head -c 32) + +seal jwt-secret --from-literal=secret-key="$(openssl rand -base64 48)" +seal postgres-credentials --from-literal=username=spectra --from-literal=password="$PG_PASS" +seal database-credentials --from-literal=postgres-url="postgresql+psycopg://spectra:${PG_PASS}@postgres:5432/spectra" +seal redis-credentials --from-literal=redis-url="redis://redis:6379/0" +seal minio-credentials --from-literal=root-user=spectra-minio-admin \ + --from-literal=root-password="$(openssl rand -base64 32 | tr -d '/+=' | head -c 32)" +unset PG_PASS + +# The overlay must (a) load the sealed manifests and (b) stop generating +# the CHANGE_ME placeholder secrets that would fight them for the same names. +cat > "$OUT/kustomization.yaml" <<'EOF' +# Generated by scripts/provision/make_sealed_secrets.sh — sealed secrets +# for the production overlay. Encrypted with this cluster's controller +# key; safe to commit, useless to anyone without the cluster. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - jwt-secret-sealed.yaml + - postgres-credentials-sealed.yaml + - database-credentials-sealed.yaml + - redis-credentials-sealed.yaml + - minio-credentials-sealed.yaml +EOF +echo " wrote: $OUT/kustomization.yaml" + +echo +echo "✅ Sealed. Remaining by hand:" +echo " 1. Reference '- sealed' in k8s/overlays/production/kustomization.yaml resources," +echo " and disable the base secretGenerator placeholders there (see runbook §5)." +echo " 2. Registry pull secret (needs YOUR GitHub PAT with read:packages):" +echo " kubectl create secret docker-registry ghcr-pull-secret \\" +echo " --namespace $NS --docker-server=ghcr.io \\" +echo " --docker-username= --docker-password= \\" +echo " --docker-email=unused@example.com" +echo " 3. Commit the sealed manifests (they are encrypted):" +echo " git add $OUT && git commit" diff --git a/scripts/provision/wire_github_and_deploy.sh b/scripts/provision/wire_github_and_deploy.sh new file mode 100755 index 00000000..bd1c7c97 --- /dev/null +++ b/scripts/provision/wire_github_and_deploy.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Runbook step 5 — hand the cluster to GitHub Actions and fire the deploy. +# Sets the KUBE_CONFIG repo secret from the current kubectl context (via +# gh, which uses your existing GitHub auth) and pushes the release tag +# that triggers the full CD run: build → scan → push → release → deploy. +# +# Prereqs: steps 2+4 done; gh CLI authenticated; clean main checkout. +# Usage: ./scripts/provision/wire_github_and_deploy.sh v0.1.1 +set -euo pipefail + +TAG="${1:?usage: wire_github_and_deploy.sh }" +[[ "$TAG" == v* ]] || { echo "tag must start with v (CD triggers on v*)"; exit 1; } + +echo "==> Cluster to be handed to CI: $(kubectl config current-context)" +read -r -p " Set KUBE_CONFIG on the GitHub repo from this context? [y/N] " ok +[[ "$ok" == y* || "$ok" == Y* ]] || { echo "aborted"; exit 1; } + +kubectl config view --raw --minify | base64 | gh secret set KUBE_CONFIG +echo " KUBE_CONFIG set." + +echo "==> tagging $TAG on current HEAD ($(git rev-parse --short HEAD))" +git tag -a "$TAG" -m "SPECTRA-Lab $TAG — cluster deploy" +git push origin "$TAG" + +echo +echo "✅ CD firing. Watch it: gh run watch \$(gh run list --workflow=cd.yml --limit 1 --json databaseId --jq '.[0].databaseId')" +echo " When green: kubectl get pods -n spectra-lab" +echo " Then runbook step 6: kubectl exec -n spectra-lab deploy/analysis-service -- \\" +echo " python bootstrap_admin.py --org \"Your Lab\" --email you@yourdomain.com"