deployment.sh

#!/usr/bin/env bash
# ============================================================================
# deployment.sh — IsItObservable Sympozium episode
# ----------------------------------------------------------------------------
# Deploys the full stack end-to-end onto the workload cluster:
#   1. Platform pieces   : MetalLB, csi-driver-nfs (+ default StorageClass),
#                          cert-manager, (optional) kgateway / Gateway API
#   1½. Observability     : OpenTelemetry Operator + two collectors in the default
#                          namespace (oteld daemonset + otel gateway statefulset)
#                          and the Dynatrace Operator/DynaKube + Dynatrace MCP.
#                          Sympozium is pointed at the `otel` gateway collector,
#                          NOT its built-in collector.
#   2. Control plane      : Sympozium via Helm (published repo: deploy.sympozium.ai/charts)
#   3. Skills + Policy    : the custom `bmad` SkillPack and the `bmad-guardrails`
#                          SympoziumPolicy (tool gating, sub-agent caps, sandbox
#                          bounds, feature gates) referenced by the Ensemble
#   4. Secrets            : Ollama placeholder + Slack tokens
#   5. Ensemble           : the `bmad-ensemble` Ensemble (coding + review +
#                          tech-lead) on local Ollama models served by the
#                          macstudio; the Ensemble carries the macstudio baseURL
#                          so every generated Agent uses the local models
#                          (+ optional gVisor Agent Sandbox via --with-sandbox)
#   6. Verification       : wait for rollouts and print the resulting topology
#
# Run this once your workload cluster is up with a working kubeconfig and a CNI
# installed (so nodes are Ready). The script verifies the cluster and applies the
# remaining platform pieces (MetalLB, csi-driver-nfs, cert-manager).
#
# Usage:
#   export MACSTUDIO_IP=192.0.2.50            # Ollama host (REQUIRED)
#   export METALLB_IP_RANGE=192.0.2.240-192.0.2.250
#   export NFS_SERVER=192.0.2.5
#   export NFS_SHARE=/export/sympozium
#   export SLACK_BOT_TOKEN=xoxb-...          # optional (Slack channel)
#   export SLACK_APP_TOKEN=xapp-...          # optional (Socket Mode)
#   export DT_TENANT_URL=https://abc12345.live.dynatrace.com  # optional (Dynatrace backend)
#   export DT_API_TOKEN=dt0c01....           # optional (Dynatrace data-ingest + API token)
#   ./deployment.sh                          # full deploy
#   ./deployment.sh --skip-platform          # cluster pieces already provisioned
#   ./deployment.sh --skip-observability     # skip OTel operator/collectors + Dynatrace
#   ./deployment.sh --skip-ensemble          # control plane only
#   ./deployment.sh --with-sandbox           # run agents in a gVisor Agent Sandbox
#   INSTALL_KGATEWAY=true ./deployment.sh    # also install kgateway + Gateway API
#
# A SympoziumPolicy ("bmad-guardrails") is always applied and referenced by the
# Ensemble (spec.policyRef). --with-sandbox additionally enables the Kubernetes
# Agent Sandbox (gVisor) on every agent — needs the agent-sandbox CRDs + a gvisor
# RuntimeClass on the nodes (GKE Sandbox, or gVisor installed on Proxmox nodes).
# ============================================================================
set -euo pipefail

# --- pinned versions --------------------------------------------------------
CERT_MANAGER_VERSION="${CERT_MANAGER_VERSION:-v1.17.1}"
METALLB_VERSION="${METALLB_VERSION:-v0.14.8}"
CSI_NFS_VERSION="${CSI_NFS_VERSION:-v4.9.0}"
GATEWAY_API_VERSION="${GATEWAY_API_VERSION:-v1.2.1}"
KGATEWAY_VERSION="${KGATEWAY_VERSION:-v2.0.0}"
OTEL_OPERATOR_VERSION="${OTEL_OPERATOR_VERSION:-latest}"   # OpenTelemetry Operator release tag, or "latest"
DT_OPERATOR_VERSION="${DT_OPERATOR_VERSION:-1.9.0}"        # Dynatrace Operator Helm chart version

# --- config -----------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EPISODE_DIR="${SCRIPT_DIR}/deploy/bmad-episode"   # platform templates (MetalLB / NFS) live here
SYMPO_DIR="${SCRIPT_DIR}/sympozium"               # all Sympozium assets: skills/, ensemble/, mcp/, values.yaml
SYMPO_NS="${SYMPO_NS:-sympozium-system}"
PACK_NS="${PACK_NS:-default}"
HELM_RELEASE="${HELM_RELEASE:-sympozium}"
SYMPO_HELM_REPO="${SYMPO_HELM_REPO:-https://deploy.sympozium.ai/charts}"
OLLAMA_PORT="${OLLAMA_PORT:-11434}"
# The BMAD crew personas (one Agent stamped out per slug as bmad-ensemble-<slug>).
CREW="cto brainstormer product-manager architect story-writer code-reviewer testing-architect challenger devops-engineer o11y-engineer"

SKIP_PLATFORM="false"
SKIP_ENSEMBLE="false"
SKIP_OBSERVABILITY="${SKIP_OBSERVABILITY:-false}"
INSTALL_KGATEWAY="${INSTALL_KGATEWAY:-false}"
WITH_SANDBOX="${WITH_SANDBOX:-false}"
SANDBOX_RUNTIME_CLASS="${SANDBOX_RUNTIME_CLASS:-gvisor}"

# --- helpers ----------------------------------------------------------------
c_info()  { printf '\033[1;34m==>\033[0m %s\n' "$*"; }
c_step()  { printf '\033[1;36m  ->\033[0m %s\n' "$*"; }
c_warn()  { printf '\033[1;33m  ! \033[0m %s\n' "$*" >&2; }
c_err()   { printf '\033[1;31m  ! \033[0m %s\n' "$*" >&2; exit 1; }
need()    { command -v "$1" >/dev/null 2>&1 || c_err "Required tool '$1' not found in PATH."; }

parse_args() {
    while [ $# -gt 0 ]; do
        case "$1" in
            --skip-platform) SKIP_PLATFORM="true" ;;
            --skip-ensemble) SKIP_ENSEMBLE="true" ;;
            --skip-observability) SKIP_OBSERVABILITY="true" ;;
            --with-kgateway) INSTALL_KGATEWAY="true" ;;
            --with-sandbox)  WITH_SANDBOX="true" ;;
            -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
            *) c_warn "Unknown argument: $1" ;;
        esac
        shift
    done
}

# ============================================================================
# 0. Preflight
# ============================================================================
preflight() {
    c_info "Preflight checks"
    need kubectl; need helm; need envsubst
    kubectl cluster-info >/dev/null 2>&1 \
        || c_err "Cannot reach the cluster. Is the kubeconfig pointed at your workload cluster?"
    c_step "Cluster reachable: $(kubectl config current-context)"

    # CNI sanity — a CNI must be installed (nodes Ready) before this runs.
    if ! kubectl get nodes 2>/dev/null | grep -q ' Ready '; then
        c_warn "No Ready nodes — install a CNI (e.g. Cilium) first so nodes go Ready."
    else
        c_step "Nodes Ready: $(kubectl get nodes --no-headers 2>/dev/null | grep -c ' Ready ')"
    fi

    [ -n "${MACSTUDIO_IP:-}" ] || c_err "MACSTUDIO_IP is required (Ollama host serving the local models)."
    OLLAMA_BASE_URL="http://${MACSTUDIO_IP}:${OLLAMA_PORT}/v1"
    c_step "Local model endpoint (macstudio): ${OLLAMA_BASE_URL}"

    # Best-effort reachability probe of the two episode models.
    if command -v curl >/dev/null 2>&1; then
        if curl -fsS --max-time 5 "http://${MACSTUDIO_IP}:${OLLAMA_PORT}/api/tags" >/dev/null 2>&1; then
            c_step "macstudio Ollama reachable from this host."
        else
            c_warn "Could not reach Ollama at ${MACSTUDIO_IP}:${OLLAMA_PORT} from here (pods may still reach it). Ensure OLLAMA_HOST=0.0.0.0 and qwen3.6:latest + qwen3.5:122b are pulled."
        fi
    fi
}

# ============================================================================
# 1. Platform pieces
# ============================================================================
install_platform() {
    if [ "$SKIP_PLATFORM" = "true" ]; then
        c_info "Skipping platform pieces (--skip-platform)"; return
    fi
    c_info "Platform: MetalLB, csi-driver-nfs, cert-manager$([ "$INSTALL_KGATEWAY" = "true" ] && echo ', kgateway')"

    # --- MetalLB ---
    [ -n "${METALLB_IP_RANGE:-}" ] || c_err "METALLB_IP_RANGE is required (e.g. 192.0.2.240-192.0.2.250) unless --skip-platform."
    c_step "Installing MetalLB ${METALLB_VERSION}"
    kubectl apply -f "https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/config/manifests/metallb-native.yaml"
    kubectl -n metallb-system rollout status deploy/controller --timeout=180s
    kubectl -n metallb-system rollout status daemonset/speaker --timeout=180s
    c_step "Applying MetalLB address pool: ${METALLB_IP_RANGE}"
    envsubst < "${EPISODE_DIR}/platform/metallb-pool.yaml.tmpl" | kubectl apply -f -

    # --- csi-driver-nfs + StorageClass ---
    [ -n "${NFS_SERVER:-}" ] && [ -n "${NFS_SHARE:-}" ] \
        || c_err "NFS_SERVER and NFS_SHARE are required (csi-driver-nfs StorageClass) unless --skip-platform."
    c_step "Installing csi-driver-nfs ${CSI_NFS_VERSION}"
    helm repo add csi-driver-nfs https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts >/dev/null 2>&1 || true
    helm repo update csi-driver-nfs >/dev/null
    helm upgrade --install csi-driver-nfs csi-driver-nfs/csi-driver-nfs \
        --namespace kube-system --version "${CSI_NFS_VERSION}" --wait
    c_step "Applying nfs-csi StorageClass (server=${NFS_SERVER} share=${NFS_SHARE})"
    envsubst < "${EPISODE_DIR}/platform/nfs-storageclass.yaml.tmpl" | kubectl apply -f -

    # --- cert-manager (Sympozium webhook TLS prereq) ---
    c_step "Installing cert-manager ${CERT_MANAGER_VERSION}"
    kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml"
    kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s

    # --- kgateway (optional Gateway API implementation) ---
    if [ "$INSTALL_KGATEWAY" = "true" ]; then
        c_step "Installing Gateway API CRDs ${GATEWAY_API_VERSION} + kgateway ${KGATEWAY_VERSION}"
        kubectl apply -f "https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/standard-install.yaml"
        helm upgrade --install kgateway-crds "oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds" \
            --version "${KGATEWAY_VERSION}" --namespace kgateway-system --create-namespace --wait
        helm upgrade --install kgateway "oci://cr.kgateway.dev/kgateway-dev/charts/kgateway" \
            --version "${KGATEWAY_VERSION}" --namespace kgateway-system --wait
    else
        c_step "kgateway install skipped (set INSTALL_KGATEWAY=true / --with-kgateway to enable)"
    fi
}

# ============================================================================
# 1½. Observability — OpenTelemetry Operator + 2 collectors + Dynatrace
# ----------------------------------------------------------------------------
# This episode does NOT use Sympozium's built-in collector. Instead it runs two
# collectors in the `default` namespace (infrastructure/observability/):
#   * oteld  (daemonset)   — node-local logs (filelog) + the collector's own
#                            Prometheus metrics, k8s-enriched, shipped to Dynatrace
#   * otel   (statefulset) — OTLP gateway for app/agent traces+metrics+logs and
#                            cluster-wide Prometheus scraping, shipped to Dynatrace
# Sympozium (values.yaml) is pointed at the `otel` gateway (otel-collector.default).
# It also installs the Dynatrace Operator + DynaKube (infra monitoring) and the
# Dynatrace MCP so the O11y Engineer agent can query Dynatrace.
#
# Requires (for the Dynatrace pieces):
#   export DT_TENANT_URL=https://abc12345.live.dynatrace.com   # OTLP/API base, no trailing /api
#   export DT_API_TOKEN=dt0c01.XXXX...                         # data-ingest + API token
#   export DT_DATA_INGEST_TOKEN=dt0c01.YYYY...                 # optional, DynaKube metric ingest
#   export DT_MCP_TOKEN=dt0s16.ZZZZ...                         # optional, platform token for the MCP
#   export DT_MCP_ENVIRONMENT=https://abc12345.apps.dynatrace.com  # optional, MCP platform URL
# Without DT_TENANT_URL/DT_API_TOKEN the operator+collectors still deploy (with a
# placeholder `dynatrace` secret) but won't export; the DynaKube/MCP are skipped.
# ============================================================================
install_observability() {
    if [ "$SKIP_OBSERVABILITY" = "true" ]; then
        c_info "Skipping observability (--skip-observability)"; return
    fi
    c_info "Observability: OpenTelemetry Operator + 2 collectors (default ns) + Dynatrace"
    local OBS_DIR="${SCRIPT_DIR}/infrastructure/observability"
    local COL_DIR="${OBS_DIR}/opentelemetry-collector"
    local CLUSTER_NAME="${DT_CLUSTER_NAME:-${CLUSTER_NAME:-sympozium-workload}}"

    # --- OpenTelemetry Operator (cert-manager prereq is installed in platform) ---
    if ! kubectl get crd opentelemetrycollectors.opentelemetry.io >/dev/null 2>&1; then
        c_step "Installing the OpenTelemetry Operator (${OTEL_OPERATOR_VERSION})"
        if [ "$OTEL_OPERATOR_VERSION" = "latest" ]; then
            kubectl apply -f "https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml"
        else
            kubectl apply -f "https://github.com/open-telemetry/opentelemetry-operator/releases/download/${OTEL_OPERATOR_VERSION}/opentelemetry-operator.yaml"
        fi
    else
        c_step "OpenTelemetry Operator CRDs already present — re-applying manifest"
        if [ "$OTEL_OPERATOR_VERSION" = "latest" ]; then
            kubectl apply -f "https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml"
        else
            kubectl apply -f "https://github.com/open-telemetry/opentelemetry-operator/releases/download/${OTEL_OPERATOR_VERSION}/opentelemetry-operator.yaml"
        fi
    fi
    kubectl -n opentelemetry-operator-system rollout status deploy/opentelemetry-operator-controller-manager --timeout=180s

    # --- dynatrace OTLP secret (consumed by BOTH collectors via secretKeyRef) ---
    if [ -n "${DT_TENANT_URL:-}" ] && [ -n "${DT_API_TOKEN:-}" ]; then
        c_step "Dynatrace OTLP secret: dynatrace (default ns)"
        kubectl -n default create secret generic dynatrace \
            --from-literal=dynatrace_oltp_url="${DT_TENANT_URL}" \
            --from-literal=dt_api_token="${DT_API_TOKEN}" \
            --dry-run=client -o yaml | kubectl apply -f -
    else
        c_warn "DT_TENANT_URL / DT_API_TOKEN not set — creating a placeholder 'dynatrace' secret so the collectors schedule (no data will be exported)."
        kubectl -n default create secret generic dynatrace \
            --from-literal=dynatrace_oltp_url="https://REPLACE_ME.live.dynatrace.com" \
            --from-literal=dt_api_token="REPLACE_ME" \
            --dry-run=client -o yaml | kubectl apply -f -
    fi

    # --- RBAC + the two collectors (CLUSTER_NAME_TO_REPLACE -> real name) ---
    c_step "Applying otelcontribcol RBAC"
    kubectl apply -f "${COL_DIR}/rbac.yaml"
    c_step "Applying oteld (daemonset) + otel (statefulset) collectors — cluster=${CLUSTER_NAME}"
    sed "s/CLUSTER_NAME_TO_REPLACE/${CLUSTER_NAME}/g" "${COL_DIR}/openTelemetry-manifest_ds.yaml"          | kubectl apply -f -
    sed "s/CLUSTER_NAME_TO_REPLACE/${CLUSTER_NAME}/g" "${COL_DIR}/openTelemetry-manifest_statefulset.yaml" | kubectl apply -f -

    # --- Dynatrace Operator + DynaKube + MCP (only with real creds) ---
    if [ -n "${DT_TENANT_URL:-}" ] && [ -n "${DT_API_TOKEN:-}" ]; then
        c_step "Installing the Dynatrace Operator ${DT_OPERATOR_VERSION} (Helm)"
        helm upgrade --install dynatrace-operator oci://public.ecr.aws/dynatrace/dynatrace-operator \
            --set "csidriver.enabled=false" --version "${DT_OPERATOR_VERSION}" \
            --create-namespace --namespace dynatrace --atomic

        c_step "DynaKube token secret: dynakube (dynatrace ns)"
        kubectl -n dynatrace create secret generic dynakube \
            --from-literal=apiToken="${DT_API_TOKEN}" \
            ${DT_DATA_INGEST_TOKEN:+--from-literal=dataIngestToken="${DT_DATA_INGEST_TOKEN}"} \
            --dry-run=client -o yaml | kubectl apply -f -

        c_step "Applying DynaKube (apiUrl=${DT_TENANT_URL}/api, cluster=${CLUSTER_NAME})"
        sed -e "s#TENANTURL_TOREPLACE#${DT_TENANT_URL}#g" \
            -e "s/CLUSTER_NAME_TO_REPLACE/${CLUSTER_NAME}/g" \
            "${OBS_DIR}/dynatrace/dynakube.yaml" | kubectl apply -f -

        c_step "Dynatrace MCP secret + MCPServer (${SYMPO_NS})"
        kubectl -n "${SYMPO_NS}" create secret generic dynatrace-mcp-secret \
            --from-literal=DT_ENVIRONMENT="${DT_MCP_ENVIRONMENT:-${DT_TENANT_URL}/api}" \
            --from-literal=DT_PLATFORM_TOKEN="${DT_MCP_TOKEN:-${DT_API_TOKEN}}" \
            --dry-run=client -o yaml | kubectl apply -f -
        kubectl apply -f "${SYMPO_DIR}/mcp/dynatrace-mcp.yaml"
    else
        c_warn "DT_TENANT_URL / DT_API_TOKEN not set — skipping Dynatrace Operator, DynaKube and Dynatrace MCP."
        c_warn "Set DT_TENANT_URL + DT_API_TOKEN and re-run to ship telemetry to Dynatrace."
    fi
}

# ============================================================================
# 2. Sympozium control plane (Helm)
# ============================================================================
install_sympozium() {
    c_info "Sympozium control plane (Helm release: ${HELM_RELEASE})"
    # This tutorial repo does not vendor the chart, so install from the published
    # Sympozium Helm repo (CRDs chart + control-plane chart). The episode values
    # (sympozium/values.yaml) pin the upstream control-plane images.
    helm repo add sympozium "${SYMPO_HELM_REPO}" >/dev/null 2>&1 || true
    helm repo update sympozium >/dev/null
    c_step "Installing CRDs chart (sympozium-crds)"
    helm upgrade --install "${HELM_RELEASE}-crds" sympozium/sympozium-crds \
        --namespace "${SYMPO_NS}" --create-namespace --wait
    c_step "Installing control plane chart (sympozium)"
    helm upgrade --install "${HELM_RELEASE}" sympozium/sympozium \
        --namespace "${SYMPO_NS}" --skip-crds --set createNamespace=false \
        -f "${SYMPO_DIR}/values.yaml" --wait --timeout 8m
    kubectl -n "${SYMPO_NS}" rollout status deploy --selector app.kubernetes.io/instance="${HELM_RELEASE}" --timeout=300s 2>/dev/null || true
    c_step "Control plane installed."
}

# ============================================================================
# 3. Custom bmad SkillPack + SympoziumPolicy guardrails
# ============================================================================
install_skills() {
    c_info "Applying custom 'bmad' SkillPack"
    kubectl apply -f "${SYMPO_DIR}/skills/bmad.yaml"
}

install_policy() {
    c_info "Applying SympoziumPolicy guardrails (bmad-guardrails)"
    # Apply before the Ensemble so spec.policyRef: bmad-guardrails resolves.
    kubectl -n "${PACK_NS}" apply -f "${SYMPO_DIR}/ensemble/sympoziumpolicy-bmad.yaml"
}

# ============================================================================
# 4. Secrets (Ollama placeholder + Slack)
# ============================================================================
install_secrets() {
    c_info "Provisioning secrets in namespace '${PACK_NS}'"

    # Ollama needs no key, but authRefs is mandatory -> placeholder secret.
    c_step "Ollama placeholder secret: bmad-ollama-key"
    kubectl -n "${PACK_NS}" create secret generic bmad-ollama-key \
        --from-literal=OPENAI_API_KEY=not-needed \
        --dry-run=client -o yaml | kubectl apply -f -

    # Slack tokens (Socket Mode). Both tokens recommended; app token enables
    # Socket Mode (no public webhook needed).
    if [ -n "${SLACK_BOT_TOKEN:-}" ]; then
        c_step "Slack secret: bmad-slack-tokens"
        kubectl -n "${PACK_NS}" create secret generic bmad-slack-tokens \
            --from-literal=SLACK_BOT_TOKEN="${SLACK_BOT_TOKEN}" \
            --from-literal=SLACK_APP_TOKEN="${SLACK_APP_TOKEN:-}" \
            --dry-run=client -o yaml | kubectl apply -f -
        [ -n "${SLACK_APP_TOKEN:-}" ] || c_warn "SLACK_APP_TOKEN empty — Slack will fall back to Events API mode (needs a public webhook)."
    else
        c_warn "SLACK_BOT_TOKEN not set — creating a placeholder Slack secret so the pack applies."
        c_warn "Set SLACK_BOT_TOKEN + SLACK_APP_TOKEN and re-run to enable Slack (see deploy-notes.md)."
        kubectl -n "${PACK_NS}" create secret generic bmad-slack-tokens \
            --from-literal=SLACK_BOT_TOKEN=REPLACE_ME \
            --from-literal=SLACK_APP_TOKEN=REPLACE_ME \
            --dry-run=client -o yaml | kubectl apply -f -
    fi
}

# ============================================================================
# 5. bmad-ensemble Ensemble (local Ollama models via spec.baseURL)
# ============================================================================
install_ensemble() {
    if [ "$SKIP_ENSEMBLE" = "true" ]; then
        c_info "Skipping ensemble (--skip-ensemble)"; return
    fi
    c_info "Activating the bmad-ensemble Ensemble"
    # The Ensemble manifest ships with a <MACSTUDIO_IP> placeholder; substitute the
    # real macstudio endpoint so spec.baseURL points every generated Agent at the
    # local Ollama host. No post-apply patch is needed — the controller propagates
    # spec.baseURL to each Agent it stamps out.
    sed "s|http://<MACSTUDIO_IP>:11434/v1|${OLLAMA_BASE_URL}|" \
        "${SYMPO_DIR}/ensemble/ensemble-bmad-ensemble.yaml" | kubectl apply -f -

    # Optional: turn on the Kubernetes Agent Sandbox (gVisor/kata) for every agent.
    # The controller propagates spec.agentSandbox to each generated Agent.
    if [ "$WITH_SANDBOX" = "true" ]; then
        c_step "Enabling Agent Sandbox (runtimeClass=${SANDBOX_RUNTIME_CLASS}) on the Ensemble"
        if ! kubectl get runtimeclass "${SANDBOX_RUNTIME_CLASS}" >/dev/null 2>&1; then
            c_warn "RuntimeClass '${SANDBOX_RUNTIME_CLASS}' not found — sandboxed runs will not schedule until it exists (GKE Sandbox, or install gVisor on the nodes)."
        fi
        kubectl -n "${PACK_NS}" patch ensemble bmad-ensemble --type=merge \
            -p "{\"spec\":{\"agentSandbox\":{\"enabled\":true,\"runtimeClass\":\"${SANDBOX_RUNTIME_CLASS}\"}}}"
    fi

    # Wait for the controller to stamp out one Agent per persona (bmad-ensemble-<persona>).
    c_step "Waiting for the controller to stamp out agents..."
    for persona in ${CREW}; do
        agent="bmad-ensemble-${persona}"
        for i in $(seq 1 30); do
            if kubectl -n "${PACK_NS}" get agent "${agent}" >/dev/null 2>&1; then break; fi
            sleep 2
        done
        if kubectl -n "${PACK_NS}" get agent "${agent}" >/dev/null 2>&1; then
            c_step "Agent ${agent} created (baseURL -> ${OLLAMA_BASE_URL})"
        else
            c_warn "Agent ${agent} not created yet — re-run after the Ensemble reconciles, or check 'kubectl get ensemble bmad-ensemble'."
        fi
    done
}

# ============================================================================
# 6. Verification
# ============================================================================
verify() {
    c_info "Verification"
    echo "--- Ensemble ---";  kubectl -n "${PACK_NS}" get ensemble bmad-ensemble 2>/dev/null || true
    echo "--- Policy ---";    kubectl -n "${PACK_NS}" get sympoziumpolicy bmad-guardrails 2>/dev/null || true
    echo "--- Agents ---";    kubectl -n "${PACK_NS}" get agent 2>/dev/null | grep -E 'NAME|bmad-ensemble' || true
    echo "--- Schedules ---"; kubectl -n "${PACK_NS}" get sympoziumschedule 2>/dev/null | grep bmad-ensemble || true
    if [ "$WITH_SANDBOX" = "true" ]; then
        echo "--- Agent Sandbox (per agent) ---"
        for persona in ${CREW}; do
            kubectl -n "${PACK_NS}" get agent "bmad-ensemble-${persona}" \
                -o jsonpath="{.metadata.name}{': enabled='}{.spec.agents.default.agentSandbox.enabled}{' runtimeClass='}{.spec.agents.default.agentSandbox.runtimeClass}{'\n'}" 2>/dev/null || true
        done
    fi
    echo "--- Model mapping (baseURL + model per agent) ---"
    for persona in ${CREW}; do
        kubectl -n "${PACK_NS}" get agent "bmad-ensemble-${persona}" \
            -o jsonpath="{.metadata.name}{'\t'}{.spec.agents.default.model}{'\t'}{.spec.agents.default.baseURL}{'\n'}" 2>/dev/null || true
    done
    echo "--- Memory ConfigMaps (showcase) ---"
    kubectl -n "${PACK_NS}" get configmap 2>/dev/null | grep -E 'bmad-ensemble.*-memory' || c_warn "Memory ConfigMaps appear after the first run of each agent."
    echo "--- Channel status (Slack) ---"
    for persona in ${CREW}; do
        kubectl -n "${PACK_NS}" get agent "bmad-ensemble-${persona}" \
            -o jsonpath="{.metadata.name}{': '}{.status.channels}{'\n'}" 2>/dev/null || true
    done
    echo
    c_info "Done. Inspect an agent's memory with:"
    echo "    kubectl -n ${PACK_NS} get configmap bmad-ensemble-code-reviewer-memory -o jsonpath='{.data.MEMORY\\.md}'"
    c_info "Tail the Slack channel deployment with:"
    echo "    kubectl -n ${PACK_NS} logs -l sympozium.ai/channel=slack -f"
    c_info "Observability — Sympozium emits OTel metrics you can graph per agent:"
    echo "    sympozium.agent.runs / sympozium.agent.duration_ms / sympozium.agent.run.completed"
    echo "    sympozium.tool.invocations / sympozium.access.denied (policy blocks) / sympozium.pod.create"
    echo "    gen_ai.usage.input_tokens / gen_ai.usage.output_tokens (LLM token cost)"
}

# ============================================================================
main() {
    parse_args "$@"
    preflight
    install_platform
    install_observability
    install_sympozium
    install_skills
    install_policy
    install_secrets
    install_ensemble
    verify
}
main "$@"