Fix(ci): fix delete-env failures and add force-delete fallback (#3819)

bosh delete-deployment used an invalid --skip-drain flag, causing it to
silently fail every run and leaving bbl down to hit "umount: target is
busy" during its own deployment teardown. Fix the flag and add a
force-delete-env.sh fallback that tears down the director/jumpbox directly
via the GCP CPI when the graceful path can't recover (e.g. abandoned BOSH
tasks or a stuck task worker).

Signed-off-by: Prem Kumar Kalle <prem.kalle@broadcom.com>

Author: prkalle

.github/bosh-lite-files/force-delete-env.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Force-tears-down a bosh-lite environment directly via the GCP CPI and
+# gcloud, bypassing bbl/bosh's graceful deployment-deletion path entirely.
+#
+# Use this when `bbl down` hangs or fails - e.g. warden bind-mount
+# "target is busy" errors during deployment deletion, or SSH-tunnel
+# connectivity issues to an unhealthy director/jumpbox. `bosh delete-env`
+# only needs the local state file + the CPI (GCP API) to destroy a VM, so it
+# works even when the director/jumpbox is unresponsive or already gone.
+#
+# Usage: force-delete-env.sh <env-name>
+# Required env vars: BBL_GCP_SERVICE_ACCOUNT_KEY_PATH, GOOGLE_CLOUD_PROJECT,
+#                     BBL_GCP_REGION
+# Expects <env-name>/bbl-state to already be downloaded in the cwd.
+
+set -uo pipefail
+
+env_name="$1"
+state_dir="${env_name}/bbl-state"
+zone="$(jq -r '.gcp.zone' "${state_dir}/bbl-state.json")"
+
+echo "Force-deleting director VM for ${env_name}"
+bosh delete-env \
+  "${state_dir}/bosh-deployment/bosh.yml" \
+  --state "${state_dir}/vars/bosh-state.json" \
+  --vars-store "${state_dir}/vars/director-vars-store.yml" \
+  --vars-file "${state_dir}/vars/director-vars-file.yml" \
+  -o "${state_dir}/bosh-deployment/gcp/cpi.yml" \
+  -o "${state_dir}/bosh-deployment/jumpbox-user.yml" \
+  -o "${state_dir}/bosh-deployment/uaa.yml" \
+  -o "${state_dir}/bosh-deployment/credhub.yml" \
+  -o "${state_dir}/bbl-ops-files/gcp/bosh-director-ephemeral-ip-ops.yml" \
+  --var-file gcp_credentials_json="${BBL_GCP_SERVICE_ACCOUNT_KEY_PATH}" \
+  -v project_id="${GOOGLE_CLOUD_PROJECT}" \
+  -v zone="${zone}" \
+  --non-interactive || true
+
+echo "Force-deleting jumpbox VM for ${env_name}"
+bosh delete-env \
+  "${state_dir}/jumpbox-deployment/jumpbox.yml" \
+  --state "${state_dir}/vars/jumpbox-state.json" \
+  --vars-store "${state_dir}/vars/jumpbox-vars-store.yml" \
+  --vars-file "${state_dir}/vars/jumpbox-vars-file.yml" \
+  -o "${state_dir}/jumpbox-deployment/gcp/cpi.yml" \
+  --var-file gcp_credentials_json="${BBL_GCP_SERVICE_ACCOUNT_KEY_PATH}" \
+  -v project_id="${GOOGLE_CLOUD_PROJECT}" \
+  -v zone="${zone}" \
+  --non-interactive || true
+
+tfstate="${state_dir}/vars/terraform.tfstate"
+if [ -f "$tfstate" ]; then
+  echo "Deleting remaining terraform-managed network resources for ${env_name}"
+
+  jq -r '.resources[] | select(.type=="google_compute_firewall") | .instances[0].attributes.name' "$tfstate" | \
+    while read -r name; do gcloud compute firewall-rules delete "$name" --quiet || true; done
+
+  jq -r '.resources[] | select(.type=="google_dns_record_set") | "\(.instances[0].attributes.name)\t\(.instances[0].attributes.managed_zone)\t\(.instances[0].attributes.type)"' "$tfstate" | \
+    while IFS=$'\t' read -r name dns_zone rtype; do gcloud dns record-sets delete "$name" --zone "$dns_zone" --type "$rtype" --quiet || true; done
+
+  jq -r '.resources[] | select(.type=="google_compute_route") | .instances[0].attributes.name' "$tfstate" | \
+    while read -r name; do gcloud compute routes delete "$name" --quiet || true; done
+
+  jq -r '.resources[] | select(.type=="google_compute_router") | .instances[0].attributes.name' "$tfstate" | \
+    while read -r name; do gcloud compute routers delete "$name" --region "${BBL_GCP_REGION}" --quiet || true; done
+
+  jq -r '.resources[] | select(.type=="google_compute_address") | .instances[0].attributes.name' "$tfstate" | \
+    while read -r name; do gcloud compute addresses delete "$name" --region "${BBL_GCP_REGION}" --quiet || true; done
+
+  jq -r '.resources[] | select(.type=="google_compute_subnetwork") | .instances[0].attributes.name' "$tfstate" | \
+    while read -r name; do gcloud compute networks subnets delete "$name" --region "${BBL_GCP_REGION}" --quiet || true; done
+
+  jq -r '.resources[] | select(.type=="google_compute_network") | .instances[0].attributes.name' "$tfstate" | \
+    while read -r name; do gcloud compute networks delete "$name" --quiet || true; done
+fi
+
+echo "Force-delete complete for ${env_name}"

.github/workflows/create-bosh-lite.yml

@@ -132,6 +132,7 @@ jobs:
           
           bosh upload-release "https://bosh.io/d/github.com/cloudfoundry/capi-release?v=$capi_release_version"          
       - name: Deploy cf
+        timeout-minutes: 90
         run: |
           env_name="${{ steps.setup-bbl-env.outputs.envName }}"
           cd $env_name/bbl-state

.github/workflows/delete-bosh-lite.yml

@@ -66,21 +66,41 @@ jobs:
 
       - name: Download file from GCS
         run: |
-          gsutil -m cp -P -R gs://cf-cli-bosh-lites/${ENV_NAME} . 
+          gcloud storage cp -r gs://cf-cli-bosh-lites/${ENV_NAME} .
 
       - name: delete bosh
+        id: delete-bosh
+        timeout-minutes: 30
+        continue-on-error: true
         run: |
           cd ${ENV_NAME}/bbl-state
           eval "$(bbl print-env --shell-type posix)"
 
           # Delete the CF deployment first so warden containers release their
-          # bind mounts on the director's persistent disk. 
+          # bind mounts on the director's persistent disk. --force makes bosh ignore
+          # per-instance errors (like a busy unmount) and still delete the
+          # deployment record.
           echo "Deleting CF deployment ${BOSH_DEPLOYMENT}"
-          bosh delete-deployment -d ${BOSH_DEPLOYMENT} --force --skip-drain || true
+          bosh delete-deployment -d ${BOSH_DEPLOYMENT} --force || true
 
           echo "Deleting env ${ENV_NAME}"
           bbl down --no-confirm
-        
+
+      - name: force delete env (fallback)
+        if: steps.delete-bosh.outcome == 'failure'
+        timeout-minutes: 15
+        run: |
+          echo "${BBL_GCP_SERVICE_ACCOUNT_KEY}" > /tmp/gcp_key.json
+          export BBL_GCP_SERVICE_ACCOUNT_KEY_PATH=/tmp/gcp_key.json
+          bash ${GITHUB_WORKSPACE}/cli/.github/bosh-lite-files/force-delete-env.sh "${ENV_NAME}"
+          rm -f /tmp/gcp_key.json
+
       - name: delete gcs bucket
+        if: always()
+        run: |
+          gcloud storage rm -r gs://cf-cli-bosh-lites/${ENV_NAME}
+
+      - name: warn if delete bosh needed the fallback
+        if: steps.delete-bosh.outcome == 'failure'
         run: |
-          gsutil rm -R gs://cf-cli-bosh-lites/${ENV_NAME} 
+          echo "::warning::The graceful 'delete bosh' step failed for ${ENV_NAME} and the environment was torn down via the force-delete-env.sh fallback instead. This usually means an abandoned BOSH task or a warden mount was still busy - see the 'delete bosh' step log for details."