diff --git a/.github/actions/helm-rollback-on-cancel/action.yml b/.github/actions/helm-rollback-on-cancel/action.yml new file mode 100644 index 0000000..28948e9 --- /dev/null +++ b/.github/actions/helm-rollback-on-cancel/action.yml @@ -0,0 +1,41 @@ +name: Entur/Helm/Rollback-On-Cancel +description: Rolls back the Helm release if the job is canceled during deployment and a pending rollout exists +inputs: + release_name: + description: "The release name" + required: true + namespace: + description: "The Kubernetes namespace" + required: true + timeout_minutes: + description: "Job timeout in minutes (same value as helm-deploy)." + required: true +runs: + using: "composite" + steps: + - shell: bash + env: + TIMEOUT: ${{ inputs.timeout_minutes }} + RELEASE_NAME: ${{ inputs.release_name }} + NAMESPACE: ${{ inputs.namespace }} + run: | + TIMEOUT=$((TIMEOUT/2-2)) + if (( TIMEOUT < 1 )); then TIMEOUT=1; fi + if (( TIMEOUT > 4 )); then TIMEOUT=4; fi + if helm list --pending -n "${NAMESPACE}" --filter "^${RELEASE_NAME}$" --short | grep -q .; then + echo "#################################################################################################" + echo " Job cancelled. Rolling back Helm release '${RELEASE_NAME}'..." + echo "#################################################################################################" + if helm rollback "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait --timeout "${TIMEOUT}m0s" --debug; then + echo "#################################################################################################" + echo " Helm rollback on cancellation successful." + echo "#################################################################################################" + else + echo "#################################################################################################" + echo " Helm rollback on cancellation failed. Please investigate manually." + echo "#################################################################################################" + exit 1 + fi + else + echo "No active rollout found for '${RELEASE_NAME}' in '${NAMESPACE}'. Nothing to roll back." + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c2b6b1..cfad493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,10 +135,63 @@ jobs: slack_channel_id: GFBL95A0J # sandbox-channel secrets: inherit - test-deploy-gcp-with-failed-rollback: + test-deploy-gcp-cancel-rollback: needs: [test-deploy-ok-gcp] runs-on: ubuntu-24.04 environment: dev + timeout-minutes: 10 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v6 + - id: install + uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 + with: + version: "v3.20.1" + - uses: entur/gha-meta/.github/actions/cloud-auth@v1 + with: + environment: dev + gcp_workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }} + gcp_service_account: ${{ vars.SERVICE_ACCOUNT }} + - uses: entur/gha-meta/.github/actions/k8s-auth@v1 + with: + environment: dev + - name: Put release in pending-upgrade state + run: | + helm upgrade amazing-app ./fixture/helm/amazing-app \ + -f fixture/helm/amazing-app/env/values-kub-ent-dev.yaml \ + --namespace gha-ci --install --wait --timeout 10m \ + --set "common.container.image=eu.gcr.io/entur-system-1287/nonexistent:dummy" & + HELM_PID=$! + for i in $(seq 1 15); do + STATUS=$(helm status amazing-app -n gha-ci -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || echo "") + [ "$STATUS" = "pending-upgrade" ] && break + sleep 1 + done + kill -9 $HELM_PID 2>/dev/null || true + wait $HELM_PID 2>/dev/null || true + STATUS=$(helm status amazing-app -n gha-ci -o json | jq -r '.info.status') + if [ "$STATUS" != "pending-upgrade" ]; then + echo "::error::Setup failed: expected pending-upgrade, got '$STATUS'" + exit 1 + fi + - id: rollback-on-cancel + uses: ./.github/actions/helm-rollback-on-cancel + with: + release_name: amazing-app + namespace: gha-ci + timeout_minutes: 10 + - name: Check rollback on cancel succeeded + if: steps.rollback-on-cancel.outcome != 'success' + uses: actions/github-script@v9 + with: + script: core.setFailed('Rollback on cancel should succeed.') + + test-deploy-gcp-with-failed-rollback: + needs: [test-deploy-gcp-cancel-rollback] + runs-on: ubuntu-24.04 + environment: dev timeout-minutes: 8 permissions: contents: read diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bd7bcbb..e8ac778 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -260,6 +260,16 @@ jobs: image_set_path: ${{ env.GHA_HELM_DEPLOY_IMAGE_SET_PATH }} container_name: ${{ env.GHA_HELM_DEPLOY_CONTAINER_NAME }} registry: ${{ env.GHA_HELM_DEPLOY_REGISTRY }} + # Helm rollback on workflow cancellation, only trigger if user cancels during deployment and the deployment + # has not succeeded, to avoid rolling back a successful deployment in case of cancel after rollout completion. + - id: helm-rollback-on-cancel + name: Helm rollback on cancellation + if: cancelled() && steps.helm-deploy.outputs.HELM_DEPLOY_STATUS != '0' + uses: ./.github/actions/helm-rollback-on-cancel + with: + release_name: ${{ env.GHA_HELM_DEPLOY_RELEASE_NAME }} + namespace: ${{ env.GHA_HELM_DEPLOY_NAMESPACE }} + timeout_minutes: ${{ inputs.timeout_minutes }} - uses: entur/gha-meta/.github/actions/posthog@v1 id: send-analytics name: Send analytics to PostHog