From dc49bbf711d7d77917f55ad6ea03e26e16df8bec Mon Sep 17 00:00:00 2001 From: Anders Isaksen Date: Mon, 8 Jun 2026 13:04:00 +0200 Subject: [PATCH 1/4] feat: Add step for rollback on cancel --- .github/actions/helm-deploy/action.yml | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/actions/helm-deploy/action.yml b/.github/actions/helm-deploy/action.yml index 5ea33d0..a6230ad 100644 --- a/.github/actions/helm-deploy/action.yml +++ b/.github/actions/helm-deploy/action.yml @@ -162,3 +162,32 @@ runs: exit 1 fi fi + # Rolls back the Helm release if the job is cancelled during deployment and a previous release exists + - id: helm-rollback-on-cancel + name: Helm rollback on cancellation + if: cancelled() && steps.helm-deploy.outputs.HELM_DEPLOY_STATUS != '0' + env: + TIMEOUT: ${{ inputs.timeout_minutes }} + GHA_HELM_DEPLOY_RELEASE_NAME: ${{ inputs.release_name }} + GHA_HELM_DEPLOY_NAMESPACE: ${{ inputs.namespace }} + shell: bash + run: | + TIMEOUT=$((TIMEOUT/2-2)) + if (( TIMEOUT < 1 )); then TIMEOUT=1; fi + if (( TIMEOUT > 4 )); then TIMEOUT=4; fi + if helm list --all -n "${GHA_HELM_DEPLOY_NAMESPACE}" --filter "^${GHA_HELM_DEPLOY_RELEASE_NAME}$" --short | grep -q .; then + echo "#################################################################################################" + echo " Job cancelled. Rolling back Helm release '${GHA_HELM_DEPLOY_RELEASE_NAME}'..." + echo "#################################################################################################" + if helm rollback "${GHA_HELM_DEPLOY_RELEASE_NAME}" --namespace "${GHA_HELM_DEPLOY_NAMESPACE}" --wait --timeout "${TIMEOUT}m0s" --debug; then + echo "#################################################################################################" + echo " Helm rollback on cancellation successful." + echo "#################################################################################################" + else + echo "#################################################################################################" + echo " Helm rollback on cancellation failed. Please investigate manually." + echo "#################################################################################################" + fi + else + echo "No existing Helm release found for '${GHA_HELM_DEPLOY_RELEASE_NAME}' in '${GHA_HELM_DEPLOY_NAMESPACE}'. Nothing to roll back." + fi From 92eb26c9721850258ffdadc3ec493fd184ece329 Mon Sep 17 00:00:00 2001 From: Anders Isaksen Date: Mon, 8 Jun 2026 14:27:01 +0200 Subject: [PATCH 2/4] Extract to composite action for testability --- .github/actions/helm-deploy/action.yml | 31 ++--------- .../helm-rollback-on-cancel/action.yml | 41 ++++++++++++++ .github/workflows/ci.yml | 55 ++++++++++++++++++- 3 files changed, 100 insertions(+), 27 deletions(-) create mode 100644 .github/actions/helm-rollback-on-cancel/action.yml diff --git a/.github/actions/helm-deploy/action.yml b/.github/actions/helm-deploy/action.yml index a6230ad..f51d5dc 100644 --- a/.github/actions/helm-deploy/action.yml +++ b/.github/actions/helm-deploy/action.yml @@ -162,32 +162,11 @@ runs: exit 1 fi fi - # Rolls back the Helm release if the job is cancelled during deployment and a previous release exists - id: helm-rollback-on-cancel name: Helm rollback on cancellation if: cancelled() && steps.helm-deploy.outputs.HELM_DEPLOY_STATUS != '0' - env: - TIMEOUT: ${{ inputs.timeout_minutes }} - GHA_HELM_DEPLOY_RELEASE_NAME: ${{ inputs.release_name }} - GHA_HELM_DEPLOY_NAMESPACE: ${{ inputs.namespace }} - shell: bash - run: | - TIMEOUT=$((TIMEOUT/2-2)) - if (( TIMEOUT < 1 )); then TIMEOUT=1; fi - if (( TIMEOUT > 4 )); then TIMEOUT=4; fi - if helm list --all -n "${GHA_HELM_DEPLOY_NAMESPACE}" --filter "^${GHA_HELM_DEPLOY_RELEASE_NAME}$" --short | grep -q .; then - echo "#################################################################################################" - echo " Job cancelled. Rolling back Helm release '${GHA_HELM_DEPLOY_RELEASE_NAME}'..." - echo "#################################################################################################" - if helm rollback "${GHA_HELM_DEPLOY_RELEASE_NAME}" --namespace "${GHA_HELM_DEPLOY_NAMESPACE}" --wait --timeout "${TIMEOUT}m0s" --debug; then - echo "#################################################################################################" - echo " Helm rollback on cancellation successful." - echo "#################################################################################################" - else - echo "#################################################################################################" - echo " Helm rollback on cancellation failed. Please investigate manually." - echo "#################################################################################################" - fi - else - echo "No existing Helm release found for '${GHA_HELM_DEPLOY_RELEASE_NAME}' in '${GHA_HELM_DEPLOY_NAMESPACE}'. Nothing to roll back." - fi + uses: ./.github/actions/helm-rollback-on-cancel + with: + release_name: ${{ inputs.release_name }} + namespace: ${{ inputs.namespace }} + timeout_minutes: ${{ inputs.timeout_minutes }} diff --git a/.github/actions/helm-rollback-on-cancel/action.yml b/.github/actions/helm-rollback-on-cancel/action.yml new file mode 100644 index 0000000..28948e9 --- /dev/null +++ b/.github/actions/helm-rollback-on-cancel/action.yml @@ -0,0 +1,41 @@ +name: Entur/Helm/Rollback-On-Cancel +description: Rolls back the Helm release if the job is canceled during deployment and a pending rollout exists +inputs: + release_name: + description: "The release name" + required: true + namespace: + description: "The Kubernetes namespace" + required: true + timeout_minutes: + description: "Job timeout in minutes (same value as helm-deploy)." + required: true +runs: + using: "composite" + steps: + - shell: bash + env: + TIMEOUT: ${{ inputs.timeout_minutes }} + RELEASE_NAME: ${{ inputs.release_name }} + NAMESPACE: ${{ inputs.namespace }} + run: | + TIMEOUT=$((TIMEOUT/2-2)) + if (( TIMEOUT < 1 )); then TIMEOUT=1; fi + if (( TIMEOUT > 4 )); then TIMEOUT=4; fi + if helm list --pending -n "${NAMESPACE}" --filter "^${RELEASE_NAME}$" --short | grep -q .; then + echo "#################################################################################################" + echo " Job cancelled. Rolling back Helm release '${RELEASE_NAME}'..." + echo "#################################################################################################" + if helm rollback "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait --timeout "${TIMEOUT}m0s" --debug; then + echo "#################################################################################################" + echo " Helm rollback on cancellation successful." + echo "#################################################################################################" + else + echo "#################################################################################################" + echo " Helm rollback on cancellation failed. Please investigate manually." + echo "#################################################################################################" + exit 1 + fi + else + echo "No active rollout found for '${RELEASE_NAME}' in '${NAMESPACE}'. Nothing to roll back." + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c2b6b1..cfad493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,10 +135,63 @@ jobs: slack_channel_id: GFBL95A0J # sandbox-channel secrets: inherit - test-deploy-gcp-with-failed-rollback: + test-deploy-gcp-cancel-rollback: needs: [test-deploy-ok-gcp] runs-on: ubuntu-24.04 environment: dev + timeout-minutes: 10 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v6 + - id: install + uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 + with: + version: "v3.20.1" + - uses: entur/gha-meta/.github/actions/cloud-auth@v1 + with: + environment: dev + gcp_workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }} + gcp_service_account: ${{ vars.SERVICE_ACCOUNT }} + - uses: entur/gha-meta/.github/actions/k8s-auth@v1 + with: + environment: dev + - name: Put release in pending-upgrade state + run: | + helm upgrade amazing-app ./fixture/helm/amazing-app \ + -f fixture/helm/amazing-app/env/values-kub-ent-dev.yaml \ + --namespace gha-ci --install --wait --timeout 10m \ + --set "common.container.image=eu.gcr.io/entur-system-1287/nonexistent:dummy" & + HELM_PID=$! + for i in $(seq 1 15); do + STATUS=$(helm status amazing-app -n gha-ci -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || echo "") + [ "$STATUS" = "pending-upgrade" ] && break + sleep 1 + done + kill -9 $HELM_PID 2>/dev/null || true + wait $HELM_PID 2>/dev/null || true + STATUS=$(helm status amazing-app -n gha-ci -o json | jq -r '.info.status') + if [ "$STATUS" != "pending-upgrade" ]; then + echo "::error::Setup failed: expected pending-upgrade, got '$STATUS'" + exit 1 + fi + - id: rollback-on-cancel + uses: ./.github/actions/helm-rollback-on-cancel + with: + release_name: amazing-app + namespace: gha-ci + timeout_minutes: 10 + - name: Check rollback on cancel succeeded + if: steps.rollback-on-cancel.outcome != 'success' + uses: actions/github-script@v9 + with: + script: core.setFailed('Rollback on cancel should succeed.') + + test-deploy-gcp-with-failed-rollback: + needs: [test-deploy-gcp-cancel-rollback] + runs-on: ubuntu-24.04 + environment: dev timeout-minutes: 8 permissions: contents: read From edeb48e5bd25fd3e6035894d243d48f6cb96c814 Mon Sep 17 00:00:00 2001 From: Anders Isaksen Date: Mon, 8 Jun 2026 15:00:40 +0200 Subject: [PATCH 3/4] Move to correct place --- .github/actions/helm-deploy/action.yml | 8 -------- .github/workflows/deploy.yml | 10 ++++++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/actions/helm-deploy/action.yml b/.github/actions/helm-deploy/action.yml index f51d5dc..5ea33d0 100644 --- a/.github/actions/helm-deploy/action.yml +++ b/.github/actions/helm-deploy/action.yml @@ -162,11 +162,3 @@ runs: exit 1 fi fi - - id: helm-rollback-on-cancel - name: Helm rollback on cancellation - if: cancelled() && steps.helm-deploy.outputs.HELM_DEPLOY_STATUS != '0' - uses: ./.github/actions/helm-rollback-on-cancel - with: - release_name: ${{ inputs.release_name }} - namespace: ${{ inputs.namespace }} - timeout_minutes: ${{ inputs.timeout_minutes }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bd7bcbb..3d9a0de 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -260,6 +260,16 @@ jobs: image_set_path: ${{ env.GHA_HELM_DEPLOY_IMAGE_SET_PATH }} container_name: ${{ env.GHA_HELM_DEPLOY_CONTAINER_NAME }} registry: ${{ env.GHA_HELM_DEPLOY_REGISTRY }} + # Helm rollback on workflow cancellation, only trigger if user cancels during deployment and the deployment + # has not succeeded, to avoid rolling back a successful deployment in case of cancel after rollout completion. + - id: helm-rollback-on-cancel + name: Helm rollback on cancellation + if: cancelled() && steps.helm-deploy.outputs.HELM_DEPLOY_STATUS != '0' + uses: entur/gha-helm/.github/actions/helm-rollback-on-cancel@v1 + with: + release_name: ${{ env.GHA_HELM_DEPLOY_RELEASE_NAME }} + namespace: ${{ env.GHA_HELM_DEPLOY_NAMESPACE }} + timeout_minutes: ${{ inputs.timeout_minutes }} - uses: entur/gha-meta/.github/actions/posthog@v1 id: send-analytics name: Send analytics to PostHog From 330719abb0f10af2949742f989bc65b68a5d4f9f Mon Sep 17 00:00:00 2001 From: Anders Isaksen Date: Mon, 8 Jun 2026 15:11:09 +0200 Subject: [PATCH 4/4] Use relative path --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3d9a0de..e8ac778 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -265,7 +265,7 @@ jobs: - id: helm-rollback-on-cancel name: Helm rollback on cancellation if: cancelled() && steps.helm-deploy.outputs.HELM_DEPLOY_STATUS != '0' - uses: entur/gha-helm/.github/actions/helm-rollback-on-cancel@v1 + uses: ./.github/actions/helm-rollback-on-cancel with: release_name: ${{ env.GHA_HELM_DEPLOY_RELEASE_NAME }} namespace: ${{ env.GHA_HELM_DEPLOY_NAMESPACE }}