From 059657957b9a57b2b263abd04ba94b8bad826fe0 Mon Sep 17 00:00:00 2001 From: Anders Isaksen Date: Wed, 10 Jun 2026 10:47:53 +0200 Subject: [PATCH 1/2] feat: Rollback on workflow cancel during rollout ETU-73230 --- .../helm-rollback-on-cancel/action.yml | 41 ++++++++++++++ .github/workflows/ci.yml | 55 ++++++++++++++++++- .github/workflows/deploy.yml | 9 +++ AGENTS.md | 5 +- 4 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 .github/actions/helm-rollback-on-cancel/action.yml diff --git a/.github/actions/helm-rollback-on-cancel/action.yml b/.github/actions/helm-rollback-on-cancel/action.yml new file mode 100644 index 0000000..dd123d6 --- /dev/null +++ b/.github/actions/helm-rollback-on-cancel/action.yml @@ -0,0 +1,41 @@ +name: Entur/Helm/Rollback-On-Cancel +description: Rolls back the Helm release if the job is canceled during deployment and a pending rollout exists +inputs: + release_name: + description: "The release name" + required: true + namespace: + description: "The Kubernetes namespace" + required: true + timeout_minutes: + description: "Job timeout in minutes (default same value as helm-deploy)." + required: true +runs: + using: "composite" + steps: + - shell: bash + env: + TIMEOUT: ${{ inputs.timeout_minutes }} + RELEASE_NAME: ${{ inputs.release_name }} + NAMESPACE: ${{ inputs.namespace }} + run: | + TIMEOUT=$((TIMEOUT/2-2)) + if (( TIMEOUT < 1 )); then TIMEOUT=1; fi + if (( TIMEOUT > 4 )); then TIMEOUT=4; fi + if helm list --pending -n "${NAMESPACE}" --filter "^${RELEASE_NAME}$" --short | grep -q .; then + echo "#################################################################################################" + echo " Job cancelled. Rolling back Helm release '${RELEASE_NAME}'..." + echo "#################################################################################################" + if helm rollback "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait --timeout "${TIMEOUT}m0s" --debug; then + echo "#################################################################################################" + echo " Helm rollback on cancellation successful." + echo "#################################################################################################" + else + echo "#################################################################################################" + echo " Helm rollback on cancellation failed. Please investigate manually." + echo "#################################################################################################" + exit 1 + fi + else + echo "No active rollout found for '${RELEASE_NAME}' in '${NAMESPACE}'. Nothing to roll back." + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf3372e..511bf20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,8 +135,61 @@ jobs: slack_channel_id: GFBL95A0J # sandbox-channel secrets: inherit + test-deploy-gcp-cancel-rollback: + needs: [ test-deploy-ok-gcp ] + runs-on: ubuntu-24.04 + environment: dev + timeout-minutes: 10 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v6 + - id: install + uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 + with: + version: "v3.20.1" + - uses: entur/gha-meta/.github/actions/cloud-auth@v1 + with: + environment: dev + gcp_workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }} + gcp_service_account: ${{ vars.SERVICE_ACCOUNT }} + - uses: entur/gha-meta/.github/actions/k8s-auth@v1 + with: + environment: dev + - name: Put release in pending-upgrade state + run: | + helm upgrade amazing-app ./fixture/helm/amazing-app \ + -f fixture/helm/amazing-app/env/values-kub-ent-dev.yaml \ + --namespace gha-ci --install --wait --timeout 10m \ + --set "common.container.image=eu.gcr.io/entur-system-1287/nonexistent:dummy" & + HELM_PID=$! + for i in $(seq 1 15); do + STATUS=$(helm status amazing-app -n gha-ci -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || echo "") + [ "$STATUS" = "pending-upgrade" ] && break + sleep 1 + done + kill -9 $HELM_PID 2>/dev/null || true + wait $HELM_PID 2>/dev/null || true + STATUS=$(helm status amazing-app -n gha-ci -o json | jq -r '.info.status') + if [ "$STATUS" != "pending-upgrade" ]; then + echo "::error::Setup failed: expected pending-upgrade, got '$STATUS'" + exit 1 + fi + - id: rollback-on-cancel + uses: entur/gha-helm/.github/actions/helm-rollback-on-cancel@feat/rollback-on-workflow-cancel # TODO Change to v1 + with: + release_name: amazing-app + namespace: gha-ci + timeout_minutes: 10 + - name: Check rollback on cancel succeeded + if: steps.rollback-on-cancel.outcome != 'success' + uses: actions/github-script@v9 + with: + script: core.setFailed('Rollback on cancel should succeed.') + test-deploy-gcp-with-rollback: - needs: [test-deploy-ok-gcp] + needs: [test-deploy-gcp-cancel-rollback] runs-on: ubuntu-24.04 environment: dev timeout-minutes: 8 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c30e96e..e2b5011 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -293,6 +293,15 @@ jobs: image_set_path: ${{ env.GHA_HELM_DEPLOY_IMAGE_SET_PATH }} container_name: ${{ env.GHA_HELM_DEPLOY_CONTAINER_NAME }} registry: ${{ env.GHA_HELM_DEPLOY_REGISTRY }} + # Helm rollback on workflow cancellation and the deployment is not already finished (status 0) + - id: helm-rollback-on-cancel + name: Helm rollback on cancellation + if: cancelled() && steps.helm-deploy.outputs.HELM_DEPLOY_STATUS != '0' + uses: entur/gha-helm/.github/actions/helm-rollback-on-cancel@feat/rollback-on-workflow-cancel # TODO Change to v1 + with: + release_name: ${{ env.GHA_HELM_DEPLOY_RELEASE_NAME }} + namespace: ${{ env.GHA_HELM_DEPLOY_NAMESPACE }} + timeout_minutes: ${{ inputs.timeout_minutes }} - id: build-slack-message name: Build Slack deploy status message if: always() && inputs.slack_channel_id != '' diff --git a/AGENTS.md b/AGENTS.md index 22d2f10..2f63459 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,8 +11,9 @@ See [entur/ai AGENTS.md](https://github.com/entur/ai/blob/main/AGENTS.md) for co ``` .github/ actions/ - helm-lint/action.yml # Composite action: helm dependency update + lint - helm-deploy/action.yml # Composite action: helm upgrade --install with rollback + helm-lint/action.yml # Composite action: helm dependency update + lint + helm-deploy/action.yml # Composite action: helm upgrade --install with rollback + helm-rollback-on-cancel/action.yml # Composite action: rollback on workflow cancellation workflows/ lint.yml # Reusable workflow: Helm lint (user-facing) deploy.yml # Reusable workflow: Helm deploy (user-facing) From f3b24200084fd7f2605c4f784f11e0079c9dd211 Mon Sep 17 00:00:00 2001 From: Anders Isaksen Date: Fri, 19 Jun 2026 11:57:20 +0200 Subject: [PATCH 2/2] Add dependency update step before upgrade --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 511bf20..c0f4d1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,7 @@ jobs: environment: dev - name: Put release in pending-upgrade state run: | + helm dependency update ./fixture/helm/amazing-app helm upgrade amazing-app ./fixture/helm/amazing-app \ -f fixture/helm/amazing-app/env/values-kub-ent-dev.yaml \ --namespace gha-ci --install --wait --timeout 10m \