Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions .github/actions/helm-rollback-on-cancel/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
name: Entur/Helm/Rollback-On-Cancel
description: Rolls back the Helm release if the job is canceled during deployment and a pending rollout exists
inputs:
release_name:
description: "The release name"
required: true
namespace:
description: "The Kubernetes namespace"
required: true
timeout_minutes:
description: "Job timeout in minutes (same value as helm-deploy)."
required: true
runs:
using: "composite"
steps:
- shell: bash
env:
TIMEOUT: ${{ inputs.timeout_minutes }}
RELEASE_NAME: ${{ inputs.release_name }}
NAMESPACE: ${{ inputs.namespace }}
run: |
TIMEOUT=$((TIMEOUT/2-2))
if (( TIMEOUT < 1 )); then TIMEOUT=1; fi
if (( TIMEOUT > 4 )); then TIMEOUT=4; fi
if helm list --pending -n "${NAMESPACE}" --filter "^${RELEASE_NAME}$" --short | grep -q .; then
echo "#################################################################################################"
echo " Job cancelled. Rolling back Helm release '${RELEASE_NAME}'..."
echo "#################################################################################################"
if helm rollback "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait --timeout "${TIMEOUT}m0s" --debug; then
echo "#################################################################################################"
echo " Helm rollback on cancellation successful."
echo "#################################################################################################"
else
echo "#################################################################################################"
echo " Helm rollback on cancellation failed. Please investigate manually."
echo "#################################################################################################"
exit 1
fi
else
echo "No active rollout found for '${RELEASE_NAME}' in '${NAMESPACE}'. Nothing to roll back."
fi
55 changes: 54 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -135,10 +135,63 @@ jobs:
slack_channel_id: GFBL95A0J # sandbox-channel
secrets: inherit

test-deploy-gcp-with-failed-rollback:
test-deploy-gcp-cancel-rollback:
needs: [test-deploy-ok-gcp]
runs-on: ubuntu-24.04
environment: dev
timeout-minutes: 10
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v6
- id: install
uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
with:
version: "v3.20.1"
- uses: entur/gha-meta/.github/actions/cloud-auth@v1
with:
environment: dev
gcp_workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }}
gcp_service_account: ${{ vars.SERVICE_ACCOUNT }}
- uses: entur/gha-meta/.github/actions/k8s-auth@v1
with:
environment: dev
- name: Put release in pending-upgrade state
run: |
helm upgrade amazing-app ./fixture/helm/amazing-app \
-f fixture/helm/amazing-app/env/values-kub-ent-dev.yaml \
--namespace gha-ci --install --wait --timeout 10m \
--set "common.container.image=eu.gcr.io/entur-system-1287/nonexistent:dummy" &
HELM_PID=$!
for i in $(seq 1 15); do
STATUS=$(helm status amazing-app -n gha-ci -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || echo "")
[ "$STATUS" = "pending-upgrade" ] && break
sleep 1
done
kill -9 $HELM_PID 2>/dev/null || true
wait $HELM_PID 2>/dev/null || true
STATUS=$(helm status amazing-app -n gha-ci -o json | jq -r '.info.status')
if [ "$STATUS" != "pending-upgrade" ]; then
echo "::error::Setup failed: expected pending-upgrade, got '$STATUS'"
exit 1
fi
- id: rollback-on-cancel
uses: ./.github/actions/helm-rollback-on-cancel
with:
release_name: amazing-app
namespace: gha-ci
timeout_minutes: 10
- name: Check rollback on cancel succeeded
if: steps.rollback-on-cancel.outcome != 'success'
uses: actions/github-script@v9
with:
script: core.setFailed('Rollback on cancel should succeed.')

test-deploy-gcp-with-failed-rollback:
needs: [test-deploy-gcp-cancel-rollback]
runs-on: ubuntu-24.04
environment: dev
timeout-minutes: 8
permissions:
contents: read
Expand Down
10 changes: 10 additions & 0 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,16 @@ jobs:
image_set_path: ${{ env.GHA_HELM_DEPLOY_IMAGE_SET_PATH }}
container_name: ${{ env.GHA_HELM_DEPLOY_CONTAINER_NAME }}
registry: ${{ env.GHA_HELM_DEPLOY_REGISTRY }}
# Helm rollback on workflow cancellation, only trigger if user cancels during deployment and the deployment
# has not succeeded, to avoid rolling back a successful deployment in case of cancel after rollout completion.
- id: helm-rollback-on-cancel
name: Helm rollback on cancellation
if: cancelled() && steps.helm-deploy.outputs.HELM_DEPLOY_STATUS != '0'
uses: ./.github/actions/helm-rollback-on-cancel
with:
release_name: ${{ env.GHA_HELM_DEPLOY_RELEASE_NAME }}
namespace: ${{ env.GHA_HELM_DEPLOY_NAMESPACE }}
timeout_minutes: ${{ inputs.timeout_minutes }}
- uses: entur/gha-meta/.github/actions/posthog@v1
id: send-analytics
name: Send analytics to PostHog
Expand Down
Loading