Skip to content
63 changes: 63 additions & 0 deletions .github/ci_scripts/helm-install-with-diagnostics.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/bin/bash
###############################################################
# Copyright (c) 2022 Contributors to the Eclipse Foundation
#
# See the NOTICE file(s) distributed with this work for additional
# information regarding copyright ownership.
#
# This program and the accompanying materials are made available under the
# terms of the Apache License, Version 2.0 which is available at
# https://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# SPDX-License-Identifier: Apache-2.0
###############################################################

# Script for Helm chart installation with comprehensive diagnostics
# Usage: ./helm-install-with-diagnostics.sh <CHART_NAME> <CHART_PATH> <VALUES_FILE> <NAMESPACE>

set -euo pipefail

CHART_NAME="${1:-}"
CHART_PATH="${2:-}"
VALUES_FILE="${3:-}"
NAMESPACE="${4:-}"

if [[ -z "$CHART_NAME" || -z "$CHART_PATH" || -z "$VALUES_FILE" || -z "$NAMESPACE" ]]; then
echo "Usage: $0 <CHART_NAME> <CHART_PATH> <VALUES_FILE> <NAMESPACE>"
echo "Example: $0 centralidp charts/centralidp charts/values-test-centralidp.yaml install"
exit 1
fi

echo "Installing $CHART_NAME chart..."
if ! helm install "$CHART_NAME" "$CHART_PATH" -f "$VALUES_FILE" --namespace "$NAMESPACE" --create-namespace --debug --wait --timeout=10m; then
echo "::error::Chart installation failed"
echo "Gathering diagnostic information..."

# Check pod status
echo "Pod Status:"
kubectl get pods -n "$NAMESPACE" -o wide || true

# Check events
echo "Namespace Events:"
kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' || true

# Get logs from failed pods
echo "Pod Logs:"
for pod in $(kubectl get pods -n "$NAMESPACE" --no-headers -o custom-columns=":metadata.name" | grep -v Running || true); do
echo "=== Logs for pod: $pod ==="
kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true --previous=true || true
kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true || true
echo "=== Describe pod: $pod ==="
kubectl describe pod "$pod" -n "$NAMESPACE" || true
done

exit 1
fi

echo "Chart installation successful"
155 changes: 155 additions & 0 deletions .github/ci_scripts/helm-upgrade-with-diagnostics.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
#!/bin/bash
###############################################################
# Copyright (c) 2022 Contributors to the Eclipse Foundation
#
# See the NOTICE file(s) distributed with this work for additional
# information regarding copyright ownership.
#
# This program and the accompanying materials are made available under the
# terms of the Apache License, Version 2.0 which is available at
# https://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# SPDX-License-Identifier: Apache-2.0
###############################################################

# Script for Helm upgrade testing with comprehensive diagnostics and fallback to --force
# Usage: ./helm-upgrade-with-diagnostics.sh <CHART_NAME> <CHART_PATH> <VALUES_FILE> <NAMESPACE> <BASE_VERSION>

set -euo pipefail

CHART_NAME="${1:-}"
CHART_PATH="${2:-}"
VALUES_FILE="${3:-}"
NAMESPACE="${4:-}"
BASE_VERSION="${5:-2.1.0}"

if [[ -z "$CHART_NAME" || -z "$CHART_PATH" || -z "$VALUES_FILE" || -z "$NAMESPACE" ]]; then
echo "Usage: $0 <CHART_NAME> <CHART_PATH> <VALUES_FILE> <NAMESPACE> [BASE_VERSION]"
echo "Example: $0 centralidp charts/centralidp charts/values-test-upgrade.yaml upgrade 2.1.0"
exit 1
fi

echo "Starting helm upgrade test for $CHART_NAME..."

# Setup helm repositories
helm repo add bitnami-full-index https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
helm repo add tractusx-dev https://eclipse-tractusx.github.io/charts/dev
helm repo update

echo "Installing base version $BASE_VERSION..."
if ! helm install "$CHART_NAME" "tractusx-dev/$CHART_NAME" -f "$VALUES_FILE" --version "$BASE_VERSION" --namespace "$NAMESPACE" --create-namespace --debug --wait --timeout=15m \
--set keycloak.image.registry=docker.io \
--set keycloak.image.repository=bitnamilegacy/keycloak \
--set keycloak.postgresql.image.registry=docker.io \
--set keycloak.postgresql.image.repository=bitnamilegacy/postgresql \
--set keycloak.startupProbe.enabled=false \
--set keycloak.livenessProbe.initialDelaySeconds=600 \
--set keycloak.readinessProbe.initialDelaySeconds=300; then
echo "::error::Base version installation failed"
echo "Gathering diagnostic information for base installation..."

# Check pod status
echo "Pod Status:"
kubectl get pods -n "$NAMESPACE" -o wide || true

# Check events
echo "Namespace Events:"
kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' || true

# Get logs from failed pods
echo "Pod Logs:"
for pod in $(kubectl get pods -n "$NAMESPACE" --no-headers -o custom-columns=":metadata.name" 2>/dev/null || true); do
echo "=== Logs for pod: $pod ==="
kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true --previous=true 2>/dev/null || true
kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true 2>/dev/null || true
echo "=== Describe pod: $pod ==="
kubectl describe pod "$pod" -n "$NAMESPACE" || true
done

exit 1
fi

echo "Base version installed successfully"
helm dependency update "$CHART_PATH"

echo "⬆️ Attempting upgrade..."
# First attempt upgrade without force
if ! helm upgrade "$CHART_NAME" "$CHART_PATH" -f "$VALUES_FILE" --namespace "$NAMESPACE" --debug --wait --timeout=10m --atomic; then
echo "::warning::Initial upgrade failed, attempting with --force flag"
echo "📋 Gathering diagnostic information before retry..."

# Check current state
echo "Current Pod Status:"
kubectl get pods -n "$NAMESPACE" -o wide || true

echo "Recent Events:"
kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | tail -20 || true

# Get logs from problematic pods
echo "Current Pod Logs:"
for pod in $(kubectl get pods -n "$NAMESPACE" --no-headers -o custom-columns=":metadata.name" 2>/dev/null || true); do
if [[ $(kubectl get pod "$pod" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null) != "Running" ]]; then
echo "=== Logs for non-running pod: $pod ==="
kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true --previous=true 2>/dev/null || true
kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true 2>/dev/null || true
kubectl describe pod "$pod" -n "$NAMESPACE" || true
fi
done

echo "Retrying upgrade with --force flag..."
# Retry with force flag if first attempt fails
if ! helm upgrade "$CHART_NAME" "$CHART_PATH" -f "$VALUES_FILE" --namespace "$NAMESPACE" --debug --wait --timeout=10m --atomic --force; then
echo "::error::Upgrade failed even with --force flag"
echo "📋 Final diagnostic information..."

# Comprehensive diagnostics for complete failure
echo "Final Pod Status:"
kubectl get pods -n "$NAMESPACE" -o wide || true

echo "All Events:"
kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' || true

echo "StatefulSet Status:"
kubectl get statefulset -n "$NAMESPACE" -o wide || true
kubectl describe statefulset -n "$NAMESPACE" || true

echo "Service Status:"
kubectl get svc -n "$NAMESPACE" -o wide || true

echo "PVC Status:"
kubectl get pvc -n "$NAMESPACE" -o wide || true

echo "All Pod Logs:"
for pod in $(kubectl get pods -n "$NAMESPACE" --no-headers -o custom-columns=":metadata.name" 2>/dev/null || true); do
echo "=== Complete logs for pod: $pod ==="
kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true --previous=true 2>/dev/null || true
kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true 2>/dev/null || true
echo "=== Complete describe for pod: $pod ==="
kubectl describe pod "$pod" -n "$NAMESPACE" || true
echo "=========================="
done

# Check for common issues
echo "Checking for common issues..."
kubectl get events -n "$NAMESPACE" --field-selector type=Warning || true

echo "Both normal and forced upgrade attempts failed"
exit 1
else
echo "::notice::Upgrade succeeded with --force flag"
echo "Final verification..."
kubectl get pods -n "$NAMESPACE" -o wide
echo "Upgrade completed successfully using --force flag"
fi
else
echo "::notice::Upgrade succeeded without force"
echo "Final verification..."
kubectl get pods -n "$NAMESPACE" -o wide
echo "Upgrade completed successfully without --force flag"
fi
57 changes: 57 additions & 0 deletions .github/ci_scripts/verify-upgrade-success.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/bin/bash
###############################################################
# Copyright (c) 2022 Contributors to the Eclipse Foundation
#
# See the NOTICE file(s) distributed with this work for additional
# information regarding copyright ownership.
#
# This program and the accompanying materials are made available under the
# terms of the Apache License, Version 2.0 which is available at
# https://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# SPDX-License-Identifier: Apache-2.0
###############################################################

# Script for post-upgrade verification
# Usage: ./verify-upgrade-success.sh <CHART_NAME> <NAMESPACE>

set -euo pipefail

CHART_NAME="${1:-}"
NAMESPACE="${2:-}"

if [[ -z "$CHART_NAME" || -z "$NAMESPACE" ]]; then
echo "Usage: $0 <CHART_NAME> <NAMESPACE>"
echo "Example: $0 centralidp upgrade"
exit 1
fi

echo "🔍 Verifying upgrade success for $CHART_NAME..."

# Wait for pods to be ready
echo "Waiting for pods to be ready..."
kubectl wait --for=condition=Ready pods --all -n "$NAMESPACE" --timeout=300s || true

# Check final status
echo "Final Pod Status:"
kubectl get pods -n "$NAMESPACE" -o wide

# Check Helm release status
echo "Helm Release Status:"
helm status "$CHART_NAME" -n "$NAMESPACE"

# Test basic connectivity if possible
echo "Testing basic service connectivity..."
kubectl get svc -n "$NAMESPACE"

# Check for any remaining issues
echo "Checking for any warnings or errors..."
kubectl get events -n "$NAMESPACE" --field-selector type=Warning | tail -10 || true

echo "Verification completed"
16 changes: 8 additions & 8 deletions .github/workflows/centralidp-chart-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,21 +89,21 @@ jobs:
echo "changed=true" >> $GITHUB_OUTPUT
fi

# run chart linting
# run chart linting
- name: Run chart-testing (lint)
run: ct lint --charts charts/centralidp --config charts/chart-testing-config.yaml

# define charts to test with the --charts parameter
- name: Run chart-testing (install)
run: helm install centralidp charts/centralidp -f charts/values-test-centralidp.yaml --namespace install --create-namespace --debug
run: ./.github/ci_scripts/helm-install-with-diagnostics.sh centralidp charts/centralidp charts/values-test-centralidp.yaml install
if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true'

# Upgrade the released centralidp chart version with the locally available chart
- name: Run helm upgrade
run: |
helm repo add bitnami-full-index https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
helm repo add tractusx-dev https://eclipse-tractusx.github.io/charts/dev
helm install centralidp tractusx-dev/centralidp -f charts/values-test-upgrade.yaml --version ${{ github.event.inputs.upgrade_from || '2.1.0' }} --namespace upgrade --create-namespace --debug
helm dependency update charts/centralidp
helm upgrade centralidp charts/centralidp -f charts/values-test-upgrade.yaml --namespace upgrade --debug
run: ./.github/ci_scripts/helm-upgrade-with-diagnostics.sh centralidp charts/centralidp charts/values-test-upgrade.yaml upgrade ${{ github.event.inputs.upgrade_from || '2.1.0' }}
if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true'

# Post-upgrade verification
- name: Verify upgrade success
run: ./.github/ci_scripts/verify-upgrade-success.sh centralidp upgrade
if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true'
18 changes: 9 additions & 9 deletions .github/workflows/sharedidp-chart-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,21 +89,21 @@ jobs:
echo "changed=true" >> $GITHUB_OUTPUT
fi

# run chart linting
# Run chart linting
- name: Run chart-testing (lint)
run: ct lint --charts charts/sharedidp --config charts/chart-testing-config.yaml

# define charts to test with the --charts parameter
- name: Run chart-testing (install)
run: helm install sharedidp charts/sharedidp -f charts/values-test-sharedidp.yaml --namespace install --create-namespace --debug
run: ./.github/ci_scripts/helm-install-with-diagnostics.sh sharedidp charts/sharedidp charts/values-test-sharedidp.yaml install
if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true'

# Upgrade the released sharedidp chart version with the locally available chart
# Upgrade the released sharedidp chart version with the locally available chart
- name: Run helm upgrade
run: |
helm repo add bitnami-full-index https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
helm repo add tractusx-dev https://eclipse-tractusx.github.io/charts/dev
helm install sharedidp tractusx-dev/sharedidp -f charts/values-test-upgrade.yaml --version ${{ github.event.inputs.upgrade_from || '2.1.0' }} --namespace upgrade --create-namespace --debug
helm dependency update charts/sharedidp
helm upgrade sharedidp charts/sharedidp -f charts/values-test-upgrade.yaml --namespace upgrade --debug
run: ./.github/ci_scripts/helm-upgrade-with-diagnostics.sh sharedidp charts/sharedidp charts/values-test-upgrade.yaml upgrade ${{ github.event.inputs.upgrade_from || '2.1.0' }}
if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true'

# Post-upgrade verification
- name: Verify upgrade success
run: ./.github/ci_scripts/verify-upgrade-success.sh sharedidp upgrade
if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true'
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,6 @@ Chart.lock

### Visual Studio Code ###
.vscode/

.DS_Store
**/.DS_Store
4 changes: 2 additions & 2 deletions charts/centralidp/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ apiVersion: v2
name: centralidp
type: application
version: 4.2.1
appVersion: 25.0.6
appVersion: 26.3.3
description: Helm chart for Central Keycloak Instance
home: https://github.com/eclipse-tractusx/portal-iam
sources:
- https://github.com/eclipse-tractusx/portal-iam
dependencies:
- name: keycloak
repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
version: 23.0.0
version: 25.2.0
8 changes: 3 additions & 5 deletions charts/centralidp/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ keycloak:
image:
registry: docker.io
repository: bitnamilegacy/keycloak
tag: 25.0.6-debian-12-r0
tag: 26.3.3-debian-12-r0
auth:
adminUser: admin
# -- centralidp Keycloak administrator password.
Expand All @@ -30,9 +30,7 @@ keycloak:
existingSecret: ""
# -- Run Keycloak in production mode. TLS configuration is required except when using proxy=edge.
production: false
# -- Setting the path relative to '/' for serving resources:
# as we're migrating from 16.1.1 version which was using the trailing 'auth', we're setting it to '/auth/'.
# ref: https://www.keycloak.org/migration/migrating-to-quarkus#_default_context_path_changed
# -- Set the path relative to '/' for serving resources (maintaining /auth for backward compatibility)
httpRelativePath: /auth/
replicaCount: 1
extraVolumes:
Expand Down Expand Up @@ -101,7 +99,7 @@ keycloak:
image:
registry: docker.io
repository: bitnamilegacy/postgresql
tag: 15-debian-11
tag: 17.6.0-debian-12-r4
commonLabels:
app.kubernetes.io/version: "15"
auth:
Expand Down
Loading
Loading