diff --git a/.github/ci_scripts/helm-install-with-diagnostics.sh b/.github/ci_scripts/helm-install-with-diagnostics.sh new file mode 100755 index 00000000..372e5f2a --- /dev/null +++ b/.github/ci_scripts/helm-install-with-diagnostics.sh @@ -0,0 +1,63 @@ +#!/bin/bash +############################################################### +# Copyright (c) 2022 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +############################################################### + +# Script for Helm chart installation with comprehensive diagnostics +# Usage: ./helm-install-with-diagnostics.sh + +set -euo pipefail + +CHART_NAME="${1:-}" +CHART_PATH="${2:-}" +VALUES_FILE="${3:-}" +NAMESPACE="${4:-}" + +if [[ -z "$CHART_NAME" || -z "$CHART_PATH" || -z "$VALUES_FILE" || -z "$NAMESPACE" ]]; then + echo "Usage: $0 " + echo "Example: $0 centralidp charts/centralidp charts/values-test-centralidp.yaml install" + exit 1 +fi + +echo "Installing $CHART_NAME chart..." +if ! helm install "$CHART_NAME" "$CHART_PATH" -f "$VALUES_FILE" --namespace "$NAMESPACE" --create-namespace --debug --wait --timeout=10m; then + echo "::error::Chart installation failed" + echo "Gathering diagnostic information..." + + # Check pod status + echo "Pod Status:" + kubectl get pods -n "$NAMESPACE" -o wide || true + + # Check events + echo "Namespace Events:" + kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' || true + + # Get logs from failed pods + echo "Pod Logs:" + for pod in $(kubectl get pods -n "$NAMESPACE" --no-headers -o custom-columns=":metadata.name" | grep -v Running || true); do + echo "=== Logs for pod: $pod ===" + kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true --previous=true || true + kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true || true + echo "=== Describe pod: $pod ===" + kubectl describe pod "$pod" -n "$NAMESPACE" || true + done + + exit 1 +fi + +echo "Chart installation successful" \ No newline at end of file diff --git a/.github/ci_scripts/helm-upgrade-with-diagnostics.sh b/.github/ci_scripts/helm-upgrade-with-diagnostics.sh new file mode 100755 index 00000000..c6ac3bcd --- /dev/null +++ b/.github/ci_scripts/helm-upgrade-with-diagnostics.sh @@ -0,0 +1,155 @@ +#!/bin/bash +############################################################### +# Copyright (c) 2022 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +############################################################### + +# Script for Helm upgrade testing with comprehensive diagnostics and fallback to --force +# Usage: ./helm-upgrade-with-diagnostics.sh + +set -euo pipefail + +CHART_NAME="${1:-}" +CHART_PATH="${2:-}" +VALUES_FILE="${3:-}" +NAMESPACE="${4:-}" +BASE_VERSION="${5:-2.1.0}" + +if [[ -z "$CHART_NAME" || -z "$CHART_PATH" || -z "$VALUES_FILE" || -z "$NAMESPACE" ]]; then + echo "Usage: $0 [BASE_VERSION]" + echo "Example: $0 centralidp charts/centralidp charts/values-test-upgrade.yaml upgrade 2.1.0" + exit 1 +fi + +echo "Starting helm upgrade test for $CHART_NAME..." + +# Setup helm repositories +helm repo add bitnami-full-index https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami +helm repo add tractusx-dev https://eclipse-tractusx.github.io/charts/dev +helm repo update + +echo "Installing base version $BASE_VERSION..." +if ! helm install "$CHART_NAME" "tractusx-dev/$CHART_NAME" -f "$VALUES_FILE" --version "$BASE_VERSION" --namespace "$NAMESPACE" --create-namespace --debug --wait --timeout=15m \ + --set keycloak.image.registry=docker.io \ + --set keycloak.image.repository=bitnamilegacy/keycloak \ + --set keycloak.postgresql.image.registry=docker.io \ + --set keycloak.postgresql.image.repository=bitnamilegacy/postgresql \ + --set keycloak.startupProbe.enabled=false \ + --set keycloak.livenessProbe.initialDelaySeconds=600 \ + --set keycloak.readinessProbe.initialDelaySeconds=300; then + echo "::error::Base version installation failed" + echo "Gathering diagnostic information for base installation..." + + # Check pod status + echo "Pod Status:" + kubectl get pods -n "$NAMESPACE" -o wide || true + + # Check events + echo "Namespace Events:" + kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' || true + + # Get logs from failed pods + echo "Pod Logs:" + for pod in $(kubectl get pods -n "$NAMESPACE" --no-headers -o custom-columns=":metadata.name" 2>/dev/null || true); do + echo "=== Logs for pod: $pod ===" + kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true --previous=true 2>/dev/null || true + kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true 2>/dev/null || true + echo "=== Describe pod: $pod ===" + kubectl describe pod "$pod" -n "$NAMESPACE" || true + done + + exit 1 +fi + +echo "Base version installed successfully" +helm dependency update "$CHART_PATH" + +echo "⬆️ Attempting upgrade..." +# First attempt upgrade without force +if ! helm upgrade "$CHART_NAME" "$CHART_PATH" -f "$VALUES_FILE" --namespace "$NAMESPACE" --debug --wait --timeout=10m --atomic; then + echo "::warning::Initial upgrade failed, attempting with --force flag" + echo "📋 Gathering diagnostic information before retry..." + + # Check current state + echo "Current Pod Status:" + kubectl get pods -n "$NAMESPACE" -o wide || true + + echo "Recent Events:" + kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | tail -20 || true + + # Get logs from problematic pods + echo "Current Pod Logs:" + for pod in $(kubectl get pods -n "$NAMESPACE" --no-headers -o custom-columns=":metadata.name" 2>/dev/null || true); do + if [[ $(kubectl get pod "$pod" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null) != "Running" ]]; then + echo "=== Logs for non-running pod: $pod ===" + kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true --previous=true 2>/dev/null || true + kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true 2>/dev/null || true + kubectl describe pod "$pod" -n "$NAMESPACE" || true + fi + done + + echo "Retrying upgrade with --force flag..." + # Retry with force flag if first attempt fails + if ! helm upgrade "$CHART_NAME" "$CHART_PATH" -f "$VALUES_FILE" --namespace "$NAMESPACE" --debug --wait --timeout=10m --atomic --force; then + echo "::error::Upgrade failed even with --force flag" + echo "📋 Final diagnostic information..." + + # Comprehensive diagnostics for complete failure + echo "Final Pod Status:" + kubectl get pods -n "$NAMESPACE" -o wide || true + + echo "All Events:" + kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' || true + + echo "StatefulSet Status:" + kubectl get statefulset -n "$NAMESPACE" -o wide || true + kubectl describe statefulset -n "$NAMESPACE" || true + + echo "Service Status:" + kubectl get svc -n "$NAMESPACE" -o wide || true + + echo "PVC Status:" + kubectl get pvc -n "$NAMESPACE" -o wide || true + + echo "All Pod Logs:" + for pod in $(kubectl get pods -n "$NAMESPACE" --no-headers -o custom-columns=":metadata.name" 2>/dev/null || true); do + echo "=== Complete logs for pod: $pod ===" + kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true --previous=true 2>/dev/null || true + kubectl logs "$pod" -n "$NAMESPACE" --all-containers=true 2>/dev/null || true + echo "=== Complete describe for pod: $pod ===" + kubectl describe pod "$pod" -n "$NAMESPACE" || true + echo "==========================" + done + + # Check for common issues + echo "Checking for common issues..." + kubectl get events -n "$NAMESPACE" --field-selector type=Warning || true + + echo "Both normal and forced upgrade attempts failed" + exit 1 + else + echo "::notice::Upgrade succeeded with --force flag" + echo "Final verification..." + kubectl get pods -n "$NAMESPACE" -o wide + echo "Upgrade completed successfully using --force flag" + fi +else + echo "::notice::Upgrade succeeded without force" + echo "Final verification..." + kubectl get pods -n "$NAMESPACE" -o wide + echo "Upgrade completed successfully without --force flag" +fi \ No newline at end of file diff --git a/.github/ci_scripts/verify-upgrade-success.sh b/.github/ci_scripts/verify-upgrade-success.sh new file mode 100755 index 00000000..f3408586 --- /dev/null +++ b/.github/ci_scripts/verify-upgrade-success.sh @@ -0,0 +1,57 @@ +#!/bin/bash +############################################################### +# Copyright (c) 2022 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +############################################################### + +# Script for post-upgrade verification +# Usage: ./verify-upgrade-success.sh + +set -euo pipefail + +CHART_NAME="${1:-}" +NAMESPACE="${2:-}" + +if [[ -z "$CHART_NAME" || -z "$NAMESPACE" ]]; then + echo "Usage: $0 " + echo "Example: $0 centralidp upgrade" + exit 1 +fi + +echo "🔍 Verifying upgrade success for $CHART_NAME..." + +# Wait for pods to be ready +echo "Waiting for pods to be ready..." +kubectl wait --for=condition=Ready pods --all -n "$NAMESPACE" --timeout=300s || true + +# Check final status +echo "Final Pod Status:" +kubectl get pods -n "$NAMESPACE" -o wide + +# Check Helm release status +echo "Helm Release Status:" +helm status "$CHART_NAME" -n "$NAMESPACE" + +# Test basic connectivity if possible +echo "Testing basic service connectivity..." +kubectl get svc -n "$NAMESPACE" + +# Check for any remaining issues +echo "Checking for any warnings or errors..." +kubectl get events -n "$NAMESPACE" --field-selector type=Warning | tail -10 || true + +echo "Verification completed" \ No newline at end of file diff --git a/.github/workflows/centralidp-chart-test.yaml b/.github/workflows/centralidp-chart-test.yaml index 6c01e015..0ecfc163 100644 --- a/.github/workflows/centralidp-chart-test.yaml +++ b/.github/workflows/centralidp-chart-test.yaml @@ -89,21 +89,21 @@ jobs: echo "changed=true" >> $GITHUB_OUTPUT fi - # run chart linting + # run chart linting - name: Run chart-testing (lint) run: ct lint --charts charts/centralidp --config charts/chart-testing-config.yaml # define charts to test with the --charts parameter - name: Run chart-testing (install) - run: helm install centralidp charts/centralidp -f charts/values-test-centralidp.yaml --namespace install --create-namespace --debug + run: ./.github/ci_scripts/helm-install-with-diagnostics.sh centralidp charts/centralidp charts/values-test-centralidp.yaml install if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true' # Upgrade the released centralidp chart version with the locally available chart - name: Run helm upgrade - run: | - helm repo add bitnami-full-index https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami - helm repo add tractusx-dev https://eclipse-tractusx.github.io/charts/dev - helm install centralidp tractusx-dev/centralidp -f charts/values-test-upgrade.yaml --version ${{ github.event.inputs.upgrade_from || '2.1.0' }} --namespace upgrade --create-namespace --debug - helm dependency update charts/centralidp - helm upgrade centralidp charts/centralidp -f charts/values-test-upgrade.yaml --namespace upgrade --debug + run: ./.github/ci_scripts/helm-upgrade-with-diagnostics.sh centralidp charts/centralidp charts/values-test-upgrade.yaml upgrade ${{ github.event.inputs.upgrade_from || '2.1.0' }} + if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true' + + # Post-upgrade verification + - name: Verify upgrade success + run: ./.github/ci_scripts/verify-upgrade-success.sh centralidp upgrade if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true' diff --git a/.github/workflows/sharedidp-chart-test.yaml b/.github/workflows/sharedidp-chart-test.yaml index 126b7f53..2f199d39 100644 --- a/.github/workflows/sharedidp-chart-test.yaml +++ b/.github/workflows/sharedidp-chart-test.yaml @@ -89,21 +89,21 @@ jobs: echo "changed=true" >> $GITHUB_OUTPUT fi - # run chart linting + # Run chart linting - name: Run chart-testing (lint) run: ct lint --charts charts/sharedidp --config charts/chart-testing-config.yaml # define charts to test with the --charts parameter - name: Run chart-testing (install) - run: helm install sharedidp charts/sharedidp -f charts/values-test-sharedidp.yaml --namespace install --create-namespace --debug + run: ./.github/ci_scripts/helm-install-with-diagnostics.sh sharedidp charts/sharedidp charts/values-test-sharedidp.yaml install if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true' - # Upgrade the released sharedidp chart version with the locally available chart + # Upgrade the released sharedidp chart version with the locally available chart - name: Run helm upgrade - run: | - helm repo add bitnami-full-index https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami - helm repo add tractusx-dev https://eclipse-tractusx.github.io/charts/dev - helm install sharedidp tractusx-dev/sharedidp -f charts/values-test-upgrade.yaml --version ${{ github.event.inputs.upgrade_from || '2.1.0' }} --namespace upgrade --create-namespace --debug - helm dependency update charts/sharedidp - helm upgrade sharedidp charts/sharedidp -f charts/values-test-upgrade.yaml --namespace upgrade --debug + run: ./.github/ci_scripts/helm-upgrade-with-diagnostics.sh sharedidp charts/sharedidp charts/values-test-upgrade.yaml upgrade ${{ github.event.inputs.upgrade_from || '2.1.0' }} + if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true' + + # Post-upgrade verification + - name: Verify upgrade success + run: ./.github/ci_scripts/verify-upgrade-success.sh sharedidp upgrade if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true' diff --git a/.gitignore b/.gitignore index 64d3c185..2cf1b97a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ Chart.lock ### Visual Studio Code ### .vscode/ + +.DS_Store +**/.DS_Store diff --git a/charts/centralidp/Chart.yaml b/charts/centralidp/Chart.yaml index ea9add1b..99fb55ba 100644 --- a/charts/centralidp/Chart.yaml +++ b/charts/centralidp/Chart.yaml @@ -21,7 +21,7 @@ apiVersion: v2 name: centralidp type: application version: 4.2.1 -appVersion: 25.0.6 +appVersion: 26.3.3 description: Helm chart for Central Keycloak Instance home: https://github.com/eclipse-tractusx/portal-iam sources: @@ -29,4 +29,4 @@ sources: dependencies: - name: keycloak repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami - version: 23.0.0 + version: 25.2.0 diff --git a/charts/centralidp/values.yaml b/charts/centralidp/values.yaml index 708c9749..4ada1408 100644 --- a/charts/centralidp/values.yaml +++ b/charts/centralidp/values.yaml @@ -21,7 +21,7 @@ keycloak: image: registry: docker.io repository: bitnamilegacy/keycloak - tag: 25.0.6-debian-12-r0 + tag: 26.3.3-debian-12-r0 auth: adminUser: admin # -- centralidp Keycloak administrator password. @@ -30,9 +30,7 @@ keycloak: existingSecret: "" # -- Run Keycloak in production mode. TLS configuration is required except when using proxy=edge. production: false - # -- Setting the path relative to '/' for serving resources: - # as we're migrating from 16.1.1 version which was using the trailing 'auth', we're setting it to '/auth/'. - # ref: https://www.keycloak.org/migration/migrating-to-quarkus#_default_context_path_changed + # -- Set the path relative to '/' for serving resources (maintaining /auth for backward compatibility) httpRelativePath: /auth/ replicaCount: 1 extraVolumes: @@ -101,7 +99,7 @@ keycloak: image: registry: docker.io repository: bitnamilegacy/postgresql - tag: 15-debian-11 + tag: 17.6.0-debian-12-r4 commonLabels: app.kubernetes.io/version: "15" auth: diff --git a/charts/sharedidp/Chart.yaml b/charts/sharedidp/Chart.yaml index 5ff1163e..3415d80b 100644 --- a/charts/sharedidp/Chart.yaml +++ b/charts/sharedidp/Chart.yaml @@ -21,7 +21,7 @@ apiVersion: v2 name: sharedidp type: application version: 4.2.1 -appVersion: 25.0.6 +appVersion: 26.3.3 description: Helm chart for Shared Keycloak Instance home: https://github.com/eclipse-tractusx/portal-iam sources: @@ -29,4 +29,4 @@ sources: dependencies: - name: keycloak repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami - version: 23.0.0 + version: 25.2.0 diff --git a/charts/sharedidp/values.yaml b/charts/sharedidp/values.yaml index cbdd7b82..8b1b453f 100644 --- a/charts/sharedidp/values.yaml +++ b/charts/sharedidp/values.yaml @@ -21,7 +21,7 @@ keycloak: image: registry: docker.io repository: bitnamilegacy/keycloak - tag: 25.0.6-debian-12-r0 + tag: 26.3.3-debian-12-r0 auth: adminUser: admin # -- sharedidp Keycloak administrator password. @@ -109,7 +109,7 @@ keycloak: image: registry: docker.io repository: bitnamilegacy/postgresql - tag: 15-debian-11 + tag: 17.6.0-debian-12-r4 commonLabels: app.kubernetes.io/version: "15" auth: