From df79fd9a017cb4ca5800d93f01e2bf210eae3f61 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 8 Jul 2026 12:21:20 -0400 Subject: [PATCH 01/34] [ACE-1975] Downgrade action versions from v6 to v4 that use Node.js 20 --- .github/workflows/deploy-aks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 6d4f4d0c..2c96328d 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -61,7 +61,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: persist-credentials: false @@ -135,7 +135,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: persist-credentials: false From 1bed9b9170ccfee6552b58b897c244ea1083c4ed Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 8 Jul 2026 14:45:36 -0400 Subject: [PATCH 02/34] [ACE-1975] Set ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION to force fallback to Node 20 --- .github/workflows/deploy-aks.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 2c96328d..af7e6304 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -1,5 +1,11 @@ name: Deploy to AKS +# WORKAROUND: Self-hosted runner (github-dockerbuild02) has outdated glibc (<2.27), +# which is incompatible with Node 24. This forces actions to use Node 20 until the +# runner OS is upgraded. Remove once the runner supports glibc 2.27+. +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + # DEPLOYMENT STRATEGY: # - Workaround reason: GitHub Environments are not available on this GitHub plan, # so we cannot use environment-based OIDC subjects. From d340efe7a9a16bdb94c0b01af7dda50736bdf9f2 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 8 Jul 2026 22:32:51 -0400 Subject: [PATCH 03/34] [ACE-1975] Switch AKS deploy to az aks command invoke via ARM API --- .github/workflows/deploy-aks.yml | 105 ++++++++++++------------------- 1 file changed, 41 insertions(+), 64 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index af7e6304..1e64667d 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -1,12 +1,9 @@ name: Deploy to AKS -# WORKAROUND: Self-hosted runner (github-dockerbuild02) has outdated glibc (<2.27), -# which is incompatible with Node 24. This forces actions to use Node 20 until the -# runner OS is upgraded. Remove once the runner supports glibc 2.27+. -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - # DEPLOYMENT STRATEGY: +# - Uses az aks command invoke to deploy through Azure ARM API, which bypasses the +# private AKS API endpoint. The runner never connects to the cluster directly. +# All cluster operations run in a temporary pod inside the cluster via Azure ARM. # - Workaround reason: GitHub Environments are not available on this GitHub plan, # so we cannot use environment-based OIDC subjects. # - Therefore deployments are executed under the trusted main ref subject @@ -60,7 +57,7 @@ jobs: deploy-orchestrator: name: Deploy Orchestrator if: ${{ inputs.deploy_orchestrator }} - runs-on: [self-hosted, Linux, X64] + runs-on: ubuntu-latest timeout-minutes: 30 permissions: id-token: write @@ -78,22 +75,6 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} - - name: Install kubelogin - uses: azure/use-kubelogin@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - kubelogin-version: 'v0.2.14' - - - name: Set AKS context - uses: azure/aks-set-context@v4 - with: - resource-group: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - cluster-name: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} - - - name: Convert kubeconfig for non-interactive login - run: kubelogin convert-kubeconfig -l azurecli - - name: Setup Helm uses: azure/setup-helm@v4 @@ -108,33 +89,39 @@ jobs: - name: Helm deploy orchestrator env: - K8S_NAMESPACE: ${{ inputs.namespace }} + RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} DRY_RUN: ${{ inputs.dry_run }} ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail - HELM_ARGS=( - -f "./helm/values-${{ inputs.environment }}.yaml" - --namespace "$K8S_NAMESPACE" - --set "image.tag=${{ inputs.image_tag }}" - --set config.mode=orchestrator - --set fullnameOverride=prl-devops-orchestrator - ) + + HELM_CMD="helm upgrade --install prl-devops-orchestrator ./helm" + HELM_CMD+=" -f ./helm/values-${{ inputs.environment }}.yaml" + HELM_CMD+=" --namespace ${{ inputs.namespace }}" + HELM_CMD+=" --set image.tag=${{ inputs.image_tag }}" + HELM_CMD+=" --set config.mode=orchestrator" + HELM_CMD+=" --set fullnameOverride=prl-devops-orchestrator" if [[ "$DRY_RUN" == "true" ]]; then - helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" --dry-run --debug + HELM_CMD+=" --dry-run --debug" else - DEPLOY_ARGS=(--wait --timeout 5m) + HELM_CMD+=" --wait --timeout 5m" if [[ "$ATOMIC" == "true" ]]; then - DEPLOY_ARGS+=("--atomic") + HELM_CMD+=" --atomic" fi - helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" fi + az aks command invoke \ + --resource-group "$RESOURCE_GROUP" \ + --name "$CLUSTER_NAME" \ + --command "$HELM_CMD" \ + --file helm + deploy-catalog: name: Deploy Catalog if: ${{ inputs.deploy_catalog }} - runs-on: [self-hosted, Linux, X64] + runs-on: ubuntu-latest timeout-minutes: 30 permissions: id-token: write @@ -152,22 +139,6 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} - - name: Install kubelogin - uses: azure/use-kubelogin@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - kubelogin-version: 'v0.2.14' - - - name: Set AKS context - uses: azure/aks-set-context@v4 - with: - resource-group: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - cluster-name: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} - - - name: Convert kubeconfig for non-interactive login - run: kubelogin convert-kubeconfig -l azurecli - - name: Setup Helm uses: azure/setup-helm@v4 @@ -182,25 +153,31 @@ jobs: - name: Helm deploy catalog env: - K8S_NAMESPACE: ${{ inputs.namespace }} + RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} DRY_RUN: ${{ inputs.dry_run }} ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail - HELM_ARGS=( - -f "./helm/values-${{ inputs.environment }}.yaml" - --namespace "$K8S_NAMESPACE" - --set "image.tag=${{ inputs.image_tag }}" - --set config.mode=catalog - --set fullnameOverride=prl-devops-catalog - ) + + HELM_CMD="helm upgrade --install prl-devops-catalog ./helm" + HELM_CMD+=" -f ./helm/values-${{ inputs.environment }}.yaml" + HELM_CMD+=" --namespace ${{ inputs.namespace }}" + HELM_CMD+=" --set image.tag=${{ inputs.image_tag }}" + HELM_CMD+=" --set config.mode=catalog" + HELM_CMD+=" --set fullnameOverride=prl-devops-catalog" if [[ "$DRY_RUN" == "true" ]]; then - helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" --dry-run --debug + HELM_CMD+=" --dry-run --debug" else - DEPLOY_ARGS=(--wait --timeout 5m) + HELM_CMD+=" --wait --timeout 5m" if [[ "$ATOMIC" == "true" ]]; then - DEPLOY_ARGS+=("--atomic") + HELM_CMD+=" --atomic" fi - helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" fi + + az aks command invoke \ + --resource-group "$RESOURCE_GROUP" \ + --name "$CLUSTER_NAME" \ + --command "$HELM_CMD" \ + --file helm From 7f3a2af5245e091afd4d108ecf6f5bbbd094e888 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 8 Jul 2026 23:33:50 -0400 Subject: [PATCH 04/34] [ACE-1975] Add .azcliignore to exclude large directories. Add verification step to confirm ./helm/ is accessible in pod working directory --- .azcliignore | 15 +++++++++++++++ .github/workflows/deploy-aks.yml | 28 ++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 .azcliignore diff --git a/.azcliignore b/.azcliignore new file mode 100644 index 00000000..698beff1 --- /dev/null +++ b/.azcliignore @@ -0,0 +1,15 @@ +# Exclude files/dirs from az aks command invoke --file . +# Reduces upload size from 305MB to ~296KB +# Only helm/ and top-level config files are uploaded to the temporary pod +# Reference: https://learn.microsoft.com/en-us/cli/azure/aks/command?view=azure-cli-latest#az-aks-command-invoke +.git +.azcliignore +src +docs +loadtest +scripts +badges +.github +.vscode +.claude +.claude-flow \ No newline at end of file diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 1e64667d..5d436167 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -87,6 +87,18 @@ jobs: --set fullnameOverride=prl-devops-orchestrator \ --set image.tag="${{ inputs.image_tag }}" + - name: Verify helm chart accessible + env: + RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} + run: | + set -euo pipefail + az aks command invoke \ + --resource-group "$RESOURCE_GROUP" \ + --name "$CLUSTER_NAME" \ + --command "ls -la ./helm/ && test -f ./helm/Chart.yaml && echo '✓ Chart.yaml found'" \ + --file . + - name: Helm deploy orchestrator env: RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} @@ -116,7 +128,7 @@ jobs: --resource-group "$RESOURCE_GROUP" \ --name "$CLUSTER_NAME" \ --command "$HELM_CMD" \ - --file helm + --file . deploy-catalog: name: Deploy Catalog @@ -151,6 +163,18 @@ jobs: --set fullnameOverride=prl-devops-catalog \ --set image.tag="${{ inputs.image_tag }}" + - name: Verify helm chart accessible + env: + RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} + run: | + set -euo pipefail + az aks command invoke \ + --resource-group "$RESOURCE_GROUP" \ + --name "$CLUSTER_NAME" \ + --command "ls -la ./helm/ && test -f ./helm/Chart.yaml && echo '✓ Chart.yaml found'" \ + --file . + - name: Helm deploy catalog env: RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} @@ -180,4 +204,4 @@ jobs: --resource-group "$RESOURCE_GROUP" \ --name "$CLUSTER_NAME" \ --command "$HELM_CMD" \ - --file helm + --file . From 0dd1ddad1180a0db24ed99e8d8cafcef4f6da9ae Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 8 Jul 2026 23:52:06 -0400 Subject: [PATCH 05/34] [ACE-1975] Update --file ./helm. Remove verification step and .azcliignore --- .azcliignore | 15 --------------- .github/workflows/deploy-aks.yml | 28 ++-------------------------- 2 files changed, 2 insertions(+), 41 deletions(-) delete mode 100644 .azcliignore diff --git a/.azcliignore b/.azcliignore deleted file mode 100644 index 698beff1..00000000 --- a/.azcliignore +++ /dev/null @@ -1,15 +0,0 @@ -# Exclude files/dirs from az aks command invoke --file . -# Reduces upload size from 305MB to ~296KB -# Only helm/ and top-level config files are uploaded to the temporary pod -# Reference: https://learn.microsoft.com/en-us/cli/azure/aks/command?view=azure-cli-latest#az-aks-command-invoke -.git -.azcliignore -src -docs -loadtest -scripts -badges -.github -.vscode -.claude -.claude-flow \ No newline at end of file diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 5d436167..34cbb24f 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -87,18 +87,6 @@ jobs: --set fullnameOverride=prl-devops-orchestrator \ --set image.tag="${{ inputs.image_tag }}" - - name: Verify helm chart accessible - env: - RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} - run: | - set -euo pipefail - az aks command invoke \ - --resource-group "$RESOURCE_GROUP" \ - --name "$CLUSTER_NAME" \ - --command "ls -la ./helm/ && test -f ./helm/Chart.yaml && echo '✓ Chart.yaml found'" \ - --file . - - name: Helm deploy orchestrator env: RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} @@ -128,7 +116,7 @@ jobs: --resource-group "$RESOURCE_GROUP" \ --name "$CLUSTER_NAME" \ --command "$HELM_CMD" \ - --file . + --file ./helm deploy-catalog: name: Deploy Catalog @@ -163,18 +151,6 @@ jobs: --set fullnameOverride=prl-devops-catalog \ --set image.tag="${{ inputs.image_tag }}" - - name: Verify helm chart accessible - env: - RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} - run: | - set -euo pipefail - az aks command invoke \ - --resource-group "$RESOURCE_GROUP" \ - --name "$CLUSTER_NAME" \ - --command "ls -la ./helm/ && test -f ./helm/Chart.yaml && echo '✓ Chart.yaml found'" \ - --file . - - name: Helm deploy catalog env: RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} @@ -204,4 +180,4 @@ jobs: --resource-group "$RESOURCE_GROUP" \ --name "$CLUSTER_NAME" \ --command "$HELM_CMD" \ - --file . + --file ./helm From ee4d4201a3faf1f88b0108c66eff6460a498f13b Mon Sep 17 00:00:00 2001 From: osh0501 Date: Thu, 9 Jul 2026 00:02:56 -0400 Subject: [PATCH 06/34] [ACE-1975] copy just the helm chart to a temp directory and run from there with --file . --- .github/workflows/deploy-aks.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 34cbb24f..87ad413e 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -96,6 +96,10 @@ jobs: run: | set -euo pipefail + # Copy helm chart to isolated dir so --file . uploads only the chart + DEPLOY_DIR=$(mktemp -d) + cp -r ./helm "$DEPLOY_DIR/helm" + HELM_CMD="helm upgrade --install prl-devops-orchestrator ./helm" HELM_CMD+=" -f ./helm/values-${{ inputs.environment }}.yaml" HELM_CMD+=" --namespace ${{ inputs.namespace }}" @@ -112,11 +116,12 @@ jobs: fi fi + cd "$DEPLOY_DIR" az aks command invoke \ --resource-group "$RESOURCE_GROUP" \ --name "$CLUSTER_NAME" \ --command "$HELM_CMD" \ - --file ./helm + --file . deploy-catalog: name: Deploy Catalog @@ -160,6 +165,10 @@ jobs: run: | set -euo pipefail + # Copy helm chart to isolated dir so --file . uploads only the chart + DEPLOY_DIR=$(mktemp -d) + cp -r ./helm "$DEPLOY_DIR/helm" + HELM_CMD="helm upgrade --install prl-devops-catalog ./helm" HELM_CMD+=" -f ./helm/values-${{ inputs.environment }}.yaml" HELM_CMD+=" --namespace ${{ inputs.namespace }}" @@ -176,8 +185,9 @@ jobs: fi fi + cd "$DEPLOY_DIR" az aks command invoke \ --resource-group "$RESOURCE_GROUP" \ --name "$CLUSTER_NAME" \ --command "$HELM_CMD" \ - --file ./helm + --file . From 4caef59d5ed35dfb22d2d2a9ffc78d6753425766 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Thu, 9 Jul 2026 16:25:06 -0400 Subject: [PATCH 07/34] [ACE-1975] Revert to use self-hosted runner and kubelogin. Install missing dependency (unzip) --- .github/workflows/deploy-aks.yml | 123 +++++++++++++++++-------------- 1 file changed, 69 insertions(+), 54 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 87ad413e..dae7b5c7 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -1,9 +1,6 @@ name: Deploy to AKS # DEPLOYMENT STRATEGY: -# - Uses az aks command invoke to deploy through Azure ARM API, which bypasses the -# private AKS API endpoint. The runner never connects to the cluster directly. -# All cluster operations run in a temporary pod inside the cluster via Azure ARM. # - Workaround reason: GitHub Environments are not available on this GitHub plan, # so we cannot use environment-based OIDC subjects. # - Therefore deployments are executed under the trusted main ref subject @@ -57,14 +54,14 @@ jobs: deploy-orchestrator: name: Deploy Orchestrator if: ${{ inputs.deploy_orchestrator }} - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64] timeout-minutes: 30 permissions: id-token: write contents: read steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: persist-credentials: false @@ -75,6 +72,26 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} + - name: Install dependencies + run: | + which unzip || sudo dnf install -y unzip + + - name: Install kubelogin + uses: azure/use-kubelogin@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + kubelogin-version: 'v0.2.14' + + - name: Set AKS context + uses: azure/aks-set-context@v4 + with: + resource-group: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + cluster-name: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} + + - name: Convert kubeconfig for non-interactive login + run: kubelogin convert-kubeconfig -l azurecli + - name: Setup Helm uses: azure/setup-helm@v4 @@ -89,51 +106,40 @@ jobs: - name: Helm deploy orchestrator env: - RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} + K8S_NAMESPACE: ${{ inputs.namespace }} DRY_RUN: ${{ inputs.dry_run }} ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail - - # Copy helm chart to isolated dir so --file . uploads only the chart - DEPLOY_DIR=$(mktemp -d) - cp -r ./helm "$DEPLOY_DIR/helm" - - HELM_CMD="helm upgrade --install prl-devops-orchestrator ./helm" - HELM_CMD+=" -f ./helm/values-${{ inputs.environment }}.yaml" - HELM_CMD+=" --namespace ${{ inputs.namespace }}" - HELM_CMD+=" --set image.tag=${{ inputs.image_tag }}" - HELM_CMD+=" --set config.mode=orchestrator" - HELM_CMD+=" --set fullnameOverride=prl-devops-orchestrator" + HELM_ARGS=( + -f "./helm/values-${{ inputs.environment }}.yaml" + --namespace "$K8S_NAMESPACE" + --set "image.tag=${{ inputs.image_tag }}" + --set config.mode=orchestrator + --set fullnameOverride=prl-devops-orchestrator + ) if [[ "$DRY_RUN" == "true" ]]; then - HELM_CMD+=" --dry-run --debug" + helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" --dry-run --debug else - HELM_CMD+=" --wait --timeout 5m" + DEPLOY_ARGS=(--wait --timeout 5m) if [[ "$ATOMIC" == "true" ]]; then - HELM_CMD+=" --atomic" + DEPLOY_ARGS+=("--atomic") fi + helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" fi - cd "$DEPLOY_DIR" - az aks command invoke \ - --resource-group "$RESOURCE_GROUP" \ - --name "$CLUSTER_NAME" \ - --command "$HELM_CMD" \ - --file . - deploy-catalog: name: Deploy Catalog if: ${{ inputs.deploy_catalog }} - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64] timeout-minutes: 30 permissions: id-token: write contents: read steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: persist-credentials: false @@ -144,6 +150,26 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} + - name: Install dependencies + run: | + which unzip || sudo dnf install -y unzip + + - name: Install kubelogin + uses: azure/use-kubelogin@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + kubelogin-version: 'v0.2.14' + + - name: Set AKS context + uses: azure/aks-set-context@v4 + with: + resource-group: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + cluster-name: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} + + - name: Convert kubeconfig for non-interactive login + run: kubelogin convert-kubeconfig -l azurecli + - name: Setup Helm uses: azure/setup-helm@v4 @@ -158,36 +184,25 @@ jobs: - name: Helm deploy catalog env: - RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} + K8S_NAMESPACE: ${{ inputs.namespace }} DRY_RUN: ${{ inputs.dry_run }} ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail - - # Copy helm chart to isolated dir so --file . uploads only the chart - DEPLOY_DIR=$(mktemp -d) - cp -r ./helm "$DEPLOY_DIR/helm" - - HELM_CMD="helm upgrade --install prl-devops-catalog ./helm" - HELM_CMD+=" -f ./helm/values-${{ inputs.environment }}.yaml" - HELM_CMD+=" --namespace ${{ inputs.namespace }}" - HELM_CMD+=" --set image.tag=${{ inputs.image_tag }}" - HELM_CMD+=" --set config.mode=catalog" - HELM_CMD+=" --set fullnameOverride=prl-devops-catalog" + HELM_ARGS=( + -f "./helm/values-${{ inputs.environment }}.yaml" + --namespace "$K8S_NAMESPACE" + --set "image.tag=${{ inputs.image_tag }}" + --set config.mode=catalog + --set fullnameOverride=prl-devops-catalog + ) if [[ "$DRY_RUN" == "true" ]]; then - HELM_CMD+=" --dry-run --debug" + helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" --dry-run --debug else - HELM_CMD+=" --wait --timeout 5m" + DEPLOY_ARGS=(--wait --timeout 5m) if [[ "$ATOMIC" == "true" ]]; then - HELM_CMD+=" --atomic" + DEPLOY_ARGS+=("--atomic") fi - fi - - cd "$DEPLOY_DIR" - az aks command invoke \ - --resource-group "$RESOURCE_GROUP" \ - --name "$CLUSTER_NAME" \ - --command "$HELM_CMD" \ - --file . + helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" + fi \ No newline at end of file From 2da9d6ba8db771fac2351471c623846d74e99ca6 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Thu, 9 Jul 2026 22:08:08 -0400 Subject: [PATCH 08/34] [ACE-1975] Install kubelogin without sudo using python3 zipfile extraction --- .github/workflows/deploy-aks.yml | 50 ++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index dae7b5c7..67c95f59 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -72,16 +72,23 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} - - name: Install dependencies - run: | - which unzip || sudo dnf install -y unzip - - name: Install kubelogin - uses: azure/use-kubelogin@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - kubelogin-version: 'v0.2.14' + run: | + set -euo pipefail + KUBELOGIN_VERSION="v0.2.14" + # Skip download if already installed at correct version + if kubelogin --version 2>/dev/null | grep -q "$KUBELOGIN_VERSION"; then + echo "kubelogin $KUBELOGIN_VERSION already installed" + exit 0 + fi + curl -sSL "https://github.com/Azure/kubelogin/releases/download/${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip" -o /tmp/kubelogin.zip + # Use python3 zipfile instead of unzip: runner has no unzip and no passwordless sudo to install it + python3 -m zipfile -e /tmp/kubelogin.zip /tmp/kubelogin + chmod +x /tmp/kubelogin/bin/linux_amd64/kubelogin + mkdir -p "$HOME/.local/bin" + cp /tmp/kubelogin/bin/linux_amd64/kubelogin "$HOME/.local/bin/kubelogin" + rm -rf /tmp/kubelogin /tmp/kubelogin.zip + kubelogin --version - name: Set AKS context uses: azure/aks-set-context@v4 @@ -150,16 +157,23 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} - - name: Install dependencies - run: | - which unzip || sudo dnf install -y unzip - - name: Install kubelogin - uses: azure/use-kubelogin@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - kubelogin-version: 'v0.2.14' + run: | + set -euo pipefail + KUBELOGIN_VERSION="v0.2.14" + # Skip download if already installed at correct version + if kubelogin --version 2>/dev/null | grep -q "$KUBELOGIN_VERSION"; then + echo "kubelogin $KUBELOGIN_VERSION already installed" + exit 0 + fi + curl -sSL "https://github.com/Azure/kubelogin/releases/download/${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip" -o /tmp/kubelogin.zip + # Use python3 zipfile instead of unzip: runner has no unzip and no passwordless sudo to install it + python3 -m zipfile -e /tmp/kubelogin.zip /tmp/kubelogin + chmod +x /tmp/kubelogin/bin/linux_amd64/kubelogin + mkdir -p "$HOME/.local/bin" + cp /tmp/kubelogin/bin/linux_amd64/kubelogin "$HOME/.local/bin/kubelogin" + rm -rf /tmp/kubelogin /tmp/kubelogin.zip + kubelogin --version - name: Set AKS context uses: azure/aks-set-context@v4 From 7caa7e36f3b1d78a647c7e3ae4a3ff311d057d6e Mon Sep 17 00:00:00 2001 From: osh0501 Date: Thu, 9 Jul 2026 22:35:21 -0400 Subject: [PATCH 09/34] [ACE-1975] Debug connectivity --- .github/workflows/deploy-aks.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 67c95f59..da34680a 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -72,6 +72,28 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} + - name: Debug connectivity + run: | + echo "=== Runner IP ===" + ip addr show | grep "inet " | grep -v 127.0.0.1 + echo "=== DNS nameservers ===" + cat /etc/resolv.conf || true + echo "=== DNS resolution ===" + nslookup di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io || true + AKS_IP=$(dig +short di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io | tail -1) + echo "Resolved AKS IP: $AKS_IP" + echo "=== Full routing table ===" + ip route show || true + echo "=== Route to AKS API ===" + ip route get 100.64.146.1 || true + echo "=== TCP port 443 test (bash socket) ===" + timeout 5 bash -c "&1 && echo "Port 443 open" || echo "Port 443 blocked/timeout" + echo "=== TLS curl test ===" + curl -vk --connect-timeout 10 https://di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io:443/version 2>&1 || true + echo "=== Environment ===" + echo "KUBECONFIG: ${KUBECONFIG:-not set}" + echo "PATH: $PATH" + - name: Install kubelogin run: | set -euo pipefail @@ -157,6 +179,7 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} + - name: Install kubelogin run: | set -euo pipefail From f45131445fa73d3e24afdd936a738bf9751a80db Mon Sep 17 00:00:00 2001 From: osh0501 Date: Fri, 10 Jul 2026 14:39:04 -0400 Subject: [PATCH 10/34] [ACE-1975] Use az aks command invoke instead of kubelogin --- .github/workflows/deploy-aks.yml | 171 +++++++++++-------------------- 1 file changed, 59 insertions(+), 112 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index da34680a..1674dbb7 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -72,91 +72,54 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} - - name: Debug connectivity - run: | - echo "=== Runner IP ===" - ip addr show | grep "inet " | grep -v 127.0.0.1 - echo "=== DNS nameservers ===" - cat /etc/resolv.conf || true - echo "=== DNS resolution ===" - nslookup di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io || true - AKS_IP=$(dig +short di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io | tail -1) - echo "Resolved AKS IP: $AKS_IP" - echo "=== Full routing table ===" - ip route show || true - echo "=== Route to AKS API ===" - ip route get 100.64.146.1 || true - echo "=== TCP port 443 test (bash socket) ===" - timeout 5 bash -c "&1 && echo "Port 443 open" || echo "Port 443 blocked/timeout" - echo "=== TLS curl test ===" - curl -vk --connect-timeout 10 https://di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io:443/version 2>&1 || true - echo "=== Environment ===" - echo "KUBECONFIG: ${KUBECONFIG:-not set}" - echo "PATH: $PATH" - - - name: Install kubelogin - run: | - set -euo pipefail - KUBELOGIN_VERSION="v0.2.14" - # Skip download if already installed at correct version - if kubelogin --version 2>/dev/null | grep -q "$KUBELOGIN_VERSION"; then - echo "kubelogin $KUBELOGIN_VERSION already installed" - exit 0 - fi - curl -sSL "https://github.com/Azure/kubelogin/releases/download/${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip" -o /tmp/kubelogin.zip - # Use python3 zipfile instead of unzip: runner has no unzip and no passwordless sudo to install it - python3 -m zipfile -e /tmp/kubelogin.zip /tmp/kubelogin - chmod +x /tmp/kubelogin/bin/linux_amd64/kubelogin - mkdir -p "$HOME/.local/bin" - cp /tmp/kubelogin/bin/linux_amd64/kubelogin "$HOME/.local/bin/kubelogin" - rm -rf /tmp/kubelogin /tmp/kubelogin.zip - kubelogin --version - - - name: Set AKS context - uses: azure/aks-set-context@v4 - with: - resource-group: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - cluster-name: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} - - - name: Convert kubeconfig for non-interactive login - run: kubelogin convert-kubeconfig -l azurecli - - name: Setup Helm uses: azure/setup-helm@v4 - name: Helm lint orchestrator + env: + AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} run: | set -euo pipefail - helm lint ./helm \ - -f "./helm/values-${{ inputs.environment }}.yaml" \ - --set config.mode=orchestrator \ - --set fullnameOverride=prl-devops-orchestrator \ - --set image.tag="${{ inputs.image_tag }}" + az aks command invoke \ + -g "$AKS_RESOURCE_GROUP" \ + -n "$AKS_CLUSTER_NAME" \ + -c "helm lint ./helm \ + -f ./helm/values-${{ inputs.environment }}.yaml \ + --set config.mode=orchestrator \ + --set fullnameOverride=prl-devops-orchestrator \ + --set image.tag=${{ inputs.image_tag }}" - name: Helm deploy orchestrator env: + AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} K8S_NAMESPACE: ${{ inputs.namespace }} DRY_RUN: ${{ inputs.dry_run }} ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail - HELM_ARGS=( - -f "./helm/values-${{ inputs.environment }}.yaml" - --namespace "$K8S_NAMESPACE" - --set "image.tag=${{ inputs.image_tag }}" - --set config.mode=orchestrator - --set fullnameOverride=prl-devops-orchestrator - ) - + + HELM_CMD="helm upgrade --install prl-devops-orchestrator ./helm \ + -f ./helm/values-${{ inputs.environment }}.yaml \ + --namespace $K8S_NAMESPACE \ + --set image.tag=${{ inputs.image_tag }} \ + --set config.mode=orchestrator \ + --set fullnameOverride=prl-devops-orchestrator" + if [[ "$DRY_RUN" == "true" ]]; then - helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" --dry-run --debug + HELM_CMD="$HELM_CMD --dry-run --debug" else - DEPLOY_ARGS=(--wait --timeout 5m) + HELM_CMD="$HELM_CMD --wait --timeout 5m" if [[ "$ATOMIC" == "true" ]]; then - DEPLOY_ARGS+=("--atomic") + HELM_CMD="$HELM_CMD --atomic" fi - helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" fi + + az aks command invoke \ + -g "$AKS_RESOURCE_GROUP" \ + -n "$AKS_CLUSTER_NAME" \ + -c "$HELM_CMD" deploy-catalog: name: Deploy Catalog @@ -179,67 +142,51 @@ jobs: tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} - - - name: Install kubelogin - run: | - set -euo pipefail - KUBELOGIN_VERSION="v0.2.14" - # Skip download if already installed at correct version - if kubelogin --version 2>/dev/null | grep -q "$KUBELOGIN_VERSION"; then - echo "kubelogin $KUBELOGIN_VERSION already installed" - exit 0 - fi - curl -sSL "https://github.com/Azure/kubelogin/releases/download/${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip" -o /tmp/kubelogin.zip - # Use python3 zipfile instead of unzip: runner has no unzip and no passwordless sudo to install it - python3 -m zipfile -e /tmp/kubelogin.zip /tmp/kubelogin - chmod +x /tmp/kubelogin/bin/linux_amd64/kubelogin - mkdir -p "$HOME/.local/bin" - cp /tmp/kubelogin/bin/linux_amd64/kubelogin "$HOME/.local/bin/kubelogin" - rm -rf /tmp/kubelogin /tmp/kubelogin.zip - kubelogin --version - - - name: Set AKS context - uses: azure/aks-set-context@v4 - with: - resource-group: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - cluster-name: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} - - - name: Convert kubeconfig for non-interactive login - run: kubelogin convert-kubeconfig -l azurecli - - name: Setup Helm uses: azure/setup-helm@v4 - name: Helm lint catalog + env: + AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} run: | set -euo pipefail - helm lint ./helm \ - -f "./helm/values-${{ inputs.environment }}.yaml" \ - --set config.mode=catalog \ - --set fullnameOverride=prl-devops-catalog \ - --set image.tag="${{ inputs.image_tag }}" + az aks command invoke \ + -g "$AKS_RESOURCE_GROUP" \ + -n "$AKS_CLUSTER_NAME" \ + -c "helm lint ./helm \ + -f ./helm/values-${{ inputs.environment }}.yaml \ + --set config.mode=catalog \ + --set fullnameOverride=prl-devops-catalog \ + --set image.tag=${{ inputs.image_tag }}" - name: Helm deploy catalog env: + AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} K8S_NAMESPACE: ${{ inputs.namespace }} DRY_RUN: ${{ inputs.dry_run }} ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail - HELM_ARGS=( - -f "./helm/values-${{ inputs.environment }}.yaml" - --namespace "$K8S_NAMESPACE" - --set "image.tag=${{ inputs.image_tag }}" - --set config.mode=catalog - --set fullnameOverride=prl-devops-catalog - ) - + + HELM_CMD="helm upgrade --install prl-devops-catalog ./helm \ + -f ./helm/values-${{ inputs.environment }}.yaml \ + --namespace $K8S_NAMESPACE \ + --set image.tag=${{ inputs.image_tag }} \ + --set config.mode=catalog \ + --set fullnameOverride=prl-devops-catalog" + if [[ "$DRY_RUN" == "true" ]]; then - helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" --dry-run --debug + HELM_CMD="$HELM_CMD --dry-run --debug" else - DEPLOY_ARGS=(--wait --timeout 5m) + HELM_CMD="$HELM_CMD --wait --timeout 5m" if [[ "$ATOMIC" == "true" ]]; then - DEPLOY_ARGS+=("--atomic") + HELM_CMD="$HELM_CMD --atomic" fi - helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" - fi \ No newline at end of file + fi + + az aks command invoke \ + -g "$AKS_RESOURCE_GROUP" \ + -n "$AKS_CLUSTER_NAME" \ + -c "$HELM_CMD" \ No newline at end of file From 86f827a3a22be0581dfb947977c176a9cb8bab7f Mon Sep 17 00:00:00 2001 From: osh0501 Date: Fri, 10 Jul 2026 15:25:59 -0400 Subject: [PATCH 11/34] [ACE-1975] Added diagnostic steps to check connectivity issues --- .github/workflows/deploy-aks.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 1674dbb7..09af4146 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -75,6 +75,28 @@ jobs: - name: Setup Helm uses: azure/setup-helm@v4 + - name: Check internal connectivity (via netcat) + run: | + echo "Testing TCP connectivity to private endpoint..." + ENDPOINT="di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io" + if command -v nc &> /dev/null; then + nc -vz "$ENDPOINT" 443 || echo "Netcat test failed" + else + echo "Netcat not available" + fi + + - name: Check internal connectivity (via bash) + run: | + echo "Testing TCP connectivity via bash..." + ENDPOINT="di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io" + timeout 5 bash -c "&1 && echo "Bash TCP test successful!" || echo "Bash TCP test failed" + + - name: Test TLS connectivity to private endpoint + run: | + echo "Testing direct TLS to private endpoint..." + PRIVATE_ENDPOINT="di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io" + curl -vv https://${PRIVATE_ENDPOINT}:443/version 2>&1 | head -50 || echo "TLS connection failed (expected with proxy)" + - name: Helm lint orchestrator env: AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} From 03f12127fc0931eaf0569c52598ff14e70416326 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Sat, 11 Jul 2026 00:08:57 -0400 Subject: [PATCH 12/34] [ACE-1975] add --file . to az aks command invoke for helm chart access --- .github/workflows/deploy-aks.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 09af4146..d19ac67f 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -106,6 +106,7 @@ jobs: az aks command invoke \ -g "$AKS_RESOURCE_GROUP" \ -n "$AKS_CLUSTER_NAME" \ + --file . \ -c "helm lint ./helm \ -f ./helm/values-${{ inputs.environment }}.yaml \ --set config.mode=orchestrator \ @@ -141,6 +142,7 @@ jobs: az aks command invoke \ -g "$AKS_RESOURCE_GROUP" \ -n "$AKS_CLUSTER_NAME" \ + --file . \ -c "$HELM_CMD" deploy-catalog: @@ -176,6 +178,7 @@ jobs: az aks command invoke \ -g "$AKS_RESOURCE_GROUP" \ -n "$AKS_CLUSTER_NAME" \ + --file . \ -c "helm lint ./helm \ -f ./helm/values-${{ inputs.environment }}.yaml \ --set config.mode=catalog \ @@ -211,4 +214,5 @@ jobs: az aks command invoke \ -g "$AKS_RESOURCE_GROUP" \ -n "$AKS_CLUSTER_NAME" \ + --file . \ -c "$HELM_CMD" \ No newline at end of file From a14b8f80080a3813b25c4481a0097b45bc79a929 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Sat, 11 Jul 2026 00:20:51 -0400 Subject: [PATCH 13/34] [ACE-1975] upload helm chart directory to az aks command invoke pod --- .github/workflows/deploy-aks.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index d19ac67f..89bb8594 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -103,12 +103,13 @@ jobs: AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} run: | set -euo pipefail + cd helm az aks command invoke \ -g "$AKS_RESOURCE_GROUP" \ -n "$AKS_CLUSTER_NAME" \ --file . \ - -c "helm lint ./helm \ - -f ./helm/values-${{ inputs.environment }}.yaml \ + -c "helm lint . \ + -f ./values-${{ inputs.environment }}.yaml \ --set config.mode=orchestrator \ --set fullnameOverride=prl-devops-orchestrator \ --set image.tag=${{ inputs.image_tag }}" @@ -122,9 +123,10 @@ jobs: ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail + cd helm - HELM_CMD="helm upgrade --install prl-devops-orchestrator ./helm \ - -f ./helm/values-${{ inputs.environment }}.yaml \ + HELM_CMD="helm upgrade --install prl-devops-orchestrator . \ + -f ./values-${{ inputs.environment }}.yaml \ --namespace $K8S_NAMESPACE \ --set image.tag=${{ inputs.image_tag }} \ --set config.mode=orchestrator \ @@ -175,12 +177,13 @@ jobs: AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} run: | set -euo pipefail + cd helm az aks command invoke \ -g "$AKS_RESOURCE_GROUP" \ -n "$AKS_CLUSTER_NAME" \ --file . \ - -c "helm lint ./helm \ - -f ./helm/values-${{ inputs.environment }}.yaml \ + -c "helm lint . \ + -f ./values-${{ inputs.environment }}.yaml \ --set config.mode=catalog \ --set fullnameOverride=prl-devops-catalog \ --set image.tag=${{ inputs.image_tag }}" @@ -194,9 +197,10 @@ jobs: ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail + cd helm - HELM_CMD="helm upgrade --install prl-devops-catalog ./helm \ - -f ./helm/values-${{ inputs.environment }}.yaml \ + HELM_CMD="helm upgrade --install prl-devops-catalog . \ + -f ./values-${{ inputs.environment }}.yaml \ --namespace $K8S_NAMESPACE \ --set image.tag=${{ inputs.image_tag }} \ --set config.mode=catalog \ From 5ed637e86f9620e6b4835697628a4386ece6a683 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Sat, 11 Jul 2026 23:58:58 -0400 Subject: [PATCH 14/34] [ACE-1975] combine orchestrator and catalog into sequential steps --- .github/workflows/deploy-aks.yml | 56 ++++---------------------------- 1 file changed, 7 insertions(+), 49 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 89bb8594..26684105 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -51,9 +51,9 @@ on: default: false jobs: - deploy-orchestrator: - name: Deploy Orchestrator - if: ${{ inputs.deploy_orchestrator }} + deploy-services: + name: Deploy Orchestrator and/or Catalog + if: ${{ inputs.deploy_orchestrator || inputs.deploy_catalog }} runs-on: [self-hosted, Linux, X64] timeout-minutes: 30 permissions: @@ -75,29 +75,8 @@ jobs: - name: Setup Helm uses: azure/setup-helm@v4 - - name: Check internal connectivity (via netcat) - run: | - echo "Testing TCP connectivity to private endpoint..." - ENDPOINT="di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io" - if command -v nc &> /dev/null; then - nc -vz "$ENDPOINT" 443 || echo "Netcat test failed" - else - echo "Netcat not available" - fi - - - name: Check internal connectivity (via bash) - run: | - echo "Testing TCP connectivity via bash..." - ENDPOINT="di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io" - timeout 5 bash -c "&1 && echo "Bash TCP test successful!" || echo "Bash TCP test failed" - - - name: Test TLS connectivity to private endpoint - run: | - echo "Testing direct TLS to private endpoint..." - PRIVATE_ENDPOINT="di-aks-plt-stg-we-2ietnykd.privatelink.westeurope.azmk8s.io" - curl -vv https://${PRIVATE_ENDPOINT}:443/version 2>&1 | head -50 || echo "TLS connection failed (expected with proxy)" - - name: Helm lint orchestrator + if: ${{ inputs.deploy_orchestrator }} env: AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} @@ -115,6 +94,7 @@ jobs: --set image.tag=${{ inputs.image_tag }}" - name: Helm deploy orchestrator + if: ${{ inputs.deploy_orchestrator }} env: AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} @@ -147,31 +127,8 @@ jobs: --file . \ -c "$HELM_CMD" - deploy-catalog: - name: Deploy Catalog - if: ${{ inputs.deploy_catalog }} - runs-on: [self-hosted, Linux, X64] - timeout-minutes: 30 - permissions: - id-token: write - contents: read - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Azure login (OIDC) - uses: azure/login@v2 - with: - client-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_CLIENT_ID || secrets.PRD_AZURE_CLIENT_ID }} - tenant-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_TENANT_ID || secrets.PRD_AZURE_TENANT_ID }} - subscription-id: ${{ inputs.environment == 'stg' && secrets.STG_AZURE_SUBSCRIPTION_ID || secrets.PRD_AZURE_SUBSCRIPTION_ID }} - - - name: Setup Helm - uses: azure/setup-helm@v4 - - name: Helm lint catalog + if: ${{ inputs.deploy_catalog }} env: AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} @@ -189,6 +146,7 @@ jobs: --set image.tag=${{ inputs.image_tag }}" - name: Helm deploy catalog + if: ${{ inputs.deploy_catalog }} env: AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} From 914598ff3792c937c7389087032888ab937b2110 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Tue, 14 Jul 2026 00:28:37 -0400 Subject: [PATCH 15/34] [ACE-1975] Fixed --wait/--atomic redundancy --- .github/workflows/deploy-aks.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 26684105..70deffab 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -114,11 +114,10 @@ jobs: if [[ "$DRY_RUN" == "true" ]]; then HELM_CMD="$HELM_CMD --dry-run --debug" + elif [[ "$ATOMIC" == "true" ]]; then + HELM_CMD="$HELM_CMD --atomic --timeout 5m" else HELM_CMD="$HELM_CMD --wait --timeout 5m" - if [[ "$ATOMIC" == "true" ]]; then - HELM_CMD="$HELM_CMD --atomic" - fi fi az aks command invoke \ @@ -166,11 +165,10 @@ jobs: if [[ "$DRY_RUN" == "true" ]]; then HELM_CMD="$HELM_CMD --dry-run --debug" + elif [[ "$ATOMIC" == "true" ]]; then + HELM_CMD="$HELM_CMD --atomic --timeout 5m" else HELM_CMD="$HELM_CMD --wait --timeout 5m" - if [[ "$ATOMIC" == "true" ]]; then - HELM_CMD="$HELM_CMD --atomic" - fi fi az aks command invoke \ From 603d81d4eccf461ec210d8863409a83d034eea23 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Tue, 14 Jul 2026 15:51:52 -0400 Subject: [PATCH 16/34] [ACE-1975] Test with ubuntu-latest, mask infra names --- .github/workflows/deploy-aks.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 70deffab..f77ce6fc 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -54,7 +54,7 @@ jobs: deploy-services: name: Deploy Orchestrator and/or Catalog if: ${{ inputs.deploy_orchestrator || inputs.deploy_catalog }} - runs-on: [self-hosted, Linux, X64] + runs-on: ubuntu-latest timeout-minutes: 30 permissions: id-token: write @@ -75,6 +75,11 @@ jobs: - name: Setup Helm uses: azure/setup-helm@v4 + - name: Mask infra names + run: | + echo "::add-mask::${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }}" + echo "::add-mask::${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }}" + - name: Helm lint orchestrator if: ${{ inputs.deploy_orchestrator }} env: From dc60c0e99b1e6e1b4312b3be3c762144eec450e8 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Tue, 14 Jul 2026 16:06:13 -0400 Subject: [PATCH 17/34] [ACE-1975] hide cluster name and rg: use env vars so the values aren't inlined --- .github/workflows/deploy-aks.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index f77ce6fc..8d558cef 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -76,9 +76,12 @@ jobs: uses: azure/setup-helm@v4 - name: Mask infra names + env: + AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} + AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} run: | - echo "::add-mask::${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }}" - echo "::add-mask::${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }}" + echo "::add-mask::$AKS_RESOURCE_GROUP" + echo "::add-mask::$AKS_CLUSTER_NAME" - name: Helm lint orchestrator if: ${{ inputs.deploy_orchestrator }} From 3776e18739c50b08c1c835a33f26e4878741d58e Mon Sep 17 00:00:00 2001 From: osh0501 Date: Tue, 14 Jul 2026 23:04:28 -0400 Subject: [PATCH 18/34] [ACE-1975] wait for final aks command result and default atomic rollback, read rg and cluster name from repo secrets --- .github/workflows/deploy-aks.yml | 138 ++++++++++++++++++++++--------- 1 file changed, 99 insertions(+), 39 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 8d558cef..3ae528c7 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -48,7 +48,7 @@ on: description: Rollback on failure (atomic deployment) required: false type: boolean - default: false + default: true jobs: deploy-services: @@ -59,6 +59,9 @@ jobs: permissions: id-token: write contents: read + env: + AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && secrets.STG_AKS_RESOURCE_GROUP || secrets.PRD_AKS_RESOURCE_GROUP }} + AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && secrets.STG_AKS_CLUSTER_NAME || secrets.PRD_AKS_CLUSTER_NAME }} steps: - name: Checkout repository uses: actions/checkout@v6 @@ -75,37 +78,99 @@ jobs: - name: Setup Helm uses: azure/setup-helm@v4 - - name: Mask infra names - env: - AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} + - name: Prepare AKS command helper run: | - echo "::add-mask::$AKS_RESOURCE_GROUP" - echo "::add-mask::$AKS_CLUSTER_NAME" + cat > "$RUNNER_TEMP/aks-command-helper.sh" <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + + resource_group="$1" + cluster_name="$2" + command="$3" + shift 3 + + invoke_json="$(az aks command invoke \ + -g "$resource_group" \ + -n "$cluster_name" \ + -c "$command" \ + "$@" \ + -o json)" + + command_id="$(printf '%s' "$invoke_json" | jq -r '.id // .commandId // empty')" + if [[ -z "$command_id" ]]; then + echo "$invoke_json" + echo "Unable to determine AKS command id from invoke response." >&2 + exit 1 + fi + + echo "AKS command id: $command_id" + + while true; do + result_json="$(az aks command result \ + -g "$resource_group" \ + -n "$cluster_name" \ + -i "$command_id" \ + -o json)" + + state="$(printf '%s' "$result_json" | jq -r '.provisioningState // .status // empty')" + output="$(printf '%s' "$result_json" | jq -r '.logs // .text // .message // empty')" + exit_code="$(printf '%s' "$result_json" | jq -r 'if .exitCode == null then "" else (.exitCode | tostring) end')" + + case "$state" in + Succeeded) + if [[ -n "$output" ]]; then + printf '%s\n' "$output" + fi + if [[ -n "$exit_code" && "$exit_code" != "0" ]]; then + echo "$result_json" + exit "$exit_code" + fi + exit 0 + ;; + Failed|Canceled) + echo "$result_json" + if [[ -n "$output" ]]; then + printf '%s\n' "$output" + fi + if [[ -n "$exit_code" ]]; then + exit "$exit_code" + fi + exit 1 + ;; + '' ) + echo "$result_json" + echo "AKS command result did not include a provisioning state." >&2 + exit 1 + ;; + *) + echo "AKS command status: $state" + sleep 10 + ;; + esac + done + EOF + + chmod +x "$RUNNER_TEMP/aks-command-helper.sh" + echo "AKS_COMMAND_HELPER=$RUNNER_TEMP/aks-command-helper.sh" >> "$GITHUB_ENV" - name: Helm lint orchestrator if: ${{ inputs.deploy_orchestrator }} - env: - AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} run: | set -euo pipefail cd helm - az aks command invoke \ - -g "$AKS_RESOURCE_GROUP" \ - -n "$AKS_CLUSTER_NAME" \ - --file . \ - -c "helm lint . \ + "$AKS_COMMAND_HELPER" \ + "$AKS_RESOURCE_GROUP" \ + "$AKS_CLUSTER_NAME" \ + "helm lint . \ -f ./values-${{ inputs.environment }}.yaml \ --set config.mode=orchestrator \ --set fullnameOverride=prl-devops-orchestrator \ - --set image.tag=${{ inputs.image_tag }}" + --set image.tag=${{ inputs.image_tag }}" \ + --file . - name: Helm deploy orchestrator if: ${{ inputs.deploy_orchestrator }} env: - AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} K8S_NAMESPACE: ${{ inputs.namespace }} DRY_RUN: ${{ inputs.dry_run }} ATOMIC: ${{ inputs.atomic }} @@ -128,35 +193,30 @@ jobs: HELM_CMD="$HELM_CMD --wait --timeout 5m" fi - az aks command invoke \ - -g "$AKS_RESOURCE_GROUP" \ - -n "$AKS_CLUSTER_NAME" \ - --file . \ - -c "$HELM_CMD" + "$AKS_COMMAND_HELPER" \ + "$AKS_RESOURCE_GROUP" \ + "$AKS_CLUSTER_NAME" \ + "$HELM_CMD" \ + --file . - name: Helm lint catalog if: ${{ inputs.deploy_catalog }} - env: - AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} run: | set -euo pipefail cd helm - az aks command invoke \ - -g "$AKS_RESOURCE_GROUP" \ - -n "$AKS_CLUSTER_NAME" \ - --file . \ - -c "helm lint . \ + "$AKS_COMMAND_HELPER" \ + "$AKS_RESOURCE_GROUP" \ + "$AKS_CLUSTER_NAME" \ + "helm lint . \ -f ./values-${{ inputs.environment }}.yaml \ --set config.mode=catalog \ --set fullnameOverride=prl-devops-catalog \ - --set image.tag=${{ inputs.image_tag }}" + --set image.tag=${{ inputs.image_tag }}" \ + --file . - name: Helm deploy catalog if: ${{ inputs.deploy_catalog }} env: - AKS_RESOURCE_GROUP: ${{ inputs.environment == 'stg' && vars.STG_AKS_RESOURCE_GROUP || vars.PRD_AKS_RESOURCE_GROUP }} - AKS_CLUSTER_NAME: ${{ inputs.environment == 'stg' && vars.STG_AKS_CLUSTER_NAME || vars.PRD_AKS_CLUSTER_NAME }} K8S_NAMESPACE: ${{ inputs.namespace }} DRY_RUN: ${{ inputs.dry_run }} ATOMIC: ${{ inputs.atomic }} @@ -179,8 +239,8 @@ jobs: HELM_CMD="$HELM_CMD --wait --timeout 5m" fi - az aks command invoke \ - -g "$AKS_RESOURCE_GROUP" \ - -n "$AKS_CLUSTER_NAME" \ - --file . \ - -c "$HELM_CMD" \ No newline at end of file + "$AKS_COMMAND_HELPER" \ + "$AKS_RESOURCE_GROUP" \ + "$AKS_CLUSTER_NAME" \ + "$HELM_CMD" \ + --file . \ No newline at end of file From bf749d542ba30f2511adbbf2a8a3e9d257ac87dd Mon Sep 17 00:00:00 2001 From: osh0501 Date: Tue, 14 Jul 2026 23:28:56 -0400 Subject: [PATCH 19/34] [ACE-1975] handle non-json running status from aks command result --- .github/workflows/deploy-aks.yml | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 3ae528c7..c3011240 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -106,15 +106,30 @@ jobs: echo "AKS command id: $command_id" while true; do - result_json="$(az aks command result \ + if ! result_raw="$(az aks command result \ -g "$resource_group" \ -n "$cluster_name" \ -i "$command_id" \ - -o json)" + -o json 2>&1)"; then + echo "$result_raw" + exit 1 + fi - state="$(printf '%s' "$result_json" | jq -r '.provisioningState // .status // empty')" - output="$(printf '%s' "$result_json" | jq -r '.logs // .text // .message // empty')" - exit_code="$(printf '%s' "$result_json" | jq -r 'if .exitCode == null then "" else (.exitCode | tostring) end')" + if ! printf '%s' "$result_raw" | jq -e . >/dev/null 2>&1; then + if printf '%s' "$result_raw" | grep -q ' is in Running state'; then + echo "$result_raw" + sleep 10 + continue + fi + + echo "$result_raw" + echo "AKS command result was not valid JSON." >&2 + exit 1 + fi + + state="$(printf '%s' "$result_raw" | jq -r '.provisioningState // .status // empty')" + output="$(printf '%s' "$result_raw" | jq -r '.logs // .text // .message // empty')" + exit_code="$(printf '%s' "$result_raw" | jq -r 'if .exitCode == null then "" else (.exitCode | tostring) end')" case "$state" in Succeeded) @@ -122,13 +137,13 @@ jobs: printf '%s\n' "$output" fi if [[ -n "$exit_code" && "$exit_code" != "0" ]]; then - echo "$result_json" + echo "$result_raw" exit "$exit_code" fi exit 0 ;; Failed|Canceled) - echo "$result_json" + echo "$result_raw" if [[ -n "$output" ]]; then printf '%s\n' "$output" fi @@ -138,7 +153,7 @@ jobs: exit 1 ;; '' ) - echo "$result_json" + echo "$result_raw" echo "AKS command result did not include a provisioning state." >&2 exit 1 ;; From fbc776efaf5e3f526f14a693fb972dcb41e6071d Mon Sep 17 00:00:00 2001 From: osh0501 Date: Tue, 14 Jul 2026 23:45:06 -0400 Subject: [PATCH 20/34] [ACE-1975] update helper to accept both Azure progress text formats --- .github/workflows/deploy-aks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index c3011240..9b2db097 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -116,7 +116,7 @@ jobs: fi if ! printf '%s' "$result_raw" | jq -e . >/dev/null 2>&1; then - if printf '%s' "$result_raw" | grep -q ' is in Running state'; then + if printf '%s' "$result_raw" | grep -Eq ' is in Running state|status: (Running|Pending|InProgress|Queued)'; then echo "$result_raw" sleep 10 continue From 708f7d3a74e7401e56f9be08b2f28b57513305e4 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 15 Jul 2026 00:11:55 -0400 Subject: [PATCH 21/34] [ACE-1975] lowering CPU limits per pod in both std & prd values so rolling upgrades can run old and new pods concurrently --- helm/values-prd.yaml | 2 +- helm/values-stg.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/values-prd.yaml b/helm/values-prd.yaml index f6fc8bac..ad23bb5b 100644 --- a/helm/values-prd.yaml +++ b/helm/values-prd.yaml @@ -118,7 +118,7 @@ resources: cpu: 200m memory: 512Mi limits: - cpu: "2" + cpu: 500m memory: 2Gi autoscaling: diff --git a/helm/values-stg.yaml b/helm/values-stg.yaml index 82387f43..7cef76d3 100644 --- a/helm/values-stg.yaml +++ b/helm/values-stg.yaml @@ -118,7 +118,7 @@ resources: cpu: 100m memory: 256Mi limits: - cpu: "1" + cpu: 500m memory: 1Gi autoscaling: From 06b2e1b18185e922e10ba0dd641ff9e07318631e Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 15 Jul 2026 01:15:29 -0400 Subject: [PATCH 22/34] [ACE-1975] lowering CPU limits from 500 to 200m --- helm/values-prd.yaml | 6 +++--- helm/values-stg.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/values-prd.yaml b/helm/values-prd.yaml index ad23bb5b..32d41a17 100644 --- a/helm/values-prd.yaml +++ b/helm/values-prd.yaml @@ -115,11 +115,11 @@ apiPrefix: /api resources: requests: + cpu: 100m + memory: 256Mi + limits: cpu: 200m memory: 512Mi - limits: - cpu: 500m - memory: 2Gi autoscaling: enabled: false diff --git a/helm/values-stg.yaml b/helm/values-stg.yaml index 7cef76d3..e212689e 100644 --- a/helm/values-stg.yaml +++ b/helm/values-stg.yaml @@ -118,8 +118,8 @@ resources: cpu: 100m memory: 256Mi limits: - cpu: 500m - memory: 1Gi + cpu: 200m + memory: 512Mi autoscaling: enabled: false From e73fef7d3be82ec9f53ed6803734402b91cf8ed4 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 15 Jul 2026 09:05:56 -0400 Subject: [PATCH 23/34] [ACE-1975] mask non-secret identifiers in the wirkflow, update simplify helper --- .github/workflows/deploy-aks.yml | 70 ++++++++++++++------------------ 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 9b2db097..b7ef9b7c 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -3,8 +3,7 @@ name: Deploy to AKS # DEPLOYMENT STRATEGY: # - Workaround reason: GitHub Environments are not available on this GitHub plan, # so we cannot use environment-based OIDC subjects. -# - Therefore deployments are executed under the trusted main ref subject -# (repo:Parallels/prl-devops-service:ref:refs/heads/main). +# - Therefore deployments are executed under a trusted branch-ref based OIDC subject. # - When dispatched from main: both Helm chart AND app image (image_tag) are deployed # - When dispatched from feature branch: only app image is deployed; chart always comes from main # (This is because chart source is determined by the checkout ref, while image_tag is independent input) @@ -68,6 +67,16 @@ jobs: with: persist-credentials: false + - name: Mask non-secret identifiers + env: + OIDC_SUBJECT: repo:${{ github.repository }}:ref:${{ github.ref }} + run: | + echo "::add-mask::$OIDC_SUBJECT" + echo "::add-mask::api://AzureADTokenExchange" + echo "::add-mask::prl-devops-services" + echo "::add-mask::prl-devops-orchestrator" + echo "::add-mask::prl-devops-catalog" + - name: Azure login (OIDC) uses: azure/login@v2 with: @@ -94,71 +103,52 @@ jobs: -n "$cluster_name" \ -c "$command" \ "$@" \ + --only-show-errors \ -o json)" command_id="$(printf '%s' "$invoke_json" | jq -r '.id // .commandId // empty')" if [[ -z "$command_id" ]]; then - echo "$invoke_json" echo "Unable to determine AKS command id from invoke response." >&2 exit 1 fi - echo "AKS command id: $command_id" - while true; do - if ! result_raw="$(az aks command result \ + result_raw="$(az aks command result \ -g "$resource_group" \ -n "$cluster_name" \ -i "$command_id" \ - -o json 2>&1)"; then - echo "$result_raw" - exit 1 - fi + --only-show-errors \ + -o json 2>&1 || true)" if ! printf '%s' "$result_raw" | jq -e . >/dev/null 2>&1; then if printf '%s' "$result_raw" | grep -Eq ' is in Running state|status: (Running|Pending|InProgress|Queued)'; then - echo "$result_raw" + echo "AKS command status: Running" sleep 10 continue fi - - echo "$result_raw" - echo "AKS command result was not valid JSON." >&2 + echo "AKS command status check failed." >&2 exit 1 fi - state="$(printf '%s' "$result_raw" | jq -r '.provisioningState // .status // empty')" - output="$(printf '%s' "$result_raw" | jq -r '.logs // .text // .message // empty')" + state="$(printf '%s' "$result_raw" | jq -r '.provisioningState // .status // ""')" exit_code="$(printf '%s' "$result_raw" | jq -r 'if .exitCode == null then "" else (.exitCode | tostring) end')" + output="$(printf '%s' "$result_raw" | jq -r '.logs // .text // .message // ""')" + + if [[ -n "$output" ]]; then + printf '%s\n' "$output" + fi case "$state" in Succeeded) - if [[ -n "$output" ]]; then - printf '%s\n' "$output" - fi - if [[ -n "$exit_code" && "$exit_code" != "0" ]]; then - echo "$result_raw" - exit "$exit_code" - fi - exit 0 + [[ -z "$exit_code" || "$exit_code" == "0" ]] && exit 0 + exit "$exit_code" ;; Failed|Canceled) - echo "$result_raw" - if [[ -n "$output" ]]; then - printf '%s\n' "$output" - fi - if [[ -n "$exit_code" ]]; then - exit "$exit_code" - fi - exit 1 - ;; - '' ) - echo "$result_raw" - echo "AKS command result did not include a provisioning state." >&2 + [[ -n "$exit_code" ]] && exit "$exit_code" exit 1 ;; *) - echo "AKS command status: $state" + echo "AKS command status: ${state:-Running}" sleep 10 ;; esac @@ -198,7 +188,8 @@ jobs: --namespace $K8S_NAMESPACE \ --set image.tag=${{ inputs.image_tag }} \ --set config.mode=orchestrator \ - --set fullnameOverride=prl-devops-orchestrator" + --set fullnameOverride=prl-devops-orchestrator \ + --hide-notes" if [[ "$DRY_RUN" == "true" ]]; then HELM_CMD="$HELM_CMD --dry-run --debug" @@ -244,7 +235,8 @@ jobs: --namespace $K8S_NAMESPACE \ --set image.tag=${{ inputs.image_tag }} \ --set config.mode=catalog \ - --set fullnameOverride=prl-devops-catalog" + --set fullnameOverride=prl-devops-catalog \ + --hide-notes" if [[ "$DRY_RUN" == "true" ]]; then HELM_CMD="$HELM_CMD --dry-run --debug" From f1f668a84bb581ab547e27f5bf09de68f4a9f427 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 15 Jul 2026 09:26:45 -0400 Subject: [PATCH 24/34] [ACE-1975] Mask runtime identifiers --- .github/workflows/deploy-aks.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index b7ef9b7c..db673af9 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -67,15 +67,11 @@ jobs: with: persist-credentials: false - - name: Mask non-secret identifiers - env: - OIDC_SUBJECT: repo:${{ github.repository }}:ref:${{ github.ref }} + - name: Mask runtime identifiers run: | - echo "::add-mask::$OIDC_SUBJECT" + oidc_subject="repo:${GITHUB_REPOSITORY}:ref:${GITHUB_REF}" + echo "::add-mask::$oidc_subject" echo "::add-mask::api://AzureADTokenExchange" - echo "::add-mask::prl-devops-services" - echo "::add-mask::prl-devops-orchestrator" - echo "::add-mask::prl-devops-catalog" - name: Azure login (OIDC) uses: azure/login@v2 From a686758c687e590b5a2ae4b289c2b1bf572f4d3d Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 15 Jul 2026 15:51:39 -0400 Subject: [PATCH 25/34] [ACE-1975] Testing: switch back to self hosted runners, kubelogin, run networks checks --- .github/workflows/deploy-aks.yml | 190 ++++++++++++++----------------- 1 file changed, 86 insertions(+), 104 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index db673af9..d153e078 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -53,7 +53,7 @@ jobs: deploy-services: name: Deploy Orchestrator and/or Catalog if: ${{ inputs.deploy_orchestrator || inputs.deploy_catalog }} - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64] timeout-minutes: 30 permissions: id-token: write @@ -83,91 +83,88 @@ jobs: - name: Setup Helm uses: azure/setup-helm@v4 - - name: Prepare AKS command helper + - name: Setup kubectl + uses: azure/setup-kubectl@v4 + + - name: Network checks to AKS private API run: | - cat > "$RUNNER_TEMP/aks-command-helper.sh" <<'EOF' - #!/usr/bin/env bash set -euo pipefail - resource_group="$1" - cluster_name="$2" - command="$3" - shift 3 - - invoke_json="$(az aks command invoke \ - -g "$resource_group" \ - -n "$cluster_name" \ - -c "$command" \ - "$@" \ - --only-show-errors \ - -o json)" - - command_id="$(printf '%s' "$invoke_json" | jq -r '.id // .commandId // empty')" - if [[ -z "$command_id" ]]; then - echo "Unable to determine AKS command id from invoke response." >&2 + runner_source_ip="$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src"){print $(i+1); exit}}')" + if [[ -n "$runner_source_ip" ]]; then + echo "Runner local source IP: $runner_source_ip" + else + echo "Runner local source IP could not be determined" + fi + + AKS_API_FQDN="$(az aks show \ + -g "$AKS_RESOURCE_GROUP" \ + -n "$AKS_CLUSTER_NAME" \ + --query privateFqdn \ + -o tsv)" + + if [[ -z "$AKS_API_FQDN" || "$AKS_API_FQDN" == "null" ]]; then + echo "Unable to read privateFqdn from AKS metadata" >&2 + exit 1 + fi + + echo "AKS private API FQDN: $AKS_API_FQDN" + + echo "DNS lookup via getent" + getent hosts "$AKS_API_FQDN" || true + + if command -v nslookup >/dev/null 2>&1; then + echo "DNS lookup via nslookup" + nslookup "$AKS_API_FQDN" || true + fi + + AKS_API_IP="$(getent ahostsv4 "$AKS_API_FQDN" | awk 'NR==1{print $1}')" + if [[ -z "$AKS_API_IP" ]]; then + echo "Failed to resolve AKS private API IP for $AKS_API_FQDN" >&2 exit 1 fi - while true; do - result_raw="$(az aks command result \ - -g "$resource_group" \ - -n "$cluster_name" \ - -i "$command_id" \ - --only-show-errors \ - -o json 2>&1 || true)" - - if ! printf '%s' "$result_raw" | jq -e . >/dev/null 2>&1; then - if printf '%s' "$result_raw" | grep -Eq ' is in Running state|status: (Running|Pending|InProgress|Queued)'; then - echo "AKS command status: Running" - sleep 10 - continue - fi - echo "AKS command status check failed." >&2 - exit 1 - fi - - state="$(printf '%s' "$result_raw" | jq -r '.provisioningState // .status // ""')" - exit_code="$(printf '%s' "$result_raw" | jq -r 'if .exitCode == null then "" else (.exitCode | tostring) end')" - output="$(printf '%s' "$result_raw" | jq -r '.logs // .text // .message // ""')" - - if [[ -n "$output" ]]; then - printf '%s\n' "$output" - fi - - case "$state" in - Succeeded) - [[ -z "$exit_code" || "$exit_code" == "0" ]] && exit 0 - exit "$exit_code" - ;; - Failed|Canceled) - [[ -n "$exit_code" ]] && exit "$exit_code" - exit 1 - ;; - *) - echo "AKS command status: ${state:-Running}" - sleep 10 - ;; - esac - done - EOF - - chmod +x "$RUNNER_TEMP/aks-command-helper.sh" - echo "AKS_COMMAND_HELPER=$RUNNER_TEMP/aks-command-helper.sh" >> "$GITHUB_ENV" + echo "AKS private API resolved IP: $AKS_API_IP" + + echo "TCP check to FQDN:443" + timeout 15 bash -c "exec 3<>/dev/tcp/$AKS_API_FQDN/443" + + echo "TCP check to resolved IP:443" + timeout 15 bash -c "exec 3<>/dev/tcp/$AKS_API_IP/443" + + - name: Install kubelogin + run: | + set -euo pipefail + az aks install-cli --kubelogin + + - name: Get AKS credentials + run: | + set -euo pipefail + az aks get-credentials \ + -g "$AKS_RESOURCE_GROUP" \ + -n "$AKS_CLUSTER_NAME" \ + --overwrite-existing + + - name: Convert kubeconfig for kubelogin + run: | + set -euo pipefail + kubelogin convert-kubeconfig -l azurecli + + - name: Validate Kubernetes API connectivity + run: | + set -euo pipefail + kubectl version --short + kubectl cluster-info - name: Helm lint orchestrator if: ${{ inputs.deploy_orchestrator }} run: | set -euo pipefail - cd helm - "$AKS_COMMAND_HELPER" \ - "$AKS_RESOURCE_GROUP" \ - "$AKS_CLUSTER_NAME" \ - "helm lint . \ - -f ./values-${{ inputs.environment }}.yaml \ - --set config.mode=orchestrator \ - --set fullnameOverride=prl-devops-orchestrator \ - --set image.tag=${{ inputs.image_tag }}" \ - --file . + helm lint helm \ + -f helm/values-${{ inputs.environment }}.yaml \ + --set config.mode=orchestrator \ + --set fullnameOverride=prl-devops-orchestrator \ + --set image.tag=${{ inputs.image_tag }} - name: Helm deploy orchestrator if: ${{ inputs.deploy_orchestrator }} @@ -177,10 +174,9 @@ jobs: ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail - cd helm - HELM_CMD="helm upgrade --install prl-devops-orchestrator . \ - -f ./values-${{ inputs.environment }}.yaml \ + HELM_CMD="helm upgrade --install prl-devops-orchestrator helm \ + -f helm/values-${{ inputs.environment }}.yaml \ --namespace $K8S_NAMESPACE \ --set image.tag=${{ inputs.image_tag }} \ --set config.mode=orchestrator \ @@ -194,27 +190,18 @@ jobs: else HELM_CMD="$HELM_CMD --wait --timeout 5m" fi - - "$AKS_COMMAND_HELPER" \ - "$AKS_RESOURCE_GROUP" \ - "$AKS_CLUSTER_NAME" \ - "$HELM_CMD" \ - --file . + + $HELM_CMD - name: Helm lint catalog if: ${{ inputs.deploy_catalog }} run: | set -euo pipefail - cd helm - "$AKS_COMMAND_HELPER" \ - "$AKS_RESOURCE_GROUP" \ - "$AKS_CLUSTER_NAME" \ - "helm lint . \ - -f ./values-${{ inputs.environment }}.yaml \ - --set config.mode=catalog \ - --set fullnameOverride=prl-devops-catalog \ - --set image.tag=${{ inputs.image_tag }}" \ - --file . + helm lint helm \ + -f helm/values-${{ inputs.environment }}.yaml \ + --set config.mode=catalog \ + --set fullnameOverride=prl-devops-catalog \ + --set image.tag=${{ inputs.image_tag }} - name: Helm deploy catalog if: ${{ inputs.deploy_catalog }} @@ -224,10 +211,9 @@ jobs: ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail - cd helm - - HELM_CMD="helm upgrade --install prl-devops-catalog . \ - -f ./values-${{ inputs.environment }}.yaml \ + + HELM_CMD="helm upgrade --install prl-devops-catalog helm \ + -f helm/values-${{ inputs.environment }}.yaml \ --namespace $K8S_NAMESPACE \ --set image.tag=${{ inputs.image_tag }} \ --set config.mode=catalog \ @@ -241,9 +227,5 @@ jobs: else HELM_CMD="$HELM_CMD --wait --timeout 5m" fi - - "$AKS_COMMAND_HELPER" \ - "$AKS_RESOURCE_GROUP" \ - "$AKS_CLUSTER_NAME" \ - "$HELM_CMD" \ - --file . \ No newline at end of file + + $HELM_CMD \ No newline at end of file From fd361420e05311375229489e39c22ecb8ee5cb4d Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 15 Jul 2026 16:18:10 -0400 Subject: [PATCH 26/34] [ACE-1975] Update install kubelogin step --- .github/workflows/deploy-aks.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index d153e078..63b0c8ea 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -133,9 +133,11 @@ jobs: timeout 15 bash -c "exec 3<>/dev/tcp/$AKS_API_IP/443" - name: Install kubelogin - run: | - set -euo pipefail - az aks install-cli --kubelogin + uses: azure/use-kubelogin@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + kubelogin-version: 'v0.2.14' - name: Get AKS credentials run: | From f7070d2fb2cd8e1ff87c6ff3fe8c9ce8b3cff4f0 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 15 Jul 2026 16:31:38 -0400 Subject: [PATCH 27/34] [ACE-1975] Install kubelogin using python3 zipfile extraction --- .github/workflows/deploy-aks.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 63b0c8ea..c9e6ffbb 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -133,11 +133,21 @@ jobs: timeout 15 bash -c "exec 3<>/dev/tcp/$AKS_API_IP/443" - name: Install kubelogin - uses: azure/use-kubelogin@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - kubelogin-version: 'v0.2.14' + run: | + set -euo pipefail + KUBELOGIN_VERSION="v0.2.14" + # Skip download if already installed at correct version + if kubelogin --version 2>/dev/null | grep -q "$KUBELOGIN_VERSION"; then + echo "kubelogin $KUBELOGIN_VERSION already installed" + exit 0 + fi + curl -sSL "https://github.com/Azure/kubelogin/releases/download/${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip" -o /tmp/kubelogin.zip + # Use python3 zipfile instead of unzip: runner has no unzip and no passwordless sudo to install it + python3 -m zipfile -e /tmp/kubelogin.zip /tmp/kubelogin + chmod +x /tmp/kubelogin/bin/linux_amd64/kubelogin + mkdir -p "$HOME/.local/bin" + cp /tmp/kubelogin/bin/linux_amd64/kubelogin "$HOME/.local/bin/kubelogin" + rm -rf /tmp/kubelogin /tmp/kubelogin.zip - name: Get AKS credentials run: | From cfadbb570750332f4eed1e9c1c837c8121fd17d6 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 15 Jul 2026 16:43:43 -0400 Subject: [PATCH 28/34] [ACE-1975] Remove Validate Kubernetes API connectivity step --- .github/workflows/deploy-aks.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index c9e6ffbb..af3863f6 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -162,12 +162,6 @@ jobs: set -euo pipefail kubelogin convert-kubeconfig -l azurecli - - name: Validate Kubernetes API connectivity - run: | - set -euo pipefail - kubectl version --short - kubectl cluster-info - - name: Helm lint orchestrator if: ${{ inputs.deploy_orchestrator }} run: | From ec25905ad3bd3bd5d4d4dd18b9e630738ae3aba7 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Wed, 15 Jul 2026 23:53:00 -0400 Subject: [PATCH 29/34] [ACE-1975] Remove network check --- .github/workflows/deploy-aks.yml | 46 -------------------------------- 1 file changed, 46 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index af3863f6..c44f17ea 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -86,52 +86,6 @@ jobs: - name: Setup kubectl uses: azure/setup-kubectl@v4 - - name: Network checks to AKS private API - run: | - set -euo pipefail - - runner_source_ip="$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src"){print $(i+1); exit}}')" - if [[ -n "$runner_source_ip" ]]; then - echo "Runner local source IP: $runner_source_ip" - else - echo "Runner local source IP could not be determined" - fi - - AKS_API_FQDN="$(az aks show \ - -g "$AKS_RESOURCE_GROUP" \ - -n "$AKS_CLUSTER_NAME" \ - --query privateFqdn \ - -o tsv)" - - if [[ -z "$AKS_API_FQDN" || "$AKS_API_FQDN" == "null" ]]; then - echo "Unable to read privateFqdn from AKS metadata" >&2 - exit 1 - fi - - echo "AKS private API FQDN: $AKS_API_FQDN" - - echo "DNS lookup via getent" - getent hosts "$AKS_API_FQDN" || true - - if command -v nslookup >/dev/null 2>&1; then - echo "DNS lookup via nslookup" - nslookup "$AKS_API_FQDN" || true - fi - - AKS_API_IP="$(getent ahostsv4 "$AKS_API_FQDN" | awk 'NR==1{print $1}')" - if [[ -z "$AKS_API_IP" ]]; then - echo "Failed to resolve AKS private API IP for $AKS_API_FQDN" >&2 - exit 1 - fi - - echo "AKS private API resolved IP: $AKS_API_IP" - - echo "TCP check to FQDN:443" - timeout 15 bash -c "exec 3<>/dev/tcp/$AKS_API_FQDN/443" - - echo "TCP check to resolved IP:443" - timeout 15 bash -c "exec 3<>/dev/tcp/$AKS_API_IP/443" - - name: Install kubelogin run: | set -euo pipefail From 1049604133e55a9fa8aa46d432aaedaa87d30469 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Thu, 16 Jul 2026 00:16:56 -0400 Subject: [PATCH 30/34] [ACE-1975] Workflow refactoring --- .github/workflows/deploy-aks.yml | 66 ++++++++++++++++---------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index c44f17ea..2e99a6c7 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -103,7 +103,7 @@ jobs: cp /tmp/kubelogin/bin/linux_amd64/kubelogin "$HOME/.local/bin/kubelogin" rm -rf /tmp/kubelogin /tmp/kubelogin.zip - - name: Get AKS credentials + - name: Set AKS context run: | set -euo pipefail az aks get-credentials \ @@ -111,10 +111,8 @@ jobs: -n "$AKS_CLUSTER_NAME" \ --overwrite-existing - - name: Convert kubeconfig for kubelogin - run: | - set -euo pipefail - kubelogin convert-kubeconfig -l azurecli + - name: Convert kubeconfig for non-interactive login + run: kubelogin convert-kubeconfig -l azurecli - name: Helm lint orchestrator if: ${{ inputs.deploy_orchestrator }} @@ -134,25 +132,26 @@ jobs: ATOMIC: ${{ inputs.atomic }} run: | set -euo pipefail - - HELM_CMD="helm upgrade --install prl-devops-orchestrator helm \ - -f helm/values-${{ inputs.environment }}.yaml \ - --namespace $K8S_NAMESPACE \ - --set image.tag=${{ inputs.image_tag }} \ - --set config.mode=orchestrator \ - --set fullnameOverride=prl-devops-orchestrator \ - --hide-notes" - + + HELM_ARGS=( + -f "helm/values-${{ inputs.environment }}.yaml" + --namespace "$K8S_NAMESPACE" + --set "image.tag=${{ inputs.image_tag }}" + --set config.mode=orchestrator + --set fullnameOverride=prl-devops-orchestrator + --hide-notes + ) + if [[ "$DRY_RUN" == "true" ]]; then - HELM_CMD="$HELM_CMD --dry-run --debug" + helm upgrade --install prl-devops-orchestrator helm "${HELM_ARGS[@]}" --dry-run --debug elif [[ "$ATOMIC" == "true" ]]; then - HELM_CMD="$HELM_CMD --atomic --timeout 5m" + DEPLOY_ARGS=(--atomic --timeout 5m) + helm upgrade --install prl-devops-orchestrator helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" else - HELM_CMD="$HELM_CMD --wait --timeout 5m" + DEPLOY_ARGS=(--wait --timeout 5m) + helm upgrade --install prl-devops-orchestrator helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" fi - $HELM_CMD - - name: Helm lint catalog if: ${{ inputs.deploy_catalog }} run: | @@ -172,20 +171,21 @@ jobs: run: | set -euo pipefail - HELM_CMD="helm upgrade --install prl-devops-catalog helm \ - -f helm/values-${{ inputs.environment }}.yaml \ - --namespace $K8S_NAMESPACE \ - --set image.tag=${{ inputs.image_tag }} \ - --set config.mode=catalog \ - --set fullnameOverride=prl-devops-catalog \ - --hide-notes" - + HELM_ARGS=( + -f "helm/values-${{ inputs.environment }}.yaml" + --namespace "$K8S_NAMESPACE" + --set "image.tag=${{ inputs.image_tag }}" + --set config.mode=catalog + --set fullnameOverride=prl-devops-catalog + --hide-notes + ) + if [[ "$DRY_RUN" == "true" ]]; then - HELM_CMD="$HELM_CMD --dry-run --debug" + helm upgrade --install prl-devops-catalog helm "${HELM_ARGS[@]}" --dry-run --debug elif [[ "$ATOMIC" == "true" ]]; then - HELM_CMD="$HELM_CMD --atomic --timeout 5m" + DEPLOY_ARGS=(--atomic --timeout 5m) + helm upgrade --install prl-devops-catalog helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" else - HELM_CMD="$HELM_CMD --wait --timeout 5m" - fi - - $HELM_CMD \ No newline at end of file + DEPLOY_ARGS=(--wait --timeout 5m) + helm upgrade --install prl-devops-catalog helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" + fi \ No newline at end of file From 64b2aefbcd9cbeb2a73dd0e2e94a8eca5993b316 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Thu, 16 Jul 2026 00:32:36 -0400 Subject: [PATCH 31/34] [ACE-1975] Workflow refactoring --- .github/workflows/deploy-aks.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 2e99a6c7..8c20cb4c 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -118,11 +118,11 @@ jobs: if: ${{ inputs.deploy_orchestrator }} run: | set -euo pipefail - helm lint helm \ - -f helm/values-${{ inputs.environment }}.yaml \ + helm lint ./helm \ + -f "./helm/values-${{ inputs.environment }}.yaml" \ --set config.mode=orchestrator \ --set fullnameOverride=prl-devops-orchestrator \ - --set image.tag=${{ inputs.image_tag }} + --set image.tag="${{ inputs.image_tag }}" - name: Helm deploy orchestrator if: ${{ inputs.deploy_orchestrator }} @@ -134,7 +134,7 @@ jobs: set -euo pipefail HELM_ARGS=( - -f "helm/values-${{ inputs.environment }}.yaml" + -f "./helm/values-${{ inputs.environment }}.yaml" --namespace "$K8S_NAMESPACE" --set "image.tag=${{ inputs.image_tag }}" --set config.mode=orchestrator @@ -143,24 +143,24 @@ jobs: ) if [[ "$DRY_RUN" == "true" ]]; then - helm upgrade --install prl-devops-orchestrator helm "${HELM_ARGS[@]}" --dry-run --debug + helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" --dry-run --debug elif [[ "$ATOMIC" == "true" ]]; then DEPLOY_ARGS=(--atomic --timeout 5m) - helm upgrade --install prl-devops-orchestrator helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" + helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" else DEPLOY_ARGS=(--wait --timeout 5m) - helm upgrade --install prl-devops-orchestrator helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" + helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" fi - name: Helm lint catalog if: ${{ inputs.deploy_catalog }} run: | set -euo pipefail - helm lint helm \ - -f helm/values-${{ inputs.environment }}.yaml \ + helm lint ./helm \ + -f "./helm/values-${{ inputs.environment }}.yaml" \ --set config.mode=catalog \ --set fullnameOverride=prl-devops-catalog \ - --set image.tag=${{ inputs.image_tag }} + --set image.tag="${{ inputs.image_tag }}" - name: Helm deploy catalog if: ${{ inputs.deploy_catalog }} @@ -172,7 +172,7 @@ jobs: set -euo pipefail HELM_ARGS=( - -f "helm/values-${{ inputs.environment }}.yaml" + -f "./helm/values-${{ inputs.environment }}.yaml" --namespace "$K8S_NAMESPACE" --set "image.tag=${{ inputs.image_tag }}" --set config.mode=catalog @@ -181,11 +181,11 @@ jobs: ) if [[ "$DRY_RUN" == "true" ]]; then - helm upgrade --install prl-devops-catalog helm "${HELM_ARGS[@]}" --dry-run --debug + helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" --dry-run --debug elif [[ "$ATOMIC" == "true" ]]; then DEPLOY_ARGS=(--atomic --timeout 5m) - helm upgrade --install prl-devops-catalog helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" + helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" else DEPLOY_ARGS=(--wait --timeout 5m) - helm upgrade --install prl-devops-catalog helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" + helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" fi \ No newline at end of file From aec93f315c364a98650a0f68b0bd80c3a3283fc7 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Thu, 16 Jul 2026 11:21:47 -0400 Subject: [PATCH 32/34] [ACE-1975] Mask out the runner name and workflow ref --- .github/workflows/deploy-aks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index 8c20cb4c..ac199098 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -72,6 +72,8 @@ jobs: oidc_subject="repo:${GITHUB_REPOSITORY}:ref:${GITHUB_REF}" echo "::add-mask::$oidc_subject" echo "::add-mask::api://AzureADTokenExchange" + echo "::add-mask::$RUNNER_NAME" + echo "::add-mask::$GITHUB_WORKFLOW_REF" - name: Azure login (OIDC) uses: azure/login@v2 From c81c47f5a7c7eed182f9ac0265a89c4765623ae5 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Thu, 16 Jul 2026 11:39:54 -0400 Subject: [PATCH 33/34] [ACE-1975] Flag --atomic has been deprecated, use --rollback-on-failure instead --- .github/workflows/deploy-aks.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy-aks.yml b/.github/workflows/deploy-aks.yml index ac199098..6b256509 100644 --- a/.github/workflows/deploy-aks.yml +++ b/.github/workflows/deploy-aks.yml @@ -43,8 +43,8 @@ on: required: false default: false type: boolean - atomic: - description: Rollback on failure (atomic deployment) + rollback_on_failure: + description: Roll back on failure required: false type: boolean default: true @@ -72,7 +72,6 @@ jobs: oidc_subject="repo:${GITHUB_REPOSITORY}:ref:${GITHUB_REF}" echo "::add-mask::$oidc_subject" echo "::add-mask::api://AzureADTokenExchange" - echo "::add-mask::$RUNNER_NAME" echo "::add-mask::$GITHUB_WORKFLOW_REF" - name: Azure login (OIDC) @@ -131,7 +130,7 @@ jobs: env: K8S_NAMESPACE: ${{ inputs.namespace }} DRY_RUN: ${{ inputs.dry_run }} - ATOMIC: ${{ inputs.atomic }} + ROLLBACK_ON_FAILURE: ${{ inputs.rollback_on_failure }} run: | set -euo pipefail @@ -146,8 +145,8 @@ jobs: if [[ "$DRY_RUN" == "true" ]]; then helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" --dry-run --debug - elif [[ "$ATOMIC" == "true" ]]; then - DEPLOY_ARGS=(--atomic --timeout 5m) + elif [[ "$ROLLBACK_ON_FAILURE" == "true" ]]; then + DEPLOY_ARGS=(--rollback-on-failure --timeout 5m) helm upgrade --install prl-devops-orchestrator ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" else DEPLOY_ARGS=(--wait --timeout 5m) @@ -169,7 +168,7 @@ jobs: env: K8S_NAMESPACE: ${{ inputs.namespace }} DRY_RUN: ${{ inputs.dry_run }} - ATOMIC: ${{ inputs.atomic }} + ROLLBACK_ON_FAILURE: ${{ inputs.rollback_on_failure }} run: | set -euo pipefail @@ -184,8 +183,8 @@ jobs: if [[ "$DRY_RUN" == "true" ]]; then helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" --dry-run --debug - elif [[ "$ATOMIC" == "true" ]]; then - DEPLOY_ARGS=(--atomic --timeout 5m) + elif [[ "$ROLLBACK_ON_FAILURE" == "true" ]]; then + DEPLOY_ARGS=(--rollback-on-failure --timeout 5m) helm upgrade --install prl-devops-catalog ./helm "${HELM_ARGS[@]}" "${DEPLOY_ARGS[@]}" else DEPLOY_ARGS=(--wait --timeout 5m) From 6d6f2b8e4df0afa60abf03cd9b37e2d648185468 Mon Sep 17 00:00:00 2001 From: osh0501 Date: Thu, 16 Jul 2026 12:33:39 -0400 Subject: [PATCH 34/34] [ACE-1975] Update workflow permissions to allow createWorkflowDispatch calls --- .github/workflows/release.yml | 2 ++ .github/workflows/release_beta.yml | 2 ++ .github/workflows/release_canary.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f9bc0e73..3019bbd6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -253,6 +253,8 @@ jobs: - release - build-containers runs-on: ubuntu-latest + permissions: + actions: write steps: - name: Dispatch deploy workflow on main uses: actions/github-script@v7 diff --git a/.github/workflows/release_beta.yml b/.github/workflows/release_beta.yml index 985b488a..4ee08b06 100644 --- a/.github/workflows/release_beta.yml +++ b/.github/workflows/release_beta.yml @@ -284,6 +284,8 @@ jobs: - beta-release - build-containers runs-on: ubuntu-latest + permissions: + actions: write steps: - name: Dispatch deploy workflow on main uses: actions/github-script@v7 diff --git a/.github/workflows/release_canary.yml b/.github/workflows/release_canary.yml index 7d6a69f7..fd3876b9 100644 --- a/.github/workflows/release_canary.yml +++ b/.github/workflows/release_canary.yml @@ -285,6 +285,8 @@ jobs: - canary-release - build-containers runs-on: ubuntu-latest + permissions: + actions: write steps: - name: Dispatch deploy workflow on main uses: actions/github-script@v7