diagnose-codebuild.yml

name: Diagnose AWS CodeBuild

on:
  workflow_dispatch:
    inputs:
      build_id:
        description: Full CodeBuild build ID (project:uuid)
        required: true
        type: string
      environment:
        description: GitHub environment whose AWS role can read the build
        required: true
        default: production
        type: choice
        options:
          - production
          - staging
      stack_name:
        description: CloudFormation stack to inspect when deployment failed
        required: false
        default: agent-commons-api-service
        type: string

permissions:
  contents: read
  id-token: write

jobs:
  diagnose:
    name: Read CodeBuild failure details
    runs-on: ubuntu-latest
    environment: ${{ inputs.environment }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v5
        with:
          role-to-assume: ${{ vars.AWS_DEPLOY_ROLE_ARN }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Show build phases and logs
        env:
          BUILD_ID: ${{ inputs.build_id }}
          STACK_NAME: ${{ inputs.stack_name }}
        run: |
          set -euo pipefail

          aws codebuild batch-get-builds \
            --ids "$BUILD_ID" \
            --query 'builds[0].{status:buildStatus,currentPhase:currentPhase,startTime:startTime,endTime:endTime,phases:phases[*].{phase:phaseType,status:phaseStatus,durationSeconds:durationInSeconds,contexts:contexts},logs:logs}' \
            --output json

          log_group=$(aws codebuild batch-get-builds \
            --ids "$BUILD_ID" \
            --query 'builds[0].logs.groupName' \
            --output text)
          log_stream=$(aws codebuild batch-get-builds \
            --ids "$BUILD_ID" \
            --query 'builds[0].logs.streamName' \
            --output text)

          if [ -n "$log_group" ] && [ "$log_group" != "None" ] && \
             [ -n "$log_stream" ] && [ "$log_stream" != "None" ]; then
            aws logs get-log-events \
              --log-group-name "$log_group" \
              --log-stream-name "$log_stream" \
              --limit 10000 \
              --query 'events[].message' \
              --output text || echo "CodeBuild phases were readable, but this role cannot read CloudWatch logs."
          fi

      - name: Inspect the failed service through the CodeBuild role
        if: ${{ inputs.stack_name != '' }}
        env:
          BUILD_ID: ${{ inputs.build_id }}
          DIAGNOSTIC_BUCKET: ${{ vars.AWS_SOURCE_BUCKET }}
          STACK_NAME: ${{ inputs.stack_name }}
        run: |
          set -euo pipefail

          project_name=${BUILD_ID%%:*}
          diagnostic_key="agent-commons/diagnostics/service-deployment.json"

          python -m pip install --quiet boto3
          export DIAGNOSTIC_KEY="$diagnostic_key"
          put_url=$(python - <<'PY'
          import os
          import boto3

          print(
              boto3.client("s3").generate_presigned_url(
                  "put_object",
                  Params={
                      "Bucket": os.environ["DIAGNOSTIC_BUCKET"],
                      "Key": os.environ["DIAGNOSTIC_KEY"],
                  },
                  ExpiresIn=900,
              )
          )
          PY
          )
          echo "::add-mask::$put_url"

          printf '{"preflight":true}\n' > /tmp/service-deployment-preflight.json
          curl --fail-with-body --silent --show-error \
            -X PUT \
            -H 'Content-Type:' \
            --data-binary @/tmp/service-deployment-preflight.json \
            "$put_url"
          aws s3 cp "s3://${DIAGNOSTIC_BUCKET}/${diagnostic_key}" - >/dev/null

          buildspec=$(cat <<'YAML'
          version: 0.2
          env:
            shell: bash
          phases:
            build:
              commands:
                - |
                  set -euo pipefail
                  stack_json=$(aws cloudformation describe-stacks \
                    --stack-name "$STACK_NAME" \
                    --query 'Stacks[0].{status:StackStatus,statusReason:StackStatusReason,lastUpdated:LastUpdatedTime,outputs:Outputs}' \
                    --output json)
                  service_arn=$(jq -r '.outputs[] | select(.OutputKey == "ServiceArn") | .OutputValue' <<<"$stack_json")
                  service_json=$(aws ecs describe-express-gateway-service \
                    --service-arn "$service_arn" \
                    --query 'service.{serviceArn:serviceArn,serviceName:serviceName,status:status,currentDeployment:currentDeployment,updatedAt:updatedAt}' \
                    --output json)
                  deployment_list=$(aws ecs list-service-deployments \
                    --service "$service_arn" \
                    --max-results 20 \
                    --query '{serviceDeployments:serviceDeployments[*].{arn:serviceDeploymentArn,status:status,statusReason:statusReason,createdAt:createdAt,startedAt:startedAt,finishedAt:finishedAt,targetRevision:targetServiceRevisionArn}}' \
                    --output json)
                  available_secret_json=$(aws secretsmanager get-secret-value \
                    --secret-id "$RUNTIME_SECRET_ARN" \
                    --query SecretString \
                    --output text)
                  available_secret_keys=$(jq -c 'keys | sort' <<<"$available_secret_json")
                  required_secret_keys=$(sed \
                    -e '/^[[:space:]]*#/d' \
                    -e '/^[[:space:]]*$/d' \
                    infra/aws/runtime-secret-keys.txt | jq -Rsc 'split("\n") | map(select(length > 0)) | sort')
                  missing_secret_keys=$(jq -nc \
                    --argjson required "$required_secret_keys" \
                    --argjson available "$available_secret_keys" \
                    '$required - $available')

                  # Re-run the pending migrations from the uploaded source and
                  # capture the real error through the diagnostic
                  # S3 channel. The GitHub deploy role intentionally cannot read
                  # CodeBuild's CloudWatch stream.
                  corepack enable pnpm
                  corepack prepare pnpm@9.15.3 --activate
                  pnpm install --frozen-lockfile --filter commons-api...
                  set +e
                  POSTGRES_HOST="$(jq -r .POSTGRES_HOST <<<"$available_secret_json")" \
                  POSTGRES_PORT="$(jq -r .POSTGRES_PORT <<<"$available_secret_json")" \
                  POSTGRES_DATABASE="$(jq -r .POSTGRES_DATABASE <<<"$available_secret_json")" \
                  POSTGRES_USER="$(jq -r .POSTGRES_USER <<<"$available_secret_json")" \
                  POSTGRES_PASSWORD="$(jq -r .POSTGRES_PASSWORD <<<"$available_secret_json")" \
                  POSTGRES_SSL="$(jq -r '.POSTGRES_SSL // "disable"' <<<"$available_secret_json")" \
                    pnpm --filter commons-api migrate > /tmp/migration-probe.log 2>&1
                  migration_exit=$?
                  set -e
                  jq -n \
                    --argjson stack "$stack_json" \
                    --argjson service "$service_json" \
                    --argjson deploymentList "$deployment_list" \
                    --argjson availableSecretKeys "$available_secret_keys" \
                    --argjson missingSecretKeys "$missing_secret_keys" \
                    --argjson migrationExit "$migration_exit" \
                    --rawfile migrationLog /tmp/migration-probe.log \
                    '{stack: $stack, service: $service, deploymentList: $deploymentList, runtimeSecret: {availableKeys: $availableSecretKeys, missingKeys: $missingSecretKeys}, migrationProbe: {exitCode: $migrationExit, log: $migrationLog}}' \
                    > /tmp/service-deployment.json
                  curl --fail-with-body --silent --show-error -X PUT -H 'Content-Type:' --data-binary @/tmp/service-deployment.json "$DIAGNOSTIC_PUT_URL"
          YAML
          )

          environment_overrides=$(jq -nc \
            --arg putUrl "$put_url" \
            --arg stackName "$STACK_NAME" \
            '[
              {name:"DIAGNOSTIC_PUT_URL", value:$putUrl, type:"PLAINTEXT"},
              {name:"STACK_NAME", value:$stackName, type:"PLAINTEXT"}
            ]')

          diagnostic_build_id=$(aws codebuild start-build \
            --project-name "$project_name" \
            --buildspec-override "$buildspec" \
            --environment-variables-override "$environment_overrides" \
            --query 'build.id' \
            --output text)

          while true; do
            status=$(aws codebuild batch-get-builds \
              --ids "$diagnostic_build_id" \
              --query 'builds[0].buildStatus' \
              --output text)
            case "$status" in
              SUCCEEDED) break ;;
              FAILED|FAULT|STOPPED|TIMED_OUT)
                aws codebuild batch-get-builds \
                  --ids "$diagnostic_build_id" \
                  --query 'builds[0].phases[*].{phase:phaseType,status:phaseStatus,contexts:contexts}' \
                  --output json
                exit 1
                ;;
              *) sleep 5 ;;
            esac
          done

          aws s3 cp "s3://${DIAGNOSTIC_BUCKET}/${diagnostic_key}" -