codebuild-deployment-template.yaml

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT-0
# Workshop deployment template for CDK projects - CloudFormation
# Version: 1.0.0 - Simplified

AWSTemplateFormatVersion: '2010-09-09'
Description: |
  Automates CDK project deployment for workshop environments. Provisions S3 storage, CodeBuild project, and monitoring resources to bootstrap AWS accounts and deploy CDK applications consistently. Includes error handling and cleanup mechanisms for failed deployments.

  The pWaitForDeployment parameter controls whether CloudFormation waits for the CDK pipeline to complete before marking the stack as CREATE_COMPLETE. Set to 'false' when manually fixing pipeline issues to allow the stack to complete without waiting for pipeline execution.

  For more details, see: https://github.com/aws-samples/one-observability-demo

Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: Repository Configuration
        Parameters:
          - pConfigFileUrl
          - pOrganizationName
          - pRepositoryName
          - pBranchName
          - pCodeConnectionArn
          - pWorkingFolder
          - pApplicationName
          - pDisableCleanup
          - pCDKStackName
          - pParameterStoreBasePath
          - pWaitForDeployment
      - Label:
          default: Network Configuration
        Parameters:
          - pVpcCidr
      - Label:
          default: User Defined Tags
        Parameters:
          - pUserDefinedTagKey1
          - pUserDefinedTagValue1
          - pUserDefinedTagKey2
          - pUserDefinedTagValue2
          - pUserDefinedTagKey3
          - pUserDefinedTagValue3
          - pUserDefinedTagKey4
          - pUserDefinedTagValue4
          - pUserDefinedTagKey5
          - pUserDefinedTagValue5

Parameters:

  pDisableCleanup:
    Type: String
    Description: Disable cleanup in post_build stage if deployment fails
    Default: 'false'
    AllowedValues:
      - 'true'
      - 'false'
    ConstraintDescription: Must be either 'true' or 'false'

  pConfigFileUrl:
    Type: String
    Description: URL to the initial configuration file
    AllowedPattern: ^https?://.*$
    ConstraintDescription: Must be a valid HTTP or HTTPS URL
    Default: https://raw.githubusercontent.com/aws-samples/one-observability-demo/refs/heads/main/src/presets/default.env

  pOrganizationName:
    Type: String
    Description: GitHub/CodeCommit organization name
    AllowedPattern: '[A-Za-z0-9_.-]+'
    ConstraintDescription: Must contain only alphanumeric characters, underscores, periods, or hyphens
    Default: aws-samples

  pRepositoryName:
    Type: String
    Description: Repository containing the CDK code
    AllowedPattern: '[A-Za-z0-9_.-]+'
    ConstraintDescription: Must contain only alphanumeric characters, underscores, periods, or hyphens
    Default: one-observability-demo

  pBranchName:
    Type: String
    Description: Branch to deploy from
    Default: main
    AllowedPattern: '[A-Za-z0-9_./-]+'
    ConstraintDescription: Must contain only alphanumeric characters, underscores,
      periods, slashes, or hyphens

  pWorkingFolder:
    Type: String
    Description: Working folder for deployment
    Default: src/cdk
    AllowedPattern: '[A-Za-z0-9_.-/]+'
    ConstraintDescription: Must contain only alphanumeric characters, underscores, periods, or hyphens

  pApplicationName:
    Type: String
    Description: Application name used for tagging deployed stacks
    Default: One Observability Workshop
    AllowedPattern: ^[A-Za-z0-9\s_.-]+$
    ConstraintDescription: Must contain only alphanumeric characters, spaces, underscores, periods, or hyphens

  pUserDefinedTagKey1:
    Type: String
    Description: Key for user defined tag 1
    AllowedPattern: ^(?!aws:)[A-Za-z0-9\s_.:=+-@/]{0,128}
    ConstraintDescription: 'Must match the allowable values for a Tag Key. This can
      only contain alphanumeric characters or special characters ( _ . : / = + -
      or @) up to 128 characters'
    Default: WorkshopName

  pUserDefinedTagValue1:
    Type: String
    Description: Value for user defined tag 1
    AllowedPattern: ^[\w\s_.:=+-@/]{0,256}$
    ConstraintDescription: 'Must match the allowable values for a Tag Value. This
      can only contain alphanumeric characters or special characters ( _ . : / =
      + - or @) up to 256 characters'
    Default: one-observability-demo

  pUserDefinedTagKey2:
    Type: String
    Description: Key for user defined tag 2
    AllowedPattern: ^(?!aws:)[A-Za-z0-9\s_.:=+-@/]{0,128}
    ConstraintDescription: 'Must match the allowable values for a Tag Key. This can
      only contain alphanumeric characters or special characters ( _ . : / = + -
      or @) up to 128 characters'
    Default: Version

  pUserDefinedTagValue2:
    Type: String
    Description: Value for user defined tag 2
    AllowedPattern: ^[\w\s_.:=+-@/]{0,256}$
    ConstraintDescription: 'Must match the allowable values for a Tag Value. This
      can only contain alphanumeric characters or special characters ( _ . : / =
      + - or @) up to 256 characters'
    Default: 1.0.0

  pUserDefinedTagKey3:
    Type: String
    Description: Key for user defined tag 3
    AllowedPattern: ^(?!aws:)[A-Za-z0-9\s_.:=+-@/]{0,128}
    ConstraintDescription: 'Must match the allowable values for a Tag Key. This can
      only contain alphanumeric characters or special characters ( _ . : / = + -
      or @) up to 128 characters'

  pUserDefinedTagValue3:
    Type: String
    Description: Value for user defined tag 3
    AllowedPattern: ^[\w\s_.:=+-@/]{0,256}$
    ConstraintDescription: 'Must match the allowable values for a Tag Value. This
      can only contain alphanumeric characters or special characters ( _ . : / =
      + - or @) up to 256 characters'

  pUserDefinedTagKey4:
    Type: String
    Description: Key for user defined tag 4
    AllowedPattern: ^(?!aws:)[A-Za-z0-9\s_.:=+-@/]{0,128}
    ConstraintDescription: 'Must match the allowable values for a Tag Key. This can
      only contain alphanumeric characters or special characters ( _ . : / = + -
      or @) up to 128 characters'

  pUserDefinedTagValue4:
    Type: String
    Description: Value for user defined tag 4
    AllowedPattern: ^[\w\s_.:=+-@/]{0,256}$
    ConstraintDescription: 'Must match the allowable values for a Tag Value. This
      can only contain alphanumeric characters or special characters ( _ . : / =
      + - or @) up to 256 characters'

  pUserDefinedTagKey5:
    Type: String
    Description: Key for user defined tag 5
    AllowedPattern: ^(?!aws:)[A-Za-z0-9\s_.:=+-@/]{0,128}
    ConstraintDescription: 'Must match the allowable values for a Tag Key. This can
      only contain alphanumeric characters or special characters ( _ . : / = + -
      or @) up to 128 characters'

  pUserDefinedTagValue5:
    Type: String
    Description: Value for user defined tag 5
    AllowedPattern: ^[\w\s_.:=+-@/]{0,256}$
    ConstraintDescription: 'Must match the allowable values for a Tag Value. This
      can only contain alphanumeric characters or special characters ( _ . : / =
      + - or @) up to 256 characters'

  pVpcCidr:
    Type: String
    Description: CIDR block for the VPC
    Default: '10.0.0.0/16'
    AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-8]))$
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28
  pCDKStackName:
    Type: String
    Description: CDK stack name to retrieve outputs from
    Default: Microservices-Microservice

  pCodeConnectionArn:
    Type: String
    Description: Optional CodeConnection ARN for GitHub integration. If provided, will be used instead of S3 as pipeline source.
    Default: ''
    AllowedPattern: '^(arn:aws:codeconnections:[a-z0-9-]+:\d{12}:connection/[a-f0-9-]+)?$'
    ConstraintDescription: Must be empty or a valid CodeConnection ARN

  pParameterStoreBasePath:
    Type: String
    Description: Base path in Parameter Store for storing configuration
    Default: /petstore
    AllowedPattern: '^/[a-zA-Z0-9/_-]+$'
    ConstraintDescription: Must be a valid Parameter Store path starting with /

  pWaitForDeployment:
    Type: String
    Description: Wait for deployment completion using CloudFormation wait condition. Set to 'false' to mark stack complete without waiting for pipeline (useful when manually fixing pipeline issues).
    Default: 'true'
    AllowedValues:
      - 'true'
      - 'false'
    ConstraintDescription: Must be either 'true' or 'false'



Conditions:
  # Conditions to control creation of user-defined tags
  cCreateTag1: !And
    - !Not
      - !Equals
        - !Ref pUserDefinedTagKey1
        - ''
    - !Not
      - !Equals
        - !Ref pUserDefinedTagValue1
        - ''

  cCreateTag2: !And
    - !Not
      - !Equals
        - !Ref pUserDefinedTagKey2
        - ''
    - !Not
      - !Equals
        - !Ref pUserDefinedTagValue2
        - ''

  cCreateTag3: !And
    - !Not
      - !Equals
        - !Ref pUserDefinedTagKey3
        - ''
    - !Not
      - !Equals
        - !Ref pUserDefinedTagValue3
        - ''

  cCreateTag4: !And
    - !Not
      - !Equals
        - !Ref pUserDefinedTagKey4
        - ''
    - !Not
      - !Equals
        - !Ref pUserDefinedTagValue4
        - ''

  cCreateTag5: !And
    - !Not
      - !Equals
        - !Ref pUserDefinedTagKey5
        - ''
    - !Not
      - !Equals
        - !Ref pUserDefinedTagValue5
        - ''

  cWaitForDeployment: !Equals
    - !Ref pWaitForDeployment
    - 'true'

Resources:
  # Lambda execution role for resource cleanup
  rResourceCleanupRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: ResourceCleanup
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3:ListBucket
                  - s3:DeleteObject
                  - s3:ListBucketVersions
                  - s3:DeleteObjectVersion
                Resource:
                  - !Sub ${rConfigBucket.Arn}/*
                  - !GetAtt rConfigBucket.Arn

  # Lambda function to clean up resources
  rResourceCleanupFunction:
    Type: AWS::Lambda::Function
    Properties:
      Runtime: python3.13
      Handler: index.handler
      Role: !GetAtt rResourceCleanupRole.Arn
      Timeout: 300
      Code:
        ZipFile: |
          import boto3
          import cfnresponse
          def handler(event, context):
              try:
                  if event['RequestType'] == 'Delete':
                      # Clean up S3 bucket
                      s3 = boto3.client('s3')
                      bucket = event['ResourceProperties']['BucketName']
                      paginator = s3.get_paginator('list_object_versions')
                      for page in paginator.paginate(Bucket=bucket):
                          delete_keys = []
                          if 'Versions' in page:
                              delete_keys.extend([{'Key': obj['Key'], 'VersionId': obj['VersionId']} for obj in page['Versions']])
                          if 'DeleteMarkers' in page:
                              delete_keys.extend([{'Key': obj['Key'], 'VersionId': obj['VersionId']} for obj in page['DeleteMarkers']])
                          if delete_keys:
                              s3.delete_objects(Bucket=bucket, Delete={'Objects': delete_keys})
                  cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
              except Exception as e:
                  cfnresponse.send(event, context, cfnresponse.FAILED, {}, str(e))

  # Create S3 bucket to store configuration files
  rConfigBucket:
    Type: AWS::S3::Bucket
    Properties:
      VersioningConfiguration:
        Status: Enabled
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-config-bucket
        - !If
          - cCreateTag1
          - Key: !Ref pUserDefinedTagKey1
            Value: !Ref pUserDefinedTagValue1
          - !Ref AWS::NoValue
        - !If
          - cCreateTag2
          - Key: !Ref pUserDefinedTagKey2
            Value: !Ref pUserDefinedTagValue2
          - !Ref AWS::NoValue
        - !If
          - cCreateTag3
          - Key: !Ref pUserDefinedTagKey3
            Value: !Ref pUserDefinedTagValue3
          - !Ref AWS::NoValue
        - !If
          - cCreateTag4
          - Key: !Ref pUserDefinedTagKey4
            Value: !Ref pUserDefinedTagValue4
          - !Ref AWS::NoValue
        - !If
          - cCreateTag5
          - Key: !Ref pUserDefinedTagKey5
            Value: !Ref pUserDefinedTagValue5
          - !Ref AWS::NoValue

  # Custom resource to clean up resources on deletion
  rResourceCleanup:
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt rResourceCleanupFunction.Arn
      BucketName: !Ref rConfigBucket
      StackName: !Ref AWS::StackName

  # Bucket policy to enforce HTTPS-only access
  rConfigBucketPolicy:
    Type: AWS::S3::BucketPolicy
    Properties:
      Bucket: !Ref rConfigBucket
      PolicyDocument:
        Statement:
          - Sid: DenyInsecureConnections
            Effect: Deny
            Principal: '*'
            Action: s3:*
            Resource:
              - !GetAtt rConfigBucket.Arn
              - !Sub ${rConfigBucket.Arn}/*
            Condition:
              Bool:
                aws:SecureTransport: 'false'

  # Parameter Store parameter to store configuration
  rWorkshopConfigParameter:
    Type: AWS::SSM::Parameter
    Properties:
      Name: !Sub '${pParameterStoreBasePath}/${AWS::StackName}/config'
      Type: String
      Value: '# Configuration will be populated by CodeBuild'
      Description: !Sub 'Workshop configuration for stack ${AWS::StackName}'
      Tags:
        Environment: workshop
        StackName: !Ref AWS::StackName

  # S3 bucket for CodeBuild cache
  # Create IAM role for CodeBuild project
  rCodeBuildServiceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: codebuild.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - !Sub arn:${AWS::Partition}:iam::aws:policy/AdministratorAccess
      Policies:
        - PolicyName: EnhancedPipelineMonitoring
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - codepipeline:ListPipelineExecutions
                  - codepipeline:GetPipelineExecution
                  - codepipeline:GetPipelineState
                  - codepipeline:ListActionExecutions
                Resource: '*'
                Condition:
                  StringEquals:
                    'aws:RequestedRegion': !Ref AWS::Region
        - PolicyName: ParameterStoreAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - ssm:GetParameter
                  - ssm:GetParameters
                  - ssm:GetParametersByPath
                  - ssm:PutParameter
                  - ssm:DeleteParameter
                Resource: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${pParameterStoreBasePath}/*'
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-codebuild-role
        - !If
          - cCreateTag1
          - Key: !Ref pUserDefinedTagKey1
            Value: !Ref pUserDefinedTagValue1
          - !Ref AWS::NoValue
        - !If
          - cCreateTag2
          - Key: !Ref pUserDefinedTagKey2
            Value: !Ref pUserDefinedTagValue2
          - !Ref AWS::NoValue
        - !If
          - cCreateTag3
          - Key: !Ref pUserDefinedTagKey3
            Value: !Ref pUserDefinedTagValue3
          - !Ref AWS::NoValue
        - !If
          - cCreateTag4
          - Key: !Ref pUserDefinedTagKey4
            Value: !Ref pUserDefinedTagValue4
          - !Ref AWS::NoValue
        - !If
          - cCreateTag5
          - Key: !Ref pUserDefinedTagKey5
            Value: !Ref pUserDefinedTagValue5
          - !Ref AWS::NoValue

  # Create wait condition handle for signaling completion
  rCDKDeploymentWaitConditionHandle:
    Type: AWS::CloudFormation::WaitConditionHandle
    Condition: cWaitForDeployment

  # Create wait condition to ensure CodeBuild project completes successfully
  rCDKDeploymentWaitCondition:
    Type: AWS::CloudFormation::WaitCondition
    Condition: cWaitForDeployment
    Properties:
      Handle: !Ref rCDKDeploymentWaitConditionHandle
      Timeout: 3600
      Count: 1

  rCDKOutputRetrieverRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: CloudFormationReadAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - cloudformation:DescribeStacks
                  - cloudformation:ListExports
                Resource: '*'

  rCDKOutputRetrieverFunction:
    Type: AWS::Lambda::Function
    Properties:
      Runtime: python3.13
      Handler: index.handler
      Role: !GetAtt rCDKOutputRetrieverRole.Arn
      Timeout: 60
      Code:
        ZipFile: |
          import boto3
          import cfnresponse

          def handler(event, context):
              try:
                  if event['RequestType'] == 'Delete':
                      cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
                      return

                  cf = boto3.client('cloudformation')
                  print("Retrieving CloudFormation exports")

                  # Get all exports
                  exports = {}
                  paginator = cf.get_paginator('list_exports')
                  for page in paginator.paginate():
                      for export in page['Exports']:
                          exports[export['Name']] = export['Value']

                  print(f"Found {len(exports)} total exports: {list(exports.keys())}")
                  outputs = {}

                  # Find CloudFront domain for dashboard URL
                  for export_name, export_value in exports.items():
                      if 'WorkshopCloudFrontDomain' in export_name:
                          print(f"Found CloudFront domain: {export_name}")
                          cloudfront_domain = export_value if export_value.startswith('https://') else f"https://{export_value}"
                          outputs['ExportsDashboardUrl'] = f"{cloudfront_domain.rstrip('/')}/workshop-exports/index.html"
                          print(f"Dashboard URL: {outputs['ExportsDashboardUrl']}")
                          break
                  else:
                      print("Warning: WorkshopCloudFrontDomain export not found")
                      outputs['ExportsDashboardUrl'] = 'missing'

                  # Find PetSite URL
                  for export_name, export_value in exports.items():
                      if 'WorkshopPetSiteUrl' in export_name:
                          print(f"Found PetSite URL: {export_name}")
                          outputs['PetSiteUrl'] = export_value
                          break
                  else:
                      print("Warning: WorkshopPetSiteUrl export not found")
                      outputs['PetSiteUrl'] = 'missing'

                  print(f"Returning {len(outputs)} outputs")
                  cfnresponse.send(event, context, cfnresponse.SUCCESS, outputs)
              except Exception as e:
                  print(f"Error: {str(e)}")
                  cfnresponse.send(event, context, cfnresponse.FAILED, {}, str(e))

  rCDKOutputs:
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt rCDKOutputRetrieverFunction.Arn
      StackName: !Ref pCDKStackName
      # This ensures the custom resource waits for the wait condition if it exists
      WaitConditionData: !If
        - cWaitForDeployment
        - !GetAtt rCDKDeploymentWaitCondition.Data
        - 'NoWait'

  # Create CodeBuild project to orchestrate CDK deployment
  rCDKDeploymentProject:
    Type: AWS::CodeBuild::Project
    Properties:
      Name: !Sub ${AWS::StackName}-cdk-deployment
      Artifacts:
        Type: NO_ARTIFACTS
      Environment:
        Type: LINUX_CONTAINER
        ComputeType: BUILD_GENERAL1_SMALL
        Image: aws/codebuild/amazonlinux2-x86_64-standard:5.0
        PrivilegedMode: true
        EnvironmentVariables:
          - Name: APPLICATION_NAME
            Value: !Ref pApplicationName
            Type: PLAINTEXT
          - Name: CONFIG_FILE_URL
            Value: !Ref pConfigFileUrl
            Type: PLAINTEXT
          - Name: CONFIG_BUCKET
            Value: !Ref rConfigBucket
            Type: PLAINTEXT
          - Name: ORGANIZATION_NAME
            Value: !Ref pOrganizationName
            Type: PLAINTEXT
          - Name: REPOSITORY_NAME
            Value: !Ref pRepositoryName
            Type: PLAINTEXT
          - Name: BRANCH_NAME
            Value: !Ref pBranchName
            Type: PLAINTEXT
          - Name: WORKING_FOLDER
            Value: !Ref pWorkingFolder
            Type: PLAINTEXT
          - Name: STACK_NAME
            Value: !Ref AWS::StackName
            Type: PLAINTEXT
          - Name: AWS_REGION
            Value: !Ref AWS::Region
            Type: PLAINTEXT
          - Name: AWS_ACCOUNT_ID
            Value: !Ref AWS::AccountId
            Type: PLAINTEXT
          - Name: DISABLE_CLEANUP
            Value: !Ref pDisableCleanup
            Type: PLAINTEXT
          - Name: VPC_CIDR
            Value: !Ref pVpcCidr
            Type: PLAINTEXT
          - Name: CODE_CONNECTION_ARN
            Value: !Ref pCodeConnectionArn
            Type: PLAINTEXT
          - Name: PARAMETER_STORE_BASE_PATH
            Value: !Ref pParameterStoreBasePath
            Type: PLAINTEXT
          - Name: WAIT_FOR_DEPLOYMENT
            Value: !Ref pWaitForDeployment
            Type: PLAINTEXT
      Cache:
        Type: LOCAL
        Modes:
          - LOCAL_DOCKER_LAYER_CACHE
      ServiceRole: !GetAtt rCodeBuildServiceRole.Arn
      Source:
        Type: NO_SOURCE
        BuildSpec: |
          version: 0.2
          phases:
            install:
              runtime-versions:
                nodejs: 22
                python: 3.12
              commands:
                - npm install -g aws-cdk
                - pip3 install git-remote-s3
            pre_build:
              commands:
                # Configure git with generic user info
                - git config --global user.email "codebuild@aws.amazon.com"
                - git config --global user.name "AWS CodeBuild"
                # Clone repository with shallow clone
                - git clone --depth 1 --branch $BRANCH_NAME https://github.com/$ORGANIZATION_NAME/$REPOSITORY_NAME.git ./repo
                - cd ./repo
                # Download configuration file and create initial configuration
                - |
                  curl -o ./config.env "$CONFIG_FILE_URL"
                - cat ./config.env >> ${WORKING_FOLDER}/.env
                # Run account validation script
                - ./${WORKING_FOLDER}/scripts/validate-account.sh ${WORKING_FOLDER}/.env --clean
                - |
                  CONFIG_PARAM_NAME="${PARAMETER_STORE_BASE_PATH}/${STACK_NAME}/config"
                  echo "Parameter path: ${CONFIG_PARAM_NAME}"

                # Append additional environment variables to .env file
                - |
                  cat >> ${WORKING_FOLDER}/.env << EOF
                  CONFIG_FILE_URL=$CONFIG_FILE_URL
                  CONFIG_BUCKET=$CONFIG_BUCKET
                  ORGANIZATION_NAME=$ORGANIZATION_NAME
                  REPOSITORY_NAME=$REPOSITORY_NAME
                  BRANCH_NAME=$BRANCH_NAME
                  WORKING_FOLDER=$WORKING_FOLDER
                  STACK_NAME=$STACK_NAME
                  AWS_REGION=$AWS_REGION
                  AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID
                  DISABLE_CLEANUP=$DISABLE_CLEANUP
                  CODE_CONNECTION_ARN=$CODE_CONNECTION_ARN
                  PARAMETER_STORE_BASE_PATH=$PARAMETER_STORE_BASE_PATH
                  CONFIG_PARAM_NAME=$CONFIG_PARAM_NAME
                  EOF
                # Store complete configuration in Parameter Store (parameter created by CloudFormation)
                - |
                  echo "Storing complete configuration in Parameter Store parameter"
                  # Update the CloudFormation-created parameter with complete configuration
                  aws ssm put-parameter \
                    --name "${CONFIG_PARAM_NAME}" \
                    --value "$(cat ${WORKING_FOLDER}/.env)" \
                    --type "String" \
                    --overwrite

                  echo "Configuration successfully stored in Parameter Store"
                  echo "Configuration content:"
                  cat ${WORKING_FOLDER}/.env
                # Handle repository source configuration based on CodeConnection ARN
                - |
                  if [ -n "$CODE_CONNECTION_ARN" ] && [ "$CODE_CONNECTION_ARN" != "" ]; then
                    echo "CodeConnection ARN provided: $CODE_CONNECTION_ARN"
                    echo "Will use CodeConnection as pipeline source"
                  else
                    echo "No CodeConnection ARN provided, using S3 as fallback source"
                    # Add .env file to repository and push to S3
                    git add ${WORKING_FOLDER}/.env -f
                    git commit -m "Add merged configuration and environment variables to .env"
                    # Configure S3 remote with s3+zip protocol for CodePipeline
                    git remote add s3 s3+zip://$CONFIG_BUCKET/repo
                    git push s3 $BRANCH_NAME --force
                  fi
                # Print configuration summary for troubleshooting
                - |
                  echo "=== Configuration Summary ==="
                  echo "Parameter Store base path: $PARAMETER_STORE_BASE_PATH"
                  echo "Configuration path: ${PARAMETER_STORE_BASE_PATH}/${STACK_NAME}/config"
                  echo "CodeConnection ARN: ${CODE_CONNECTION_ARN:-'Not provided (using S3 fallback)'}"
                  echo "=== End Configuration Summary ==="
                # Bootstrap current region
                - ./${WORKING_FOLDER}/scripts/bootstrap-account.sh ${AWS_ACCOUNT_ID} ${AWS_REGION}
                # Bootstrap us-east-1 if WAF is enabled
                - |
                  echo "Checking for WAF configuration..."
                  grep "CUSTOM_ENABLE_WAF" ${WORKING_FOLDER}/.env || echo "WAF setting not found"
                  if grep -q "CUSTOM_ENABLE_WAF=true" ${WORKING_FOLDER}/.env; then
                    echo "WAF enabled, bootstrapping us-east-1..."
                    ./${WORKING_FOLDER}/scripts/bootstrap-account.sh ${AWS_ACCOUNT_ID} us-east-1
                  else
                    echo "WAF not enabled, skipping us-east-1 bootstrap"
                  fi
            build:
              commands:
                - cd $WORKING_FOLDER
                # Install dependencies and synthesize CDK
                - npm install
                - cdk synth --quiet
                # Deploy with proper tags so cleanup function can find the stack
                - |
                  cdk deploy OneObservability \
                    --require-approval never \
                    --outputs-file cdk-outputs.json \
                    --tags application="$APPLICATION_NAME" \
                    --tags sequence="0" \
                    --quiet
                # Extract pipeline ARN for later use
                - PIPELINE_ARN=$(cat cdk-outputs.json | jq -r '.[] | select(has("PipelineArn")) | .PipelineArn')
                - |
                  if [ -z "$PIPELINE_ARN" ] || [ "$PIPELINE_ARN" = "null" ]; then
                    echo "ERROR: Pipeline ARN is empty or null. CDK deployment failed to create pipeline."
                    exit 1
                  fi
                - |
                  echo "Pipeline ARN: $PIPELINE_ARN"
                - |
                  PIPELINE_NAME=$(echo $PIPELINE_ARN | cut -d':' -f6)
                - |
                  echo "Pipeline Name: $PIPELINE_NAME"
                # Wait for pipeline to complete
                - |
                  ./scripts/wait-for-pipeline.sh "$PIPELINE_NAME" "$STACK_NAME" "$AWS_REGION"
            post_build:
              commands:
                - |
                  if [ "$CODEBUILD_BUILD_SUCCEEDING" = "0" ]; then
                    if [ "$DISABLE_CLEANUP" = "true" ]; then
                      echo "Build failed, but cleanup is disabled. Skipping resource cleanup."
                    else
                      echo "Build failed, cleaning up S3 bucket for rollback"
                      aws s3 rm s3://$CONFIG_BUCKET --recursive
                      echo "Triggering CDK stack cleanup via Step Function"
                      aws stepfunctions start-execution \
                        --state-machine-arn "arn:aws:states:$AWS_REGION:$AWS_ACCOUNT_ID:stateMachine:$STACK_NAME-cdk-cleanup" \
                        --input '{}' || echo "Failed to trigger cleanup, continuing..."
                    fi
                    WAIT_HANDLE_URL=$(aws cloudformation describe-stack-resource --stack-name $STACK_NAME --logical-resource-id rCDKDeploymentWaitConditionHandle --query 'StackResourceDetail.PhysicalResourceId' --output text --region $AWS_REGION)
                    curl -X PUT -H 'Content-Type:' --data-binary '{"Status" : "FAILURE","Reason" : "Build failed","UniqueId" : "'$(uuidgen)'","Data" : "Build execution failed"}' "$WAIT_HANDLE_URL"
                  else
                    echo "Build completed - signal already sent during pipeline monitoring"
                  fi
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-cdk-deployment
        - !If
          - cCreateTag1
          - Key: !Ref pUserDefinedTagKey1
            Value: !Ref pUserDefinedTagValue1
          - !Ref AWS::NoValue
        - !If
          - cCreateTag2
          - Key: !Ref pUserDefinedTagKey2
            Value: !Ref pUserDefinedTagValue2
          - !Ref AWS::NoValue
        - !If
          - cCreateTag3
          - Key: !Ref pUserDefinedTagKey3
            Value: !Ref pUserDefinedTagValue3
          - !Ref AWS::NoValue
        - !If
          - cCreateTag4
          - Key: !Ref pUserDefinedTagKey4
            Value: !Ref pUserDefinedTagValue4
          - !Ref AWS::NoValue
        - !If
          - cCreateTag5
          - Key: !Ref pUserDefinedTagKey5
            Value: !Ref pUserDefinedTagValue5
          - !Ref AWS::NoValue
  # Start the CodeBuild project to begin deployment
  rStartDeployment:
    Type: Custom::StartDeployment
    DependsOn:
      - rConfigBucket
      - rCodeBuildServiceRole
    Properties:
      ServiceToken: !GetAtt rStartDeploymentFunction.Arn
      CodeBuildProjectName: !Ref rCDKDeploymentProject
      WaitConditionHandle: !If
        - cWaitForDeployment
        - !Ref rCDKDeploymentWaitConditionHandle
        - !Ref AWS::NoValue

  # Create Lambda function to start the CodeBuild project
  rStartDeploymentFunction:
    Type: AWS::Lambda::Function
    Properties:
      Runtime: python3.13
      Handler: index.handler
      Role: !GetAtt rStartDeploymentFunctionRole.Arn
      Code:
        ZipFile: |
          import boto3
          import cfnresponse
          import time

          def handler(event, context):
              if event['RequestType'] in ['Create', 'Update']:
                  try:
                      codebuild = boto3.client('codebuild')
                      project_name = event['ResourceProperties']['CodeBuildProjectName']

                      # Start the CodeBuild project
                      response = codebuild.start_build(projectName=project_name)
                      build_id = response['build']['id']

                      # Wait a few seconds to ensure the build starts
                      time.sleep(5)

                      cfnresponse.send(event, context, cfnresponse.SUCCESS, {
                          'BuildId': build_id
                      })
                  except Exception as e:
                      print(f"Error starting CodeBuild project: {str(e)}")
                      cfnresponse.send(event, context, cfnresponse.FAILED, {
                          'Error': str(e)
                      })
              else:  # Delete
                  cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
      Timeout: 30
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-start-deployment-function
        - !If
          - cCreateTag1
          - Key: !Ref pUserDefinedTagKey1
            Value: !Ref pUserDefinedTagValue1
          - !Ref AWS::NoValue
        - !If
          - cCreateTag2
          - Key: !Ref pUserDefinedTagKey2
            Value: !Ref pUserDefinedTagValue2
          - !Ref AWS::NoValue
        - !If
          - cCreateTag3
          - Key: !Ref pUserDefinedTagKey3
            Value: !Ref pUserDefinedTagValue3
          - !Ref AWS::NoValue
        - !If
          - cCreateTag4
          - Key: !Ref pUserDefinedTagKey4
            Value: !Ref pUserDefinedTagValue4
          - !Ref AWS::NoValue
        - !If
          - cCreateTag5
          - Key: !Ref pUserDefinedTagKey5
            Value: !Ref pUserDefinedTagValue5
          - !Ref AWS::NoValue

  # Create IAM role for start deployment Lambda function
  rStartDeploymentFunctionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: CodeBuildAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - codebuild:StartBuild
                Resource: !GetAtt rCDKDeploymentProject.Arn
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-start-deployment-role
        - !If
          - cCreateTag1
          - Key: !Ref pUserDefinedTagKey1
            Value: !Ref pUserDefinedTagValue1
          - !Ref AWS::NoValue
        - !If
          - cCreateTag2
          - Key: !Ref pUserDefinedTagKey2
            Value: !Ref pUserDefinedTagValue2
          - !Ref AWS::NoValue
        - !If
          - cCreateTag3
          - Key: !Ref pUserDefinedTagKey3
            Value: !Ref pUserDefinedTagValue3
          - !Ref AWS::NoValue
        - !If
          - cCreateTag4
          - Key: !Ref pUserDefinedTagKey4
            Value: !Ref pUserDefinedTagValue4
          - !Ref AWS::NoValue
        - !If
          - cCreateTag5
          - Key: !Ref pUserDefinedTagKey5
            Value: !Ref pUserDefinedTagValue5
          - !Ref AWS::NoValue

  # Step Function for CDK Stack Cleanup
  rCDKCleanupStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      StateMachineName: !Sub ${AWS::StackName}-cdk-cleanup
      RoleArn: !GetAtt rCDKCleanupRole.Arn
      DefinitionString: !Sub |
        {
          "Comment": "CDK Stack Cleanup State Machine",
          "StartAt": "ListTaggedStacks",
          "States": {
            "ListTaggedStacks": {
              "Type": "Task",
              "Resource": "arn:aws:states:::lambda:invoke",
              "Parameters": {
                "FunctionName": "${rCDKStackListerFunction}",
                "Payload": {}
              },
              "ResultPath": "$.stackList",
              "Next": "CheckStacksFound"
            },
            "CheckStacksFound": {
              "Type": "Choice",
              "Choices": [
                {
                  "Variable": "$.stackList.Payload.stacks",
                  "IsPresent": true,
                  "Next": "DeleteStacks"
                }
              ],
              "Default": "CleanupCDKStagingBucket"
            },
            "DeleteStacks": {
              "Type": "Map",
              "ItemsPath": "$.stackList.Payload.stacks",
              "MaxConcurrency": 1,
              "ResultPath": "$.deletionResults",
              "Iterator": {
                "StartAt": "DescribeStack",
                "States": {
                  "DescribeStack": {
                    "Type": "Task",
                    "Resource": "arn:aws:states:::lambda:invoke",
                    "Parameters": {
                      "FunctionName": "${rCrossRegionStackOperationFunction}",
                      "Payload.$": "$"
                    },
                    "ResultPath": "$.stackInfo",
                    "Next": "DeleteStack",
                    "Catch": [
                      {
                        "ErrorEquals": ["States.TaskFailed"],
                        "ResultPath": "$.error",
                        "Next": "CheckDescribeError"
                      }
                    ]
                  },
                  "CheckDescribeError": {
                    "Type": "Choice",
                    "Choices": [
                      {
                        "Variable": "$.error.Cause",
                        "StringMatches": "*ValidationError*",
                        "Next": "StackAlreadyDeleted"
                      }
                    ],
                    "Default": "DeletionFailed"
                  },
                  "DeleteStack": {
                    "Type": "Task",
                    "Resource": "arn:aws:states:::lambda:invoke",
                    "Parameters": {
                      "FunctionName": "${rCrossRegionStackOperationFunction}",
                      "Payload": {
                        "operation": "delete",
                        "name.$": "$.name",
                        "region.$": "$.region"
                      }
                    },
                    "ResultPath": "$.deleteResult",
                    "Next": "WaitForDeletion",
                    "Catch": [
                      {
                        "ErrorEquals": ["States.ALL"],
                        "ResultPath": "$.error",
                        "Next": "DeletionFailed"
                      }
                    ]
                  },
                  "WaitForDeletion": {
                    "Type": "Wait",
                    "Seconds": 30,
                    "Next": "CheckDeletionStatus"
                  },
                  "CheckDeletionStatus": {
                    "Type": "Task",
                    "Resource": "arn:aws:states:::lambda:invoke",
                    "Parameters": {
                      "FunctionName": "${rCrossRegionStackOperationFunction}",
                      "Payload": {
                        "operation": "status",
                        "name.$": "$.name",
                        "region.$": "$.region"
                      }
                    },
                    "ResultPath": "$.stackStatus",
                    "Next": "EvaluateDeletionStatus",
                    "Catch": [
                      {
                        "ErrorEquals": ["States.ALL"],
                        "ResultPath": "$.error",
                        "Next": "DeletionComplete"
                      }
                    ]
                  },
                  "EvaluateDeletionStatus": {
                    "Type": "Choice",
                    "Choices": [
                      {
                        "Variable": "$.stackStatus.Payload.status",
                        "StringEquals": "DELETE_IN_PROGRESS",
                        "Next": "WaitForDeletion"
                      },
                      {
                        "Variable": "$.stackStatus.Payload.status",
                        "StringEquals": "DELETE_COMPLETE",
                        "Next": "DeletionComplete"
                      }
                    ],
                    "Default": "DeletionFailed"
                  },
                  "DeletionComplete": {
                    "Type": "Pass",
                    "Result": {"status": "success"},
                    "End": true
                  },
                  "DeletionFailed": {
                    "Type": "Pass",
                    "Result": {"status": "failed"},
                    "End": true
                  },
                  "StackAlreadyDeleted": {
                    "Type": "Pass",
                    "Result": {"status": "success"},
                    "End": true
                  }
                }
              },
              "Next": "CheckDeletionResults"
            },
            "CheckDeletionResults": {
              "Type": "Task",
              "Resource": "arn:aws:states:::lambda:invoke",
              "Parameters": {
                "FunctionName": "${rDeletionResultCheckerFunction}",
                "Payload.$": "$.deletionResults"
              },
              "ResultPath": "$.allSuccessful",
              "Next": "EvaluateOverallSuccess"
            },
            "EvaluateOverallSuccess": {
              "Type": "Choice",
              "Choices": [
                {
                  "Variable": "$.allSuccessful.Payload.allSuccessful",
                  "BooleanEquals": true,
                  "Next": "CleanupCDKStagingBucket"
                }
              ],
              "Default": "SkipCleanup"
            },
            "SkipCleanup": {
              "Type": "Pass",
              "Result": "Stack deletion failed. Skipping Step Function and CDK bootstrap cleanup to allow retry.",
              "End": true
            },
            "CleanupCDKStagingBucket": {
              "Type": "Task",
              "Resource": "arn:aws:states:::lambda:invoke",
              "Parameters": {
                "FunctionName": "${rBucketCleanupFunction}",
                "Payload": {
                  "bucketName": "cdk-petsite-assets-${AWS::AccountId}-${AWS::Region}",
                  "deleteBucket": true
                }
              },
              "Next": "DeleteCDKToolkitStack"
            },
            "DeleteCDKToolkitStack": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:cloudformation:deleteStack",
              "Parameters": {
                "StackName": "CDKToolkitPetsite"
              },
              "Next": "WaitForCDKToolkitDeletion",
              "Catch": [
                {
                  "ErrorEquals": ["States.ALL"],
                  "Next": "CleanupComplete"
                }
              ]
            },
            "WaitForCDKToolkitDeletion": {
              "Type": "Wait",
              "Seconds": 30,
              "Next": "CheckCDKToolkitDeletionStatus"
            },
            "CheckCDKToolkitDeletionStatus": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:cloudformation:describeStacks",
              "Parameters": {
                "StackName": "CDKToolkitPetsite"
              },
              "ResultPath": "$.cdkStackStatus",
              "Next": "EvaluateCDKToolkitDeletionStatus",
              "Catch": [
                {
                  "ErrorEquals": ["CloudFormation.ValidationError", "CloudFormation.CloudFormationException"],
                  "Next": "CleanupComplete"
                }
              ]
            },
            "EvaluateCDKToolkitDeletionStatus": {
              "Type": "Choice",
              "Choices": [
                {
                  "Variable": "$.cdkStackStatus.Stacks[0].StackStatus",
                  "StringEquals": "DELETE_IN_PROGRESS",
                  "Next": "WaitForCDKToolkitDeletion"
                },
                {
                  "Variable": "$.cdkStackStatus.Stacks[0].StackStatus",
                  "StringEquals": "DELETE_COMPLETE",
                  "Next": "CleanupComplete"
                }
              ],
              "Default": "CleanupComplete"
            },
            "CleanupComplete": {
              "Type": "Task",
              "Resource": "arn:aws:states:::lambda:invoke",
              "Parameters": {
                "FunctionName": "${rCleanupCompletionFunction}",
                "Payload": {
                  "StackName": "${AWS::StackName}"
                }
              },
              "End": true
            }
          }
        }

  # IAM Role for Step Function
  rCDKCleanupRole:
    Type: AWS::IAM::Role
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: states.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: CDKCleanupPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - lambda:InvokeFunction
                Resource: !GetAtt rCDKStackListerFunction.Arn
              - Effect: Allow
                Action:
                  - cloudformation:DescribeStacks
                Resource: "*"
              - Effect: Allow
                Action:
                  - cloudformation:DeleteStack
                Resource: "*"
                Condition:
                  StringEquals:
                    "aws:ResourceTag/application": !Ref pApplicationName
              - Effect: Allow
                Action:
                  - cloudformation:DeleteStack
                  - cloudformation:DescribeStackResources
                Resource: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/CDKToolkitPetsite/*"
              - Effect: Allow
                Action:
                  - s3:ListBucket
                  - s3:ListBucketVersions
                  - s3:DeleteObject
                  - s3:DeleteObjectVersion
                  - s3:DeleteBucket
                  - s3:DeleteBucketPolicy
                  - s3:GetBucketPolicy
                Resource:
                  - !Sub 'arn:aws:s3:::cdk-petsite-assets-${AWS::AccountId}-${AWS::Region}'
                  - !Sub 'arn:aws:s3:::cdk-petsite-assets-${AWS::AccountId}-${AWS::Region}/*'
              - Effect: Allow
                Action:
                  - ecr:DeleteRepository
                  - ecr:BatchDeleteImage
                  - ecr:ListImages
                Resource: !Sub 'arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/cdk-petsite-container-assets-${AWS::AccountId}-${AWS::Region}'
              - Effect: Allow
                Action:
                  - iam:DeleteRole
                  - iam:DeleteRolePolicy
                  - iam:DetachRolePolicy
                  - iam:ListRolePolicies
                  - iam:ListAttachedRolePolicies
                  - iam:GetRole
                Resource: !Sub 'arn:aws:iam::${AWS::AccountId}:role/cdk-petsite-*'
              - Effect: Allow
                Action:
                  - ssm:DeleteParameter
                  - ssm:GetParameter
                Resource: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/cdk-bootstrap/petsite/*'
              - Effect: Allow
                Action:
                  - lambda:InvokeFunction
                Resource:
                  - !GetAtt rCleanupCompletionFunction.Arn
                  - !GetAtt rBucketCleanupFunction.Arn
                  - !GetAtt rDeletionResultCheckerFunction.Arn
                  - !GetAtt rCrossRegionStackOperationFunction.Arn

  # Lambda function to check deletion results
  rDeletionResultCheckerFunction:
    Type: AWS::Lambda::Function
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      FunctionName: !Sub ${AWS::StackName}-deletion-result-checker
      Runtime: python3.13
      Handler: index.handler
      Role: !GetAtt rDeletionResultCheckerRole.Arn
      Timeout: 30
      Code:
        ZipFile: |
          def handler(event, context):
              # Check if all deletions were successful
              all_successful = all(
                  result.get('status') == 'success'
                  for result in event
                  if isinstance(result, dict)
              )
              return {'allSuccessful': all_successful}

  # IAM Role for Deletion Result Checker Lambda
  rDeletionResultCheckerRole:
    Type: AWS::IAM::Role
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole

  # Lambda function to list CDK stacks by tag
  rCDKStackListerFunction:
    Type: AWS::Lambda::Function
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      FunctionName: !Sub ${AWS::StackName}-cdk-stack-lister
      Runtime: python3.13
      Handler: index.handler
      Role: !GetAtt rCDKStackListerRole.Arn
      Timeout: 60
      Code:
        ZipFile: !Sub |
          import boto3

          def handler(event, context):
              current_region = '${AWS::Region}'
              cf_client = boto3.client('cloudformation', region_name=current_region)
              stacks = []

              try:
                  # List stacks in current region
                  paginator = cf_client.get_paginator('list_stacks')
                  for page in paginator.paginate(StackStatusFilter=['CREATE_COMPLETE', 'CREATE_IN_PROGRESS', 'UPDATE_COMPLETE', 'DELETE_FAILED', 'ROLLBACK_COMPLETE', 'ROLLBACK_FAILED']):
                      for stack in page['StackSummaries']:
                          # Skip nested stacks
                          if 'ParentId' in stack:
                              continue

                          try:
                              # Get stack tags
                              stack_detail = cf_client.describe_stacks(StackName=stack['StackName'])
                              tags = stack_detail['Stacks'][0].get('Tags', [])

                              # Check if stack has the required application tag
                              has_app_tag = False
                              sequence_value = 1  # Default sequence value

                              for tag in tags:
                                  if tag['Key'] == 'application' and tag['Value'] == '${pApplicationName}':
                                      has_app_tag = True
                                  elif tag['Key'] == 'sequence':
                                      try:
                                          sequence_value = int(tag['Value'])
                                      except ValueError:
                                          sequence_value = 1

                              if has_app_tag:
                                  stacks.append({
                                      'name': stack['StackName'],
                                      'region': current_region,
                                      'sequence': sequence_value
                                  })
                          except Exception as e:
                              print(f"Error checking stack {stack['StackName']}: {str(e)}")
                              continue

                  # Check for us-east-1 support stack if not already in us-east-1
                  if current_region != 'us-east-1':
                      try:
                          cf_useast1 = boto3.client('cloudformation', region_name='us-east-1')
                          support_stack_name = 'OneObservability-support-us-east-1'
                          stack_detail = cf_useast1.describe_stacks(StackName=support_stack_name)
                          # Support stacks are created by CDK cross-region references and may not have tags
                          # Include them based on name pattern
                          stacks.append({
                              'name': support_stack_name,
                              'region': 'us-east-1',
                              'sequence': 0  # Delete after main stacks
                          })
                      except Exception as e:
                          print(f"No support stack in us-east-1 or error checking: {str(e)}")

                  # Sort stacks by sequence in descending order (highest first)
                  stacks.sort(key=lambda x: x['sequence'], reverse=True)

                  # Return stacks with region information
                  return {'stacks': stacks}
              except Exception as e:
                  print(f"Error listing stacks: {str(e)}")
                  return {'stacks': []}

  # IAM Role for CDK Stack Lister Lambda
  rCDKStackListerRole:
    Type: AWS::IAM::Role
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: CDKStackListerPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - cloudformation:ListStacks
                  - cloudformation:DescribeStacks
                Resource: "*"

  # Lambda function for cross-region stack operations
  rCrossRegionStackOperationFunction:
    Type: AWS::Lambda::Function
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      FunctionName: !Sub ${AWS::StackName}-cross-region-stack-ops
      Runtime: python3.13
      Handler: index.handler
      Role: !GetAtt rCrossRegionStackOperationRole.Arn
      Timeout: 60
      Code:
        ZipFile: |
          import boto3
          from datetime import datetime

          def handler(event, context):
              stack_name = event.get('name')
              region = event.get('region')
              operation = event.get('operation', 'describe')

              cf_client = boto3.client('cloudformation', region_name=region)

              try:
                  if operation == 'delete':
                      cf_client.delete_stack(StackName=stack_name)
                      return {'status': 'DELETE_IN_PROGRESS'}
                  elif operation == 'status':
                      response = cf_client.describe_stacks(StackName=stack_name)
                      return {'status': response['Stacks'][0]['StackStatus']}
                  else:
                      response = cf_client.describe_stacks(StackName=stack_name)
                      stack = response['Stacks'][0]
                      # Convert datetime objects to ISO format strings
                      if 'CreationTime' in stack:
                          stack['CreationTime'] = stack['CreationTime'].isoformat()
                      if 'LastUpdatedTime' in stack:
                          stack['LastUpdatedTime'] = stack['LastUpdatedTime'].isoformat()
                      if 'DeletionTime' in stack:
                          stack['DeletionTime'] = stack['DeletionTime'].isoformat()
                      return {'stack': stack}
              except cf_client.exceptions.ValidationError:
                  return {'status': 'DELETE_COMPLETE'}
              except Exception as e:
                  raise Exception(f"Error in {operation} operation for {stack_name} in {region}: {str(e)}")

  # IAM Role for Cross-Region Stack Operation Lambda
  rCrossRegionStackOperationRole:
    Type: AWS::IAM::Role
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: CrossRegionStackOperationPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - cloudformation:DescribeStacks
                  - cloudformation:DeleteStack
                Resource: !Sub "arn:${AWS::Partition}:cloudformation:*:${AWS::AccountId}:stack/*/*"

  # Lambda function to clean up retained resources after Step Function completes
  rCleanupCompletionFunction:
    Type: AWS::Lambda::Function
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      FunctionName: !Sub ${AWS::StackName}-cleanup-completion
      Runtime: python3.13
      Handler: index.handler
      Role: !GetAtt rCleanupCompletionRole.Arn
      Timeout: 300
      Code:
        ZipFile: !Sub |
          import boto3
          import time

          def handler(event, context):
              stack_name = event.get('StackName')
              if not stack_name:
                  print("No stack name provided")
                  return

              time.sleep(30)

              try:
                  lambda_client = boto3.client('lambda')
                  sf_client = boto3.client('stepfunctions')
                  events_client = boto3.client('events')
                  iam_client = boto3.client('iam')

                  # Delete Lambda functions
                  for func_name in [
                      f"{stack_name}-cdk-stack-lister",
                      f"{stack_name}-deletion-result-checker",
                      f"{stack_name}-bucket-cleanup",
                      f"{stack_name}-cleanup-monitor"
                  ]:
                      try:
                          lambda_client.delete_function(FunctionName=func_name)
                          print(f"Deleted Lambda: {func_name}")
                      except Exception as e:
                          print(f"Error deleting {func_name}: {str(e)}")

                  # Delete EventBridge rules
                  try:
                      rules = events_client.list_rules()['Rules']
                      for rule in rules:
                          if stack_name in rule.get('Description', '') or stack_name in rule['Name']:
                              try:
                                  targets = events_client.list_targets_by_rule(Rule=rule['Name'])
                                  if targets['Targets']:
                                      events_client.remove_targets(Rule=rule['Name'], Ids=[t['Id'] for t in targets['Targets']])
                                  events_client.delete_rule(Name=rule['Name'])
                                  print(f"Deleted EventBridge rule: {rule['Name']}")
                              except Exception as e:
                                  print(f"Error deleting rule {rule['Name']}: {str(e)}")
                  except Exception as e:
                      print(f"Error deleting EventBridge rules: {str(e)}")

                  # Delete IAM roles
                  for role_name in [
                      f"{stack_name}-cdk-cleanup-role",
                      f"{stack_name}-cdk-stack-lister-role",
                      f"{stack_name}-deletion-result-checker-role",
                      f"{stack_name}-bucket-cleanup-role",
                      f"{stack_name}-cleanup-completion-role",
                      f"{stack_name}-cleanup-monitor-role",
                      f"{stack_name}-eventbridge-role"
                  ]:
                      try:
                          iam_client.list_attached_role_policies(RoleName=role_name)
                          for policy in iam_client.list_attached_role_policies(RoleName=role_name)['AttachedPolicies']:
                              iam_client.detach_role_policy(RoleName=role_name, PolicyArn=policy['PolicyArn'])
                          for policy in iam_client.list_role_policies(RoleName=role_name)['PolicyNames']:
                              iam_client.delete_role_policy(RoleName=role_name, PolicyName=policy)
                          iam_client.delete_role(RoleName=role_name)
                          print(f"Deleted IAM role: {role_name}")
                      except Exception as e:
                          print(f"Error deleting role {role_name}: {str(e)}")

                  # Delete Step Function
                  try:
                      sf_arn = f"arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:{stack_name}-cdk-cleanup"
                      sf_client.delete_state_machine(stateMachineArn=sf_arn)
                      print(f"Deleted Step Function: {sf_arn}")
                  except Exception as e:
                      print(f"Error deleting Step Function: {str(e)}")

                  # Delete this function itself
                  try:
                      lambda_client.delete_function(FunctionName=f"{stack_name}-cleanup-completion")
                      print("Deleted cleanup completion function")
                  except Exception as e:
                      print(f"Error deleting cleanup completion function: {str(e)}")

              except Exception as e:
                  print(f"Error in cleanup completion: {str(e)}")

  # IAM Role for Cleanup Completion Lambda
  rCleanupCompletionRole:
    Type: AWS::IAM::Role
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: CleanupCompletionPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              # Step Function deletion - specific to this stack's state machine
              - Effect: Allow
                Action:
                  - states:DeleteStateMachine
                Resource: !Sub "arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${AWS::StackName}-cdk-cleanup"
              # Lambda function deletion - specific to this stack's functions
              - Effect: Allow
                Action:
                  - lambda:DeleteFunction
                Resource:
                  - !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${AWS::StackName}-*"
              # EventBridge rules - list and manage rules for this stack
              - Effect: Allow
                Action:
                  - events:ListRules
                Resource: "*"
              - Effect: Allow
                Action:
                  - events:ListTargetsByRule
                  - events:RemoveTargets
                  - events:DeleteRule
                Resource: !Sub "arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/*"
                Condition:
                  StringLike:
                    "events:source": "aws.cloudformation"
              # IAM role management - only for roles created by this stack
              - Effect: Allow
                Action:
                  - iam:DeleteRole
                  - iam:DetachRolePolicy
                  - iam:DeleteRolePolicy
                  - iam:ListAttachedRolePolicies
                  - iam:ListRolePolicies
                Resource: !Sub "arn:aws:iam::${AWS::AccountId}:role/${AWS::StackName}-*"

  # Lambda function to clean up S3 buckets
  rBucketCleanupFunction:
    Type: AWS::Lambda::Function
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      FunctionName: !Sub ${AWS::StackName}-bucket-cleanup
      Runtime: python3.13
      Handler: index.handler
      Role: !GetAtt rBucketCleanupRole.Arn
      Timeout: 300
      Code:
        ZipFile: |
          import boto3

          def handler(event, context):
              s3_client = boto3.client('s3')
              bucket_name = event.get('bucketName')
              delete_bucket = event.get('deleteBucket', False)

              if not bucket_name:
                  return {'statusCode': 400, 'body': 'Bucket name required'}

              try:
                  try:
                      s3_client.head_bucket(Bucket=bucket_name)
                      print(f"Found bucket: {bucket_name}")
                  except s3_client.exceptions.NoSuchBucket:
                      print(f"Bucket {bucket_name} does not exist")
                      return {'statusCode': 200, 'body': 'Bucket does not exist'}

                  paginator = s3_client.get_paginator('list_object_versions')
                  delete_keys = []

                  for page in paginator.paginate(Bucket=bucket_name):
                      if 'Versions' in page:
                          delete_keys.extend([{'Key': obj['Key'], 'VersionId': obj['VersionId']} for obj in page['Versions']])
                      if 'DeleteMarkers' in page:
                          delete_keys.extend([{'Key': obj['Key'], 'VersionId': obj['VersionId']} for obj in page['DeleteMarkers']])

                  if delete_keys:
                      for i in range(0, len(delete_keys), 1000):
                          s3_client.delete_objects(
                              Bucket=bucket_name,
                              Delete={'Objects': delete_keys[i:i+1000]}
                          )
                      print(f"Deleted {len(delete_keys)} objects from {bucket_name}")

                  if delete_bucket:
                      s3_client.delete_bucket(Bucket=bucket_name)
                      print(f"Deleted bucket: {bucket_name}")

                  return {'statusCode': 200, 'body': f'Successfully cleaned up {bucket_name}'}
              except Exception as e:
                  print(f"Error cleaning up bucket: {str(e)}")
                  return {'statusCode': 500, 'body': f'Error: {str(e)}'}

  # IAM Role for Bucket Cleanup Lambda
  rBucketCleanupRole:
    Type: AWS::IAM::Role
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: BucketCleanupPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3:ListBucket
                  - s3:ListBucketVersions
                  - s3:DeleteObject
                  - s3:DeleteObjectVersion
                  - s3:DeleteBucket
                Resource:
                  - !Sub 'arn:aws:s3:::cdk-petsite-assets-${AWS::AccountId}-${AWS::Region}'
                  - !Sub 'arn:aws:s3:::cdk-petsite-assets-${AWS::AccountId}-${AWS::Region}/*'


  # Custom resource to monitor Step Function execution during stack deletion
  rCleanupMonitor:
    Type: AWS::CloudFormation::CustomResource
    DependsOn:
      - rCDKStackListerFunction
      - rDeletionResultCheckerFunction
      - rCrossRegionStackOperationFunction
      - rBucketCleanupFunction
      - rCleanupCompletionFunction
    Properties:
      ServiceToken: !GetAtt rCleanupMonitorFunction.Arn
      StateMachineArn: !GetAtt rCDKCleanupStateMachine.Arn

  # Lambda function to monitor cleanup execution
  rCleanupMonitorFunction:
    Type: AWS::Lambda::Function
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      FunctionName: !Sub ${AWS::StackName}-cleanup-monitor
      Runtime: python3.13
      Handler: index.handler
      Role: !GetAtt rCleanupMonitorRole.Arn
      Timeout: 900
      Code:
        ZipFile: |
          import boto3
          import cfnresponse
          import time

          def handler(event, context):
              if event['RequestType'] in ['Create', 'Update']:
                  cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
                  return

              # Delete operation - monitor Step Function
              sf_client = boto3.client('stepfunctions')
              state_machine_arn = event['ResourceProperties']['StateMachineArn']

              try:
                  # Start the Step Function
                  response = sf_client.start_execution(
                      stateMachineArn=state_machine_arn,
                      input='{}'
                  )
                  execution_arn = response['executionArn']
                  print(f"Started Step Function execution: {execution_arn}")

                  # Poll for completion
                  max_wait = 840  # 14 minutes
                  start_time = time.time()

                  while time.time() - start_time < max_wait:
                      exec_response = sf_client.describe_execution(executionArn=execution_arn)
                      status = exec_response['status']

                      if status == 'SUCCEEDED':
                          print("Step Function completed successfully")
                          cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
                          return
                      elif status in ['FAILED', 'TIMED_OUT', 'ABORTED']:
                          error_msg = f"Step Function failed with status: {status}"
                          print(error_msg)
                          cfnresponse.send(event, context, cfnresponse.FAILED, {}, error_msg)
                          return

                      time.sleep(10)

                  # Timeout
                  cfnresponse.send(event, context, cfnresponse.FAILED, {}, "Step Function execution timeout")
              except Exception as e:
                  print(f"Error monitoring Step Function: {str(e)}")
                  cfnresponse.send(event, context, cfnresponse.FAILED, {}, str(e))

  # IAM Role for Cleanup Monitor Lambda
  rCleanupMonitorRole:
    Type: AWS::IAM::Role
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: StepFunctionMonitorPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - states:StartExecution
                Resource: !GetAtt rCDKCleanupStateMachine.Arn
              - Effect: Allow
                Action:
                  - states:DescribeExecution
                Resource: !Sub "arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${AWS::StackName}-cdk-cleanup:*"

  # EventBridge Rule for Stack Deletion
  rStackDeletionRule:
    Type: AWS::Events::Rule
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      Description: Trigger cleanup when stack is being deleted
      EventPattern:
        source: ["aws.cloudformation"]
        detail-type: ["CloudFormation Stack Status Change"]
        detail:
          stack-id: [!Ref "AWS::StackId"]
          status-details:
            status: ["DELETE_IN_PROGRESS"]
      State: ENABLED
      Targets:
        - Arn: !GetAtt rCDKCleanupStateMachine.Arn
          Id: "CDKCleanupTarget"
          RoleArn: !GetAtt rEventBridgeRole.Arn


  # IAM Role for EventBridge to invoke Step Function
  rEventBridgeRole:
    Type: AWS::IAM::Role
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: events.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: StepFunctionExecutionPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - states:StartExecution
                Resource: !GetAtt rCDKCleanupStateMachine.Arn


Outputs:
  oConfigBucketName:
    Description: Name of the S3 bucket storing configuration files
    Value: !Ref rConfigBucket
    Export:
      Name: !Sub ${AWS::StackName}-ConfigBucketName

  oCodeBuildProjectName:
    Description: Name of the CodeBuild project orchestrating CDK deployment
    Value: !Ref rCDKDeploymentProject
    Export:
      Name: !Sub ${AWS::StackName}-CodeBuildProjectName

  oDeploymentStatus:
    Description: Status of the CDK deployment
    Value: !Sub Check CodeBuild project ${rCDKDeploymentProject} for deployment status
    Export:
      Name: !Sub ${AWS::StackName}-DeploymentStatus

  oCDKCleanupStateMachine:
    Description: ARN of the Step Function that handles CDK stack cleanup
    Value: !GetAtt rCDKCleanupStateMachine.Arn
    Export:
      Name: !Sub ${AWS::StackName}-CDKCleanupStateMachine

  oRepositoryInfo:
    Description: Repository information used for deployment
    Value: !Sub 'Organization: ${pOrganizationName}, Repository: ${pRepositoryName},
      Branch: ${pBranchName}'
    Export:
      Name: !Sub ${AWS::StackName}-RepositoryInfo

  oPetSiteUrl:
    Description: PetSite URL from CDK stack
    Value: !GetAtt rCDKOutputs.PetSiteUrl

  oExportsDashboardUrl:
    Description: URL to access the CDK exports dashboard
    Value: !GetAtt rCDKOutputs.ExportsDashboardUrl
    Export:
      Name: !Sub ${AWS::StackName}-ExportsDashboardUrl