Improve AWS runner attribution and watchdog coverage

.github/workflows/aws_gpu_benchmarks.yml

@@ -34,6 +34,10 @@ on:
         required: false
         type: string
         default: ''
+      trigger-category:
+        description: 'Normalized workflow trigger category'
+        required: true
+        type: string
     secrets:
       GH_PERSONAL_ACCESS_TOKEN:
         required: true
@@ -95,7 +99,11 @@ jobs:
             [
               {"Key": "Name", "Value": "ec2-github-runner"},
               {"Key": "created-by", "Value": "github-actions-newton-role"},
-              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"}
+              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"},
+              {"Key": "Newton-Trigger", "Value": ${{ toJSON(inputs['trigger-category']) }}},
+              {"Key": "Newton-Workload", "Value": "gpu-benchmarks"},
+              {"Key": "GitHub-Run-ID", "Value": "${{ github.run_id }}"},
+              {"Key": "GitHub-Run-Attempt", "Value": "${{ github.run_attempt }}"}
             ]
 
   gpu-benchmarks:

.github/workflows/aws_gpu_tests.yml

@@ -42,6 +42,10 @@ on:
         required: false
         type: string
         default: ''
+      trigger-category:
+        description: 'Normalized workflow trigger category'
+        required: true
+        type: string
     secrets:
       GH_PERSONAL_ACCESS_TOKEN:
         required: true
@@ -59,6 +63,11 @@ on:
         required: false
         type: string
         default: ''
+      trigger-category:
+        description: 'Normalized workflow trigger category'
+        required: false
+        type: string
+        default: 'manual'
 
 jobs:
   start-runner:
@@ -117,7 +126,11 @@ jobs:
             [
               {"Key": "Name", "Value": "ec2-github-runner"},
               {"Key": "created-by", "Value": "github-actions-newton-role"},
-              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"}
+              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"},
+              {"Key": "Newton-Trigger", "Value": ${{ toJSON(inputs['trigger-category']) }}},
+              {"Key": "Newton-Workload", "Value": "gpu-unit-tests"},
+              {"Key": "GitHub-Run-ID", "Value": "${{ github.run_id }}"},
+              {"Key": "GitHub-Run-Attempt", "Value": "${{ github.run_attempt }}"}
             ]
 
   gpu-unit-tests:

.github/workflows/merge_queue_aws_gpu.yml

@@ -10,6 +10,7 @@ jobs:
     uses: ./.github/workflows/aws_gpu_tests.yml
     with:
       ref: ${{ github.sha }}
+      trigger-category: merge-queue
     secrets: inherit
     permissions:
       id-token: write   # Required for AWS OIDC authentication in start-runner/stop-runner

.github/workflows/minimum_deps_tests.yml

@@ -17,12 +17,23 @@ env:
 
 on:
   workflow_call:
+    inputs:
+      trigger-category:
+        description: 'Normalized workflow trigger category'
+        required: true
+        type: string
     secrets:
       GH_PERSONAL_ACCESS_TOKEN:
         required: true
       CODECOV_TOKEN:
         required: true
   workflow_dispatch:
+    inputs:
+      trigger-category:
+        description: 'Normalized workflow trigger category'
+        required: false
+        type: string
+        default: 'manual'
 
 jobs:
   start-runner:
@@ -81,7 +92,11 @@ jobs:
             [
               {"Key": "Name", "Value": "ec2-github-runner"},
               {"Key": "created-by", "Value": "github-actions-newton-role"},
-              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"}
+              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"},
+              {"Key": "Newton-Trigger", "Value": ${{ toJSON(inputs['trigger-category']) }}},
+              {"Key": "Newton-Workload", "Value": "minimum-deps-tests"},
+              {"Key": "GitHub-Run-ID", "Value": "${{ github.run_id }}"},
+              {"Key": "GitHub-Run-Attempt", "Value": "${{ github.run_attempt }}"}
             ]
 
   minimum-deps-tests:

.github/workflows/mujoco_warp_tests.yml

@@ -16,12 +16,23 @@ env:
 
 on:
   workflow_call:
+    inputs:
+      trigger-category:
+        description: 'Normalized workflow trigger category'
+        required: true
+        type: string
     secrets:
       GH_PERSONAL_ACCESS_TOKEN:
         required: true
       CODECOV_TOKEN:
         required: true
   workflow_dispatch:
+    inputs:
+      trigger-category:
+        description: 'Normalized workflow trigger category'
+        required: false
+        type: string
+        default: 'manual'
 
 jobs:
   start-runner:
@@ -80,7 +91,11 @@ jobs:
             [
               {"Key": "Name", "Value": "ec2-github-runner"},
               {"Key": "created-by", "Value": "github-actions-newton-role"},
-              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"}
+              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"},
+              {"Key": "Newton-Trigger", "Value": ${{ toJSON(inputs['trigger-category']) }}},
+              {"Key": "Newton-Workload", "Value": "mujoco-warp-tests"},
+              {"Key": "GitHub-Run-ID", "Value": "${{ github.run_id }}"},
+              {"Key": "GitHub-Run-Attempt", "Value": "${{ github.run_attempt }}"}
             ]
 
   mujoco-warp-tests:

.github/workflows/pr_target_aws_gpu_benchmarks.yml

@@ -111,6 +111,7 @@ jobs:
     with:
       ref: ${{ github.event.pull_request.head.sha }}
       base_ref: ${{ github.event.pull_request.base.sha }}
+      trigger-category: pull-request
     secrets: inherit
     permissions:
       id-token: write   # Required for AWS OIDC authentication in start-runner/stop-runner

.github/workflows/pr_target_aws_gpu_tests.yml

@@ -110,6 +110,7 @@ jobs:
     uses: ./.github/workflows/aws_gpu_tests.yml
     with:
       ref: ${{ github.event.pull_request.head.sha }}
+      trigger-category: pull-request
     secrets: inherit
     permissions:
       id-token: write   # Required for AWS OIDC authentication in start-runner/stop-runner

.github/workflows/push_aws_gpu.yml

@@ -15,6 +15,8 @@ jobs:
   run-tests:
     if: github.repository == 'newton-physics/newton'
     uses: ./.github/workflows/aws_gpu_tests.yml
+    with:
+      trigger-category: push
     secrets: inherit
     permissions:
       id-token: write   # Required for AWS OIDC authentication in start-runner/stop-runner

.github/workflows/scheduled_nightly.yml

@@ -90,16 +90,16 @@ jobs:
 
       - name: Dispatch and wait for GPU tests
         id: gpu-tests
-        run: uv run --no-project scripts/ci/dispatch_workflow_and_wait.py aws_gpu_tests.yml -f "inputs[instance-type]=g7e.12xlarge"
+        run: uv run --no-project scripts/ci/dispatch_workflow_and_wait.py aws_gpu_tests.yml -f "inputs[instance-type]=g7e.12xlarge" -f "inputs[trigger-category]=scheduled-nightly"
 
       - name: Dispatch and wait for Minimum Deps tests
         id: minimum-deps-tests
-        run: uv run --no-project scripts/ci/dispatch_workflow_and_wait.py minimum_deps_tests.yml
+        run: uv run --no-project scripts/ci/dispatch_workflow_and_wait.py minimum_deps_tests.yml -f "inputs[trigger-category]=scheduled-nightly"
 
       - name: Dispatch and wait for Warp Nightly tests
         id: warp-nightly-tests
         if: needs.check-warp-update.result == 'success' && needs.check-warp-update.outputs.warp-updated == 'true'
-        run: uv run --no-project scripts/ci/dispatch_workflow_and_wait.py warp_nightly_tests.yml
+        run: uv run --no-project scripts/ci/dispatch_workflow_and_wait.py warp_nightly_tests.yml -f "inputs[trigger-category]=scheduled-nightly"
 
   notify-on-failure:
     name: Notify on failure

.github/workflows/warp_nightly_tests.yml

@@ -16,12 +16,23 @@ env:
 
 on:
   workflow_call:
+    inputs:
+      trigger-category:
+        description: 'Normalized workflow trigger category'
+        required: true
+        type: string
     secrets:
       GH_PERSONAL_ACCESS_TOKEN:
         required: true
       CODECOV_TOKEN:
         required: true
   workflow_dispatch:
+    inputs:
+      trigger-category:
+        description: 'Normalized workflow trigger category'
+        required: false
+        type: string
+        default: 'manual'
 
 jobs:
   start-runner:
@@ -80,7 +91,11 @@ jobs:
             [
               {"Key": "Name", "Value": "ec2-github-runner"},
               {"Key": "created-by", "Value": "github-actions-newton-role"},
-              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"}
+              {"Key": "GitHub-Repository", "Value": "${{ github.repository }}"},
+              {"Key": "Newton-Trigger", "Value": ${{ toJSON(inputs['trigger-category']) }}},
+              {"Key": "Newton-Workload", "Value": "warp-nightly-tests"},
+              {"Key": "GitHub-Run-ID", "Value": "${{ github.run_id }}"},
+              {"Key": "GitHub-Run-Attempt", "Value": "${{ github.run_attempt }}"}
             ]
 
   warp-nightly-tests:

scripts/ci/aws/README.md

@@ -0,0 +1,104 @@
+# AWS CI infrastructure
+
+This directory contains the source templates for AWS resources used by Newton
+CI. If the templates exist only in AWS, maintainers have to reconstruct the
+deployed configuration from console history during an incident. Checking them
+into Git gives us a reviewable change history and a known version to restore.
+
+That history is why these files belong in the repository. They are source files,
+not examples or generated output. The templates contain no credentials,
+secrets, private endpoints, or notification subscribers. They do not grant
+access to AWS, and commits to this directory do not deploy automatically.
+
+Put future AWS CI templates here too. This gives the small group that maintains
+the infrastructure one place to look and keeps AWS details out of the rest of
+the project.
+
+## Inventory
+
+| Template | Stack | Region | Required parameters |
+| --- | --- | --- | --- |
+| `overdue-newton-github-runner-watchdog.yaml` | `overdue-newton-github-runner-watchdog` | `us-east-1` | `AlertTopicArn` |
+
+The deployer must supply `AlertTopicArn` because the template has no default.
+This keeps notification routing and subscriber details outside the repository.
+
+## Validate and review a change
+
+Authenticate the AWS CLI, check that the selected profile points to the intended
+account, then validate the template:
+
+```bash
+NEWTON_AWS_PROFILE=isaac-sim-CS-Admin
+NEWTON_AWS_REGION=us-east-1
+NEWTON_WATCHDOG_STACK=overdue-newton-github-runner-watchdog
+NEWTON_CHANGE_SET=aws-runner-watchdog-update
+
+aws --profile "$NEWTON_AWS_PROFILE" sts get-caller-identity
+aws --profile "$NEWTON_AWS_PROFILE" cloudformation validate-template \
+  --region "$NEWTON_AWS_REGION" \
+  --template-body file://scripts/ci/aws/overdue-newton-github-runner-watchdog.yaml
+```
+
+Before creating a change set, save the stack's current template and parameters
+outside the repository in case you need to roll back. Read the alert action on
+the deployed alarm and verify that it points to the intended SNS topic:
+
+```bash
+NEWTON_ALERT_TOPIC_ARN="$(
+  aws --profile "$NEWTON_AWS_PROFILE" cloudwatch describe-alarms \
+    --region "$NEWTON_AWS_REGION" \
+    --alarm-names overdue-newton-github-runner-watchdog \
+    --query 'MetricAlarms[0].AlarmActions[0]' \
+    --output text
+)"
+
+case "$NEWTON_ALERT_TOPIC_ARN" in
+  arn:aws:sns:us-east-1:*) ;;
+  *) echo "Unexpected alert topic ARN" >&2; exit 1 ;;
+esac
+
+aws --profile "$NEWTON_AWS_PROFILE" cloudformation create-change-set \
+  --region "$NEWTON_AWS_REGION" \
+  --stack-name "$NEWTON_WATCHDOG_STACK" \
+  --change-set-name "$NEWTON_CHANGE_SET" \
+  --change-set-type UPDATE \
+  --template-body file://scripts/ci/aws/overdue-newton-github-runner-watchdog.yaml \
+  --parameters "ParameterKey=AlertTopicArn,ParameterValue=$NEWTON_ALERT_TOPIC_ARN" \
+  --capabilities CAPABILITY_NAMED_IAM
+
+aws --profile "$NEWTON_AWS_PROFILE" cloudformation wait change-set-create-complete \
+  --region "$NEWTON_AWS_REGION" \
+  --stack-name "$NEWTON_WATCHDOG_STACK" \
+  --change-set-name "$NEWTON_CHANGE_SET"
+
+aws --profile "$NEWTON_AWS_PROFILE" cloudformation describe-change-set \
+  --region "$NEWTON_AWS_REGION" \
+  --stack-name "$NEWTON_WATCHDOG_STACK" \
+  --change-set-name "$NEWTON_CHANGE_SET"
+```
+
+Read the complete change set before executing it. This stack has a named IAM
+role, so the update requires `CAPABILITY_NAMED_IAM`. Pay close attention to IAM
+changes, and stop if the change set contains an unexpected replacement,
+deletion, resource, or permission change.
+
+Execute only after review:
+
+```bash
+aws --profile "$NEWTON_AWS_PROFILE" cloudformation execute-change-set \
+  --region "$NEWTON_AWS_REGION" \
+  --stack-name "$NEWTON_WATCHDOG_STACK" \
+  --change-set-name "$NEWTON_CHANGE_SET"
+
+aws --profile "$NEWTON_AWS_PROFILE" cloudformation wait stack-update-complete \
+  --region "$NEWTON_AWS_REGION" \
+  --stack-name "$NEWTON_WATCHDOG_STACK"
+```
+
+After deployment, invoke the watchdog, confirm that it emits fresh metrics and
+logs, then check the alarm state. Investigate any stack drift before making
+another change. If verification fails, restore the saved template and parameters
+with a reviewed reverse change set. Do not update stack-managed resources
+directly because CloudFormation will no longer have an accurate record of their
+configuration.