From 863c461e27113c508a9b8f40348e9efb1871dfba Mon Sep 17 00:00:00 2001
From: Jesse Turner <ajesstur@amazon.com>
Date: Wed, 22 Jul 2026 22:32:19 +0000
Subject: [PATCH] ci: emit canary heartbeat metric for external failure alarm

Every canary run now emits a Success=1|0 heartbeat metric to CloudWatch
via an if: always() step. An alarm in the canary's AWS account watches
the metric and alerts the team when the canary fails or stops reporting.

The alarm treats missing data as breaching, so a heartbeat (rather than
a failure-triggered push) also covers failure modes that produce no
signal at all: runner never starts, broken credentials, silently
disabled cron.
---
 .github/scripts/emit-canary-heartbeat.sh | 39 ++++++++++++++++++++++++
 .github/workflows/canary.yml             | 12 ++++++++
 2 files changed, 51 insertions(+)
 create mode 100755 .github/scripts/emit-canary-heartbeat.sh

diff --git a/.github/scripts/emit-canary-heartbeat.sh b/.github/scripts/emit-canary-heartbeat.sh
new file mode 100755
index 000000000..2abb99283
--- /dev/null
+++ b/.github/scripts/emit-canary-heartbeat.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Emits a per-run heartbeat metric for the canary workflow. A CloudWatch
+# alarm in the AWS account the canary deploys to watches this metric and
+# alerts the team when the canary fails or stops reporting.
+#
+# Called from canary.yml with `if: always()`: every matrix cell reports
+# Success=1 or Success=0 on every run. The alarm treats missing data as
+# breaching, so silence (dead runner, broken credentials, disabled cron)
+# trips it just like an explicit failure does — which is why this step
+# must never be made conditional on failure only.
+#
+# Usage: emit-canary-heartbeat.sh <job-status>
+#   job-status: GitHub's ${{ job.status }} — "success", "failure", or "cancelled"
+set -euo pipefail
+
+# The alarm lives in us-east-1. Always emit there — even when the canary is
+# manually dispatched against another region — so an off-region run can't
+# starve the alarm into a false missing-data breach.
+readonly MONITOR_REGION=us-east-1
+readonly NAMESPACE=AgentCoreCLI/Canary
+readonly METRIC_NAME=Success
+
+STATUS="${1:?usage: emit-canary-heartbeat.sh <job-status>}"
+
+# Anything other than success (failure, cancelled/timeout) counts as a failed
+# canary run.
+if [[ "$STATUS" == "success" ]]; then
+  VALUE=1
+else
+  VALUE=0
+fi
+
+aws cloudwatch put-metric-data \
+  --region "$MONITOR_REGION" \
+  --namespace "$NAMESPACE" \
+  --metric-name "$METRIC_NAME" \
+  --value "$VALUE"
+
+echo "Emitted ${NAMESPACE}/${METRIC_NAME}=${VALUE} (job status: ${STATUS}) to ${MONITOR_REGION}"
diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml
index 966f9b2b4..941662eac 100644
--- a/.github/workflows/canary.yml
+++ b/.github/workflows/canary.yml
@@ -3,6 +3,9 @@ name: Canary
 on:
   # Run every 15 minutes to catch dependency/service breakages (e.g. CDK
   # schema mismatches) quickly instead of up to an hour later.
+  # NOTE: a CloudWatch alarm in the canary's AWS account expects a heartbeat
+  # metric at this cadence (see the "Emit canary heartbeat" step) — if you
+  # change this schedule, update the alarm period to match.
   schedule:
     - cron: '*/15 * * * *'
   workflow_dispatch:
@@ -96,6 +99,15 @@ jobs:
         # Smoke test for now, only runs strands-bedrock.test.ts
         run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts --retry 3
 
+      - name: Emit canary heartbeat
+        # Every run emits AgentCoreCLI/Canary :: Success = 1|0, watched by a
+        # CloudWatch alarm in the canary's AWS account. The alarm treats
+        # MISSING data as breaching, so this step must run on success AND
+        # failure — and if it stops running entirely, that absence is itself
+        # what fires the alarm.
+        if: always()
+        run: .github/scripts/emit-canary-heartbeat.sh "${{ job.status }}"
+
       - name: Generate GitHub App Token
         if: failure()
         id: app-token