From dd386778736be51f514c58856a4cf1974db61970 Mon Sep 17 00:00:00 2001 From: Dillon Chen <5334557+dillchen@users.noreply.github.com> Date: Fri, 2 Jan 2026 01:11:25 -0500 Subject: [PATCH] Add observability-to-Seer auto-fix workflow --- .../workflows/observability-seer-autofix.yml | 91 +++++++++++++++++++ common_knowledge/Observability-AI-Workflow.md | 61 +++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 .github/workflows/observability-seer-autofix.yml create mode 100644 common_knowledge/Observability-AI-Workflow.md diff --git a/.github/workflows/observability-seer-autofix.yml b/.github/workflows/observability-seer-autofix.yml new file mode 100644 index 00000000000..a13bc1b16ca --- /dev/null +++ b/.github/workflows/observability-seer-autofix.yml @@ -0,0 +1,91 @@ +name: Observability ➜ Seer autofix + +on: + repository_dispatch: + types: [observability_error] + workflow_dispatch: + inputs: + source: + description: Source of alert (datadog, sentry, etc) + default: manual + required: true + title: + description: Alert title/summary + required: true + severity: + description: Severity level + default: warning + required: true + message: + description: Full alert text or description + required: true + link: + description: Link back to observability system (Datadog/Sentry/etc) + required: false + +jobs: + forward-to-seer: + runs-on: ubuntu-latest + steps: + - name: Validate required secrets + run: | + if [[ -z "${{ secrets.SEER_WEBHOOK_URL }}" ]]; then + echo "Missing SEER_WEBHOOK_URL secret." + exit 1 + fi + if [[ -z "${{ secrets.SEER_WEBHOOK_TOKEN }}" ]]; then + echo "Missing SEER_WEBHOOK_TOKEN secret." + exit 1 + fi + + - name: Prepare payload + id: payload + env: + SOURCE: ${{ github.event.client_payload.source || github.event.inputs.source || 'unknown' }} + TITLE: ${{ github.event.client_payload.title || github.event.inputs.title || 'Unknown alert' }} + SEVERITY: ${{ github.event.client_payload.severity || github.event.inputs.severity || 'unknown' }} + MESSAGE: ${{ github.event.client_payload.message || github.event.inputs.message || '' }} + LINK: ${{ github.event.client_payload.link || github.event.inputs.link || secrets.OBSERVABILITY_SOURCE_URL || '' }} + SERVICE: ${{ github.event.client_payload.service || '' }} + HOST: ${{ github.event.client_payload.host || '' }} + TIMESTAMP: ${{ github.event.client_payload.timestamp || '' }} + COMMIT_SHA: ${{ github.sha }} + REPO: ${{ github.repository }} + run: | + cat > seer-payload.json <<'JSON' + { + "source": "${SOURCE}", + "title": "${TITLE}", + "severity": "${SEVERITY}", + "message": "${MESSAGE}", + "link": "${LINK}", + "service": "${SERVICE}", + "host": "${HOST}", + "timestamp": "${TIMESTAMP}", + "repository": "${REPO}", + "commit": "${COMMIT_SHA}" + } + JSON + echo "payload_path=seer-payload.json" >> "$GITHUB_OUTPUT" + + - name: Send to Seer/Codex endpoint + env: + SEER_WEBHOOK_URL: ${{ secrets.SEER_WEBHOOK_URL }} + SEER_WEBHOOK_TOKEN: ${{ secrets.SEER_WEBHOOK_TOKEN }} + PAYLOAD_PATH: ${{ steps.payload.outputs.payload_path }} + run: | + echo "Dispatching payload to Seer endpoint..." + curl -sSfL -X POST \ + -H "Authorization: Bearer ${SEER_WEBHOOK_TOKEN}" \ + -H "Content-Type: application/json" \ + --data @"${PAYLOAD_PATH}" \ + "${SEER_WEBHOOK_URL}" + + - name: Summary + run: | + echo "Alert forwarded to Seer/Codex endpoint." + echo "Source: ${{ github.event.client_payload.source || github.event.inputs.source || 'unknown' }}" + echo "Title: ${{ github.event.client_payload.title || github.event.inputs.title || 'Unknown alert' }}" + if [[ -n "${{ github.event.client_payload.link || github.event.inputs.link || secrets.OBSERVABILITY_SOURCE_URL }}" ]]; then + echo "Link: ${{ github.event.client_payload.link || github.event.inputs.link || secrets.OBSERVABILITY_SOURCE_URL }}" + fi diff --git a/common_knowledge/Observability-AI-Workflow.md b/common_knowledge/Observability-AI-Workflow.md new file mode 100644 index 00000000000..d15300f94dc --- /dev/null +++ b/common_knowledge/Observability-AI-Workflow.md @@ -0,0 +1,61 @@ +## Observability → Seer → Claude workflow + +This repo already uses Datadog for runtime observability (see [`common_knowledge/Datadog.md`](./Datadog.md)). This guide and accompanying GitHub Action wire that signal into an AI-driven remediation loop inspired by the "Sentry + Seer + Claude" workflow described in the shared post. + +### What you get +- Datadog (or Sentry, if enabled) raises an error and calls a GitHub repository dispatch webhook. +- The new `observability-seer-autofix.yml` workflow forwards that payload to Seer (or any compatible webhook endpoint) so it can attempt a fix and open a PR. +- Once the PR is opened, the Claude Code Review GitHub App can automatically review it. + +### 1) Configure secrets in GitHub +Add these to the repo’s **Settings → Secrets and variables → Actions**: +- `SEER_WEBHOOK_URL`: The Seer/Codex/AI remediation webhook that accepts a JSON payload describing the incident. +- `SEER_WEBHOOK_TOKEN`: Bearer/API token for that endpoint. +- `OBSERVABILITY_SOURCE_URL` (optional): Link back to Datadog/Sentry for the alert; used only for context in the payload and Action summary. + +### 2) Wire Datadog (current logging/alerting stack) +1. Create or update a monitor and add a **Custom Webhook** notification. +2. Point it to GitHub’s repository dispatch endpoint: + ``` + POST https://api.github.com/repos/hicommonwealth/commonwealth/dispatches + Authorization: Bearer + Accept: application/vnd.github+json + + { + "event_type": "observability_error", + "client_payload": { + "source": "datadog", + "title": "{{title}}", + "severity": "{{severity}}", + "service": "{{service.name}}", + "link": "{{link}}", + "message": "{{text}}", + "host": "{{host.name}}", + "timestamp": "{{timestamp}}" + } + } + ``` +3. Save the monitor; Datadog will now ping the repo when the alert fires. + +### 3) (Optional) Wire Sentry +If you also run Sentry, add a **Custom Webhook** integration pointing to the same dispatch URL. Map fields such as `event_type="observability_error"`, `source="sentry"`, `title`, `culprit`, `timestamp`, and a link to the issue/permalink. + +### 4) Claude Code Review +Install the Claude Code Review GitHub App for this repo. No workflow changes are needed—Claude will review any PR Seer opens. + +### 5) How the GitHub Action works +- File: [`.github/workflows/observability-seer-autofix.yml`](../.github/workflows/observability-seer-autofix.yml) +- Triggers: `repository_dispatch` with `event_type: observability_error` or manual `workflow_dispatch`. +- Steps: + 1. Validate that `SEER_WEBHOOK_URL` and `SEER_WEBHOOK_TOKEN` are present; fail fast if not. + 2. Capture the incoming payload into `seer-payload.json` (includes source, title, severity, links, and commit info). + 3. `curl` the payload to the configured Seer webhook so Seer can attempt the fix and open a PR. + 4. Publish a short summary (and link back to Datadog/Sentry if provided) in the GitHub Actions log. + +### 6) Manual kick-off for testing +You can dry-run via **Actions → Observability → Run workflow** and provide sample inputs (title, severity, message, link). The Action will send those to Seer using the same secrets. + +### Notes & guardrails +- The Action does not modify code; it delegates remediation to your Seer/Codex endpoint. Ensure that endpoint is responsible for opening the PR with changes. +- Use a narrowly scoped PAT for the Datadog/Sentry webhook that can only dispatch events (no write code permissions). +- If you later add other telemetry systems, reuse the same `event_type` and payload shape so the workflow stays compatible.