Watchdog: scan (cron) #485
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Watchdog: Stuck Jobs" | |
| run-name: "Watchdog: scan ${{ github.event_name == 'workflow_dispatch' && '(manual)' || '(cron)' }}" | |
| # Periodic safety net for the impl pipeline. Catches the failure modes | |
| # the regular workflows miss: | |
| # 1. PRs labeled `ai-review-failed` without `ai-review-rescued` | |
| # (impl-review-retry.yml normally handles these; watchdog covers | |
| # the case where that listener missed the labeled event). | |
| # 2. PRs with `ai-attempt-N` + `quality:*` but no decision label | |
| # (review handed off to repair, repair crashed, nothing else fired). | |
| # 2b. PRs with `ai-rejected` + `ai-attempt-N`: the review rejected and | |
| # dispatched a repair that then crashed (its own crash-retry | |
| # exhausted). Neither 2 nor 4 matched this state before 2026-09-02. | |
| # 3. spec-ready issues with `generate:<lib>` or `impl:<lib>:failed` and | |
| # no open PR for that (spec, lib) pair after the staleness window. | |
| # 4. daily-regen's cron silently starved by GitHub's scheduler (no run | |
| # for >10 h outside the quiet window) — re-dispatched manually. | |
| # | |
| # Retries are bounded per-cause via marker labels: | |
| # - `ai-review-rescued` (review failures) | |
| # - `watchdog:repair-rescued-<N>` (repair failures, one per attempt#) | |
| # - `watchdog:retried-<lib>` (per-library generation failures) | |
| # A marker is set only after the rescue dispatch succeeded (and, for | |
| # generation retries, after the run is visible), so a lost dispatch is | |
| # retried on the next scan instead of being recorded as done. When a | |
| # marker is already present, the watchdog skips and emits a warning so | |
| # a human can pick the case up. | |
| on: | |
| schedule: | |
| - cron: '0 */6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| stale_hours: | |
| description: "Hours of inactivity before a job is considered stuck" | |
| required: false | |
| default: '4' | |
| dry_run: | |
| description: "Log decisions without dispatching workflows" | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| actions: write | |
| concurrency: | |
| group: watchdog-stuck-jobs | |
| cancel-in-progress: false | |
| jobs: | |
| scan: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 1 | |
| - name: Scan and dispatch | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GH_REPO: ${{ github.repository }} | |
| STALE_HOURS: ${{ inputs.stale_hours || '4' }} | |
| DRY_RUN: ${{ inputs.dry_run || 'false' }} | |
| run: | | |
| set -euo pipefail | |
| STALE_SEC=$(( STALE_HOURS * 3600 )) | |
| NOW=$(date -u +%s) | |
| # Reports the outcome AFTER the attempt and returns non-zero on | |
| # failure instead of aborting the scan: the previous form printed | |
| # "→ dispatching" first and then let one failed `gh workflow run` | |
| # kill the whole run under `set -e`, so every PR after it went | |
| # unscanned while the log claimed a rescue that never happened. | |
| dispatch() { | |
| local label="$1"; shift | |
| if [[ "$DRY_RUN" == "true" ]]; then | |
| echo "::notice::[dry-run] $label → gh workflow run $*" | |
| return 0 | |
| fi | |
| if gh workflow run "$@"; then | |
| echo "::notice::$label → dispatched" | |
| else | |
| echo "::warning::$label → dispatch FAILED (gh workflow run $*)" | |
| return 1 | |
| fi | |
| } | |
| # Re-dispatch one (spec, library) pair straight to impl-generate.yml, | |
| # NOT via bulk-generate.yml: that workflow serialises on one global | |
| # concurrency group, and GitHub keeps at most one pending run per | |
| # group — each newer dispatch cancels the older pending one. Nine | |
| # rescues fired 5 s apart on 2026-09-02 produced two runs (the first | |
| # and the last) and seven silent `cancelled`s, while every pair had | |
| # already been labelled `watchdog:retried-<lib>` and so dropped out | |
| # of all future scans. impl-generate's group is per pair. | |
| # Returns 0 only once the new run is visible in the run list, so the | |
| # caller marks exactly the pairs that really got their retry. | |
| dispatch_generate() { | |
| local label="$1" spec="$2" lib="$3" issue="$4" | |
| local t0 n i | |
| t0=$(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| dispatch "$label" impl-generate.yml \ | |
| -f specification_id="$spec" -f library="$lib" -f issue_number="$issue" || return 1 | |
| [[ "$DRY_RUN" == "true" ]] && return 0 | |
| for i in 1 2 3 4 5 6; do | |
| sleep 10 | |
| n=$(gh run list --workflow=impl-generate.yml --limit 30 --json displayTitle,createdAt \ | |
| --jq "[.[] | select(.displayTitle == \"Generate: $lib for $spec\" and .createdAt >= \"$t0\")] | length" \ | |
| 2>/dev/null || echo 0) | |
| if (( n > 0 )); then | |
| return 0 | |
| fi | |
| done | |
| echo "::warning::$label → dispatched but no run appeared within 60 s; pair left unmarked for the next scan" | |
| return 1 | |
| } | |
| ensure_label() { | |
| if [[ "$DRY_RUN" == "true" ]]; then | |
| echo "::notice::[dry-run] would ensure label: $1" | |
| return 0 | |
| fi | |
| gh label create "$1" --color "$2" --description "$3" 2>/dev/null || true | |
| } | |
| # Resolve issue number → spec_id by parsing the issue title prefix | |
| # `[spec-id] ...` (set by spec-create.yml when the spec lands). | |
| spec_id_for_issue() { | |
| local issue_num="$1" | |
| gh issue view "$issue_num" --json title --jq '.title' \ | |
| | grep -oP '^\[\K[^]]+' || true | |
| } | |
| ##### A) Scan open implementation PRs ############################## | |
| # `gh pr list --search "head:..."` does not support prefix matching, | |
| # so list all open PRs and filter client-side by branch name. | |
| PRS_JSON=$(gh pr list --state open --limit 200 \ | |
| --json number,labels,headRefName,updatedAt \ | |
| --jq '[.[] | select(.headRefName | startswith("implementation/"))]') | |
| echo "Scanning $(jq 'length' <<<"$PRS_JSON") open implementation PR(s)" | |
| while IFS= read -r row; do | |
| num=$(jq -r '.number' <<<"$row") | |
| branch=$(jq -r '.headRefName' <<<"$row") | |
| updated=$(jq -r '.updatedAt' <<<"$row") | |
| labels=$(jq -r '[.labels[].name] | join(" ")' <<<"$row") | |
| updated_sec=$(date -u -d "$updated" +%s) | |
| age=$(( NOW - updated_sec )) | |
| spec_id=$(echo "$branch" | cut -d'/' -f2) | |
| library=$(echo "$branch" | cut -d'/' -f3) | |
| # Case 1: review failed, not yet rescued → re-dispatch review | |
| if echo " $labels " | grep -q " ai-review-failed "; then | |
| if echo " $labels " | grep -q " ai-review-rescued "; then | |
| echo "::warning::PR #$num: ai-review-failed persists after rescue — needs manual attention" | |
| else | |
| ensure_label "ai-review-rescued" "5319e7" "Review re-dispatched once after ai-review-failed" | |
| if dispatch "PR #$num: review (failed)" impl-review.yml -f pr_number="$num" \ | |
| && [[ "$DRY_RUN" != "true" ]]; then | |
| gh pr edit "$num" --add-label "ai-review-rescued" --remove-label "ai-review-failed" 2>/dev/null || true | |
| fi | |
| fi | |
| continue | |
| fi | |
| # Case 2: stalled repair handoff | |
| # has ai-attempt-N + quality:M, no ai-approved/ai-rejected, | |
| # PR untouched for stale_hours → re-dispatch impl-repair | |
| if echo " $labels " | grep -qE " ai-attempt-[0-9]+ " \ | |
| && echo " $labels " | grep -qE " quality:[0-9]+ " \ | |
| && ! echo " $labels " | grep -qE " (ai-approved|ai-rejected) " \ | |
| && (( age > STALE_SEC )); then | |
| attempt=$(echo " $labels " | grep -oP "ai-attempt-\K[0-9]+" | sort -nr | head -1) | |
| marker="watchdog:repair-rescued-$attempt" | |
| if echo " $labels " | grep -q " $marker "; then | |
| echo "::warning::PR #$num: repair attempt $attempt already rescued — needs manual attention" | |
| continue | |
| fi | |
| ensure_label "$marker" "5319e7" "Watchdog re-dispatched repair attempt $attempt" | |
| if dispatch "PR #$num: repair (stalled, attempt=$attempt)" impl-repair.yml \ | |
| -f pr_number="$num" \ | |
| -f specification_id="$spec_id" \ | |
| -f library="$library" \ | |
| -f attempt="$attempt" \ | |
| && [[ "$DRY_RUN" != "true" ]]; then | |
| gh pr edit "$num" --add-label "$marker" 2>/dev/null || true | |
| fi | |
| continue | |
| fi | |
| # Case 2b: repair crashed after a rejection | |
| # has ai-rejected + ai-attempt-N, PR untouched for stale_hours. | |
| # impl-review set both labels and dispatched impl-repair, which | |
| # then died (its crash-retry exhausted, e.g. during the Claude | |
| # outage of 2026-09-02 03:15–03:45 UTC). Case 2 excludes | |
| # ai-rejected and Case 4 requires no attempt label, so this state | |
| # matched nothing and three PRs sat for hours until a human | |
| # re-dispatched the repair. Same marker as Case 2: one rescue per | |
| # attempt number. | |
| if echo " $labels " | grep -q " ai-rejected " \ | |
| && echo " $labels " | grep -qE " ai-attempt-[0-9]+ " \ | |
| && (( age > STALE_SEC )); then | |
| attempt=$(echo " $labels " | grep -oP "ai-attempt-\K[0-9]+" | sort -nr | head -1) | |
| marker="watchdog:repair-rescued-$attempt" | |
| if echo " $labels " | grep -q " $marker "; then | |
| echo "::warning::PR #$num: crashed repair attempt $attempt already rescued — needs manual attention" | |
| continue | |
| fi | |
| ensure_label "$marker" "5319e7" "Watchdog re-dispatched repair attempt $attempt" | |
| if dispatch "PR #$num: repair (crashed after rejection, attempt=$attempt)" impl-repair.yml \ | |
| -f pr_number="$num" \ | |
| -f specification_id="$spec_id" \ | |
| -f library="$library" \ | |
| -f attempt="$attempt" \ | |
| && [[ "$DRY_RUN" != "true" ]]; then | |
| gh pr edit "$num" --add-label "$marker" 2>/dev/null || true | |
| fi | |
| continue | |
| fi | |
| # Case 3: ai-approved but never merged — impl-merge `pull_request: labeled` | |
| # event was lost. Re-dispatch impl-merge.yml manually. | |
| if echo " $labels " | grep -q " ai-approved " \ | |
| && (( age > STALE_SEC )); then | |
| marker="watchdog:merge-rescued" | |
| if echo " $labels " | grep -q " $marker "; then | |
| echo "::warning::PR #$num: ai-approved merge already rescued — needs manual attention" | |
| continue | |
| fi | |
| ensure_label "$marker" "5319e7" "Watchdog re-dispatched impl-merge once" | |
| if dispatch "PR #$num: merge (ai-approved stuck)" impl-merge.yml \ | |
| -f pr_number="$num" \ | |
| && [[ "$DRY_RUN" != "true" ]]; then | |
| gh pr edit "$num" --add-label "$marker" 2>/dev/null || true | |
| fi | |
| continue | |
| fi | |
| # Case 4: ai-rejected but no ai-attempt-N label — impl-repair `labeled` | |
| # trigger inside impl-review.yml never fired. Re-dispatch attempt 1. | |
| if echo " $labels " | grep -q " ai-rejected " \ | |
| && ! echo " $labels " | grep -qE " ai-attempt-[0-9]+ " \ | |
| && (( age > STALE_SEC )); then | |
| marker="watchdog:repair-rescued-1" | |
| if echo " $labels " | grep -q " $marker "; then | |
| echo "::warning::PR #$num: ai-rejected initial-repair already rescued — needs manual attention" | |
| continue | |
| fi | |
| ensure_label "$marker" "5319e7" "Watchdog re-dispatched repair attempt 1" | |
| if dispatch "PR #$num: repair (initial, ai-rejected, no attempt label)" impl-repair.yml \ | |
| -f pr_number="$num" \ | |
| -f specification_id="$spec_id" \ | |
| -f library="$library" \ | |
| -f attempt="1" \ | |
| && [[ "$DRY_RUN" != "true" ]]; then | |
| gh pr edit "$num" --add-label "$marker" 2>/dev/null || true | |
| fi | |
| continue | |
| fi | |
| # Case 5: implementation PR with only watchdog:* markers (or none) — | |
| # impl-review never started. Happens when impl-generate.yml's | |
| # repository_dispatch to impl-review didn't land. | |
| # | |
| # We treat "labels are exactly watchdog:* markers, no review/repair | |
| # labels" as the same situation as "no labels at all" — without | |
| # that, our own previously-added marker would hide the PR from | |
| # detection. To keep the one-shot semantics ("rescue once, then | |
| # warn"), we check the marker explicitly. | |
| non_marker_labels=$(echo " $labels " | tr ' ' '\n' | grep -v '^$' | grep -vE '^watchdog:' || true) | |
| if [[ -z "$non_marker_labels" ]] && (( age > STALE_SEC )); then | |
| marker="watchdog:review-bootstrap" | |
| if echo " $labels " | grep -q " $marker "; then | |
| echo "::warning::PR #$num: review never started even after watchdog bootstrap — needs manual attention" | |
| continue | |
| fi | |
| ensure_label "$marker" "5319e7" "Watchdog bootstrapped initial review" | |
| if dispatch "PR #$num: review (never started, no labels)" impl-review.yml \ | |
| -f pr_number="$num" \ | |
| && [[ "$DRY_RUN" != "true" ]]; then | |
| gh pr edit "$num" --add-label "$marker" 2>/dev/null || true | |
| fi | |
| continue | |
| fi | |
| done < <(echo "$PRS_JSON" | jq -c '.[]') | |
| ##### B) Scan spec-ready issues for stuck generation ############### | |
| ISS_JSON=$(gh issue list --state open --label spec-ready --limit 300 \ | |
| --json number,labels,updatedAt) | |
| echo "Scanning $(jq 'length' <<<"$ISS_JSON") spec-ready issue(s)" | |
| while IFS= read -r row; do | |
| num=$(jq -r '.number' <<<"$row") | |
| updated=$(jq -r '.updatedAt' <<<"$row") | |
| labels=$(jq -r '[.labels[].name] | join(" ")' <<<"$row") | |
| updated_sec=$(date -u -d "$updated" +%s) | |
| age=$(( NOW - updated_sec )) | |
| (( age > STALE_SEC )) || continue | |
| spec_id="" | |
| spec_id_resolved=false | |
| # Iterate label tokens | |
| for label in $labels; do | |
| case "$label" in | |
| generate:*) | |
| lib="${label#generate:}" | |
| marker="watchdog:retried-$lib" | |
| if echo " $labels " | grep -q " $marker "; then | |
| echo "::warning::Issue #$num: generate:$lib already retried by watchdog — needs manual attention" | |
| continue | |
| fi | |
| if ! $spec_id_resolved; then | |
| spec_id=$(spec_id_for_issue "$num"); spec_id_resolved=true | |
| fi | |
| if [ -z "$spec_id" ]; then | |
| echo "::warning::Issue #$num: cannot resolve spec_id from title; skipping generate:$lib" | |
| continue | |
| fi | |
| open_pr=$(gh pr list --state open \ | |
| --search "head:implementation/$spec_id/$lib" \ | |
| --json number --jq 'length') | |
| if [[ "$open_pr" == "0" ]]; then | |
| ensure_label "$marker" "5319e7" "Watchdog retried $lib once" | |
| if dispatch_generate "Issue #$num: generate (stuck pending) $lib" \ | |
| "$spec_id" "$lib" "$num" \ | |
| && [[ "$DRY_RUN" != "true" ]]; then | |
| gh issue edit "$num" --add-label "$marker" 2>/dev/null || true | |
| fi | |
| fi | |
| ;; | |
| impl:*:failed) | |
| lib_failed="${label#impl:}" | |
| lib="${lib_failed%:failed}" | |
| marker="watchdog:retried-$lib" | |
| if echo " $labels " | grep -q " $marker "; then | |
| echo "::warning::Issue #$num: $lib already retried by watchdog — needs manual attention" | |
| continue | |
| fi | |
| if ! $spec_id_resolved; then | |
| spec_id=$(spec_id_for_issue "$num"); spec_id_resolved=true | |
| fi | |
| if [ -z "$spec_id" ]; then | |
| echo "::warning::Issue #$num: cannot resolve spec_id from title; skipping impl:$lib:failed" | |
| continue | |
| fi | |
| open_pr=$(gh pr list --state open \ | |
| --search "head:implementation/$spec_id/$lib" \ | |
| --json number --jq 'length') | |
| if [[ "$open_pr" == "0" ]]; then | |
| ensure_label "$marker" "5319e7" "Watchdog retried $lib once" | |
| if dispatch_generate "Issue #$num: generate (failed) $lib" \ | |
| "$spec_id" "$lib" "$num" \ | |
| && [[ "$DRY_RUN" != "true" ]]; then | |
| gh issue edit "$num" --add-label "$marker" 2>/dev/null || true | |
| fi | |
| fi | |
| ;; | |
| esac | |
| done | |
| done < <(echo "$ISS_JSON" | jq -c '.[]') | |
| ##### C) Cron-liveness: rescue a starved daily-regen schedule ###### | |
| # GitHub silently drops `schedule` ticks for daily-regen (the | |
| # repo's most frequent cron) during scheduler overload — observed | |
| # as multi-day gaps while this watchdog and bot-serving-check kept | |
| # firing. daily-regen ticks are ≤6 h apart (2 h cadence plus the | |
| # 18–21 UTC quiet window), so a >10 h silence means the schedule | |
| # is starved; re-dispatch it manually with default inputs. | |
| # Hours 17–21 UTC are skipped so the rescue never launches a | |
| # pipeline into the fixed 18–21 UTC quiet window the cron itself | |
| # deliberately avoids (17 included because bulk-generate starts | |
| # immediately after dispatch and would spill into it). Like the | |
| # cron, the window is UTC-fixed: it matches Berlin evening under | |
| # CEST and sits an hour earlier in local terms under CET. | |
| LIVENESS_HOURS=10 | |
| HOUR=$(date -u +%-H) | |
| # A manually disabled daily-regen is a deliberate pause (e.g. while a | |
| # sequential backfill owns the rate limit), not a starved schedule: | |
| # its cron does not tick at all, so the gap grows without bound, and | |
| # `gh workflow run` on a disabled workflow fails with HTTP 422 — which | |
| # under `set -e` takes the whole scan down after A and B already ran. | |
| REGEN_STATE=$(gh api "repos/${GH_REPO}/actions/workflows/daily-regen.yml" \ | |
| --jq '.state' 2>/dev/null || echo unknown) | |
| if [[ "$REGEN_STATE" != "active" ]]; then | |
| echo "::notice::daily-regen liveness: workflow state is '${REGEN_STATE}', not active — rescue skipped" | |
| elif (( HOUR >= 17 && HOUR <= 21 )); then | |
| echo "::notice::daily-regen liveness: inside/adjacent to quiet window (UTC hour $HOUR) — check skipped" | |
| else | |
| # --branch main: schedule runs (and rescue dispatches) live on the | |
| # default branch; a manual run on a feature branch must not mask a | |
| # starved main schedule. createdAt (not startedAt): always set, | |
| # even while the newest run is still queued. | |
| LAST=$(gh run list --workflow daily-regen.yml --branch main --limit 1 \ | |
| --json createdAt --jq '.[0].createdAt // empty') | |
| if [[ -z "$LAST" ]]; then | |
| echo "::warning::daily-regen liveness: no runs found on main — skipping rescue" | |
| else | |
| last_sec=$(date -u -d "$LAST" +%s) | |
| # Fresh timestamp: NOW from the top of the script is stale by | |
| # however long the A/B scans took. | |
| now_c=$(date -u +%s) | |
| gap=$(( now_c - last_sec )) | |
| gap_hm=$(printf '%dh%02dm' $(( gap / 3600 )) $(( (gap % 3600) / 60 ))) | |
| if (( gap > LIVENESS_HOURS * 3600 )); then | |
| echo "::warning::daily-regen: no new run on main for ${gap_hm} (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" | |
| dispatch "daily-regen: cron starved (${gap_hm} silent)" daily-regen.yml || true | |
| else | |
| echo "daily-regen liveness: newest run on main created ${gap_hm} ago — healthy" | |
| fi | |
| fi | |
| fi | |
| echo "::notice::Watchdog scan complete" |