From 2e9aa1d6f438bab4a647b4bafb7e0d82baabb732 Mon Sep 17 00:00:00 2001 From: tend-agent <270458913+tend-agent@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:24:03 +0000 Subject: [PATCH 1/3] fix(list-recent-runs): anchor window recovery on the last successful run `--status completed` includes failures, so a run that died before its agent produced any analysis advanced the recovery anchor as if it had covered its hour. After an outage the next green run resumed one tick back and reported an all-clear for a window nothing had looked at. Anchor on the previous successful run, scan deeper than one cron period (during an outage the recent completed runs are all failures), and announce on stderr when the 6h cap clamps the recovered floor so the caller records a coverage gap instead of a false all-clear. --- .../scripts/list-recent-runs.sh | 50 +++++++++++++------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/plugins/tend-ci-runner/scripts/list-recent-runs.sh b/plugins/tend-ci-runner/scripts/list-recent-runs.sh index c16ffa30..200444f7 100755 --- a/plugins/tend-ci-runner/scripts/list-recent-runs.sh +++ b/plugins/tend-ci-runner/scripts/list-recent-runs.sh @@ -97,22 +97,37 @@ if [ -n "$cron_minute" ]; then # Default floor: one cron period back. Consecutive ticks tile exactly. COMPLETED_AFTER=$((intended - 3600)) - # Dropped-tick recovery. GHA doesn't only *delay* scheduled ticks, it also - # *drops* them: a tick that fires zero times leaves that hour's completions - # in the gap between the previous and next cycle's windows (the skipped-tick - # case #526 deferred as acceptable). Rather than assume the previous tick - # fired, resume from where the previous *actual* completed run of this - # workflow left off: recover that run's intended tick and floor the window - # there. When every tick fires, the previous run's intended tick == the + # Dropped-tick and failed-run recovery. GHA doesn't only *delay* scheduled + # ticks, it also *drops* them: a tick that fires zero times leaves that hour's + # completions in the gap between the previous and next cycle's windows (the + # skipped-tick case #526 deferred as acceptable). Rather than assume the + # previous tick fired, resume from where the previous run that actually + # analyzed something left off: recover that run's intended tick and floor the + # window there. + # + # Anchor on the previous *successful* run, not merely a completed one. A run + # that failed before its agent produced any analysis — harness crash, model + # quota exhaustion, timeout — covered no window at all, so resuming from it + # discards every hour the outage spanned and the next green run reports a + # one-hour all-clear for a window nothing ever looked at. For the same reason + # the scan reaches well past one cron period: during an outage the most recent + # completed runs are all failures, and a short limit finds no success to + # anchor on. A partially-failed matrix run counts as a failure here, which + # only ever widens the window — overlap is re-offered work the caller dedups + # against its own evidence log, where a gap is silently unanalyzed. + # + # When every tick fires and succeeds, the previous run's intended tick == the # default (intended - 3600), so this is a byte-identical no-op — still no - # overlap between consecutive cycles. When a tick was dropped, it reaches - # back to cover the orphaned hour. Capped at 6h so a sustained outage can't - # create an unbounded window. The analyzing workflow runs on the current - # repo, so this query omits TARGET_REPO's -R. + # overlap between consecutive cycles. Capped at 6h so a sustained outage can't + # create an unbounded window; when the cap bites, say so on stderr so the + # caller records a coverage gap instead of a false all-clear. The analyzing + # workflow runs on the current repo, so this query omits TARGET_REPO's -R. if [ -n "${GITHUB_WORKFLOW:-}" ]; then + floor_cap=$((intended - 21600)) # never reach back more than 6h + floor_cap_iso=$(date -u -d "@$floor_cap" +%Y-%m-%dT%H:%M:%SZ) prev_start=$(gh run list --workflow "$GITHUB_WORKFLOW" --status completed \ - --limit 10 --json databaseId,createdAt \ - --jq "[.[] | select(.databaseId != (${GITHUB_RUN_ID:-0}))] | .[0].createdAt // empty" \ + --limit 50 --json databaseId,createdAt,conclusion \ + --jq "[.[] | select(.databaseId != (${GITHUB_RUN_ID:-0})) | select(.conclusion == \"success\")] | .[0].createdAt // empty" \ 2>/dev/null || true) if [ -n "$prev_start" ]; then prev_ts=$(date -u -d "$prev_start" +%s 2>/dev/null || echo "") @@ -123,10 +138,15 @@ if [ -n "$cron_minute" ]; then else prev_intended=$((prev_hour_tick - 3600)) fi - floor_cap=$((intended - 21600)) # never reach back more than 6h - [ "$prev_intended" -lt "$floor_cap" ] && prev_intended=$floor_cap + if [ "$prev_intended" -lt "$floor_cap" ]; then + echo "WARNING: the last successful '$GITHUB_WORKFLOW' run started $prev_start, more than 6h back. Window floored at $floor_cap_iso; runs that completed before it are NOT in this list. Record a coverage gap, not an all-clear." >&2 + prev_intended=$floor_cap + fi [ "$prev_intended" -lt "$COMPLETED_AFTER" ] && COMPLETED_AFTER=$prev_intended fi + else + echo "WARNING: no successful '$GITHUB_WORKFLOW' run among the last 50 completed ones. Window floored at $floor_cap_iso; anything earlier is NOT in this list. Record a coverage gap, not an all-clear." >&2 + [ "$floor_cap" -lt "$COMPLETED_AFTER" ] && COMPLETED_AFTER=$floor_cap fi fi From fb2637629f1eb116c5976f4fd16a8b9598b8cee3 Mon Sep 17 00:00:00 2001 From: tend-agent <270458913+tend-agent@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:35:22 +0000 Subject: [PATCH 2/3] fix(list-recent-runs): stop the fetch limit truncating a recovered window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the recovery path: - The fetch loop still passed `--limit 50` while recovery routinely widens the window to 8h. `gh run list` is newest-first, so a workflow over the limit silently drops its *oldest* runs — exactly the ones in the gap the anchor reached back for, restoring the false all-clear this PR exists to prevent. `tend-mention` alone produced 57 runs in an 8h window. Raise to 200 and warn when a workflow returns exactly the limit, so a truncation is visible rather than silent. - Filter the anchor query server-side with `--status success` (it accepts conclusions as well as statuses). The scan-depth heuristic goes away with it, so an outage longer than the scan can no longer bury the anchor. - Route the anchor query through `gh_retry` and fail loud. It kept `2>/dev/null || true`, so a transient API error was indistinguishable from "no successful run" and produced a confident warning naming a cause that hadn't happened. - The consuming skill still said "if empty, record all-clear", which contradicts every warning above; note that a WARNING means a coverage gap regardless of whether the list came back empty. --- .../scripts/list-recent-runs.sh | 44 ++++++++++++++----- .../skills/review-reviewers/SKILL.md | 2 + 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/plugins/tend-ci-runner/scripts/list-recent-runs.sh b/plugins/tend-ci-runner/scripts/list-recent-runs.sh index 200444f7..1fc9dd3e 100755 --- a/plugins/tend-ci-runner/scripts/list-recent-runs.sh +++ b/plugins/tend-ci-runner/scripts/list-recent-runs.sh @@ -109,12 +109,12 @@ if [ -n "$cron_minute" ]; then # that failed before its agent produced any analysis — harness crash, model # quota exhaustion, timeout — covered no window at all, so resuming from it # discards every hour the outage spanned and the next green run reports a - # one-hour all-clear for a window nothing ever looked at. For the same reason - # the scan reaches well past one cron period: during an outage the most recent - # completed runs are all failures, and a short limit finds no success to - # anchor on. A partially-failed matrix run counts as a failure here, which - # only ever widens the window — overlap is re-offered work the caller dedups - # against its own evidence log, where a gap is silently unanalyzed. + # one-hour all-clear for a window nothing ever looked at. `--status` takes + # conclusions as well as statuses, so the filter runs server-side and no scan + # depth is involved: an outage of any length can't bury the anchor. A + # partially-failed matrix run counts as a failure here, which only ever widens + # the window — overlap is re-offered work the caller dedups against its own + # evidence log, where a gap is silently unanalyzed. # # When every tick fires and succeeds, the previous run's intended tick == the # default (intended - 3600), so this is a byte-identical no-op — still no @@ -125,10 +125,15 @@ if [ -n "$cron_minute" ]; then if [ -n "${GITHUB_WORKFLOW:-}" ]; then floor_cap=$((intended - 21600)) # never reach back more than 6h floor_cap_iso=$(date -u -d "@$floor_cap" +%Y-%m-%dT%H:%M:%SZ) - prev_start=$(gh run list --workflow "$GITHUB_WORKFLOW" --status completed \ - --limit 50 --json databaseId,createdAt,conclusion \ - --jq "[.[] | select(.databaseId != (${GITHUB_RUN_ID:-0})) | select(.conclusion == \"success\")] | .[0].createdAt // empty" \ - 2>/dev/null || true) + # Route through gh_retry and fail loud, like the other queries here: a + # transient API error must not masquerade as "no successful run ever", which + # would emit a confidently-worded warning naming a cause that didn't happen. + if ! prev_start=$(gh_retry gh run list --workflow "$GITHUB_WORKFLOW" \ + --status success --limit 5 --json databaseId,createdAt \ + --jq "[.[] | select(.databaseId != (${GITHUB_RUN_ID:-0}))] | .[0].createdAt // empty"); then + echo "ERROR: 'gh run list' for the window anchor failed after retries — refusing to guess a floor that would read as a false all-clear" >&2 + exit 1 + fi if [ -n "$prev_start" ]; then prev_ts=$(date -u -d "$prev_start" +%s 2>/dev/null || echo "") if [ -n "$prev_ts" ]; then @@ -145,7 +150,7 @@ if [ -n "$cron_minute" ]; then [ "$prev_intended" -lt "$COMPLETED_AFTER" ] && COMPLETED_AFTER=$prev_intended fi else - echo "WARNING: no successful '$GITHUB_WORKFLOW' run among the last 50 completed ones. Window floored at $floor_cap_iso; anything earlier is NOT in this list. Record a coverage gap, not an all-clear." >&2 + echo "WARNING: no successful '$GITHUB_WORKFLOW' run found at all. Window floored at $floor_cap_iso; anything earlier is NOT in this list. Record a coverage gap, not an all-clear." >&2 [ "$floor_cap" -lt "$COMPLETED_AFTER" ] && COMPLETED_AFTER=$floor_cap fi fi @@ -158,16 +163,31 @@ fi all_runs="[]" +# `gh run list` returns newest-first, so a workflow with more runs in the window +# than this limit silently drops the *oldest* ones — exactly the runs sitting in +# a gap the recovery above just reached back for, which would restore the false +# all-clear that recovery exists to prevent. A recovered window spans up to 8h, +# and a single busy workflow clears 50 runs in that span, so the limit is sized +# well past one workflow's output; gh paginates in hundreds, so the headroom +# costs at most one extra page per workflow. +RUN_LIMIT=200 + for wf in "${WORKFLOWS[@]}"; do if ! runs=$(gh_retry gh run list \ "${repo_args[@]}" \ --workflow "${wf}" \ --created ">=${CREATED_SINCE}" \ --json databaseId,conclusion,createdAt,updatedAt \ - --limit 50); then + --limit "$RUN_LIMIT"); then echo "ERROR: 'gh run list' for workflow '$wf' failed after retries — refusing to report a partial run list" >&2 exit 1 fi + # Exactly $RUN_LIMIT rows means the list may be capped rather than complete. + # Warn rather than exit: unlike a failed fetch, the rows in hand are still + # worth analyzing — the caller just can't read a short list as an all-clear. + if [ "$(printf '%s' "$runs" | jq 'length')" -ge "$RUN_LIMIT" ]; then + echo "WARNING: '$wf' returned $RUN_LIMIT runs, the fetch limit — older runs in this window are likely missing from the list. Record a coverage gap, not an all-clear." >&2 + fi all_runs=$(echo "$all_runs" "$runs" | jq -s 'add') done diff --git a/plugins/tend-ci-runner/skills/review-reviewers/SKILL.md b/plugins/tend-ci-runner/skills/review-reviewers/SKILL.md index b9af0f82..a23ce9fa 100644 --- a/plugins/tend-ci-runner/skills/review-reviewers/SKILL.md +++ b/plugins/tend-ci-runner/skills/review-reviewers/SKILL.md @@ -215,6 +215,8 @@ The script discovers `tend-*` workflows by default. Pass additional prefixes as If empty, record the run as all-clear per "Recording below-threshold findings" above, then skip to Step 6. +If the script printed a `WARNING:` on stderr, the list is known-incomplete — the window was clamped, no anchor was found, or a workflow hit the fetch limit. Record a coverage gap naming the missing span instead of an all-clear, whether or not the list came back empty; the next run's floor advances past that span regardless, so an unrecorded gap is never revisited. + ## Step 2: Survey outcomes via cheap subagent Spawn a cheap subagent to check outcomes across all runs from Step 1. The subagent does the token-heavy work of mapping runs to PRs/issues and checking acceptance signals. From 3521f8b26e7900a89d7c566d36555e1c36d7a75f Mon Sep 17 00:00:00 2001 From: tend-agent <270458913+tend-agent@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:45:23 +0000 Subject: [PATCH 3/3] fix(list-recent-runs): anchor on scheduled runs only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `workflow_dispatch` run has no `.schedule` in its event payload, so it takes the non-cron branch and covers a now-anchored 1h window rather than a tiled one. Anchoring on it therefore floors the next scheduled run just past a dispatch that never covered the gap — and unlike the clamp, it warns about nothing, because the anchor looks recent. Dispatching the workflow by hand to check on a fix mid-outage is the natural thing to do and is exactly when that bites, so it's the same class of hole as anchoring on a failure. Filter the anchor query with `--event schedule`. No-op today (no dispatch runs on record for `review-reviewers`, which does declare `workflow_dispatch:`), so it only bites the case above. --- plugins/tend-ci-runner/scripts/list-recent-runs.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/plugins/tend-ci-runner/scripts/list-recent-runs.sh b/plugins/tend-ci-runner/scripts/list-recent-runs.sh index 1fc9dd3e..d7067e18 100755 --- a/plugins/tend-ci-runner/scripts/list-recent-runs.sh +++ b/plugins/tend-ci-runner/scripts/list-recent-runs.sh @@ -111,7 +111,14 @@ if [ -n "$cron_minute" ]; then # discards every hour the outage spanned and the next green run reports a # one-hour all-clear for a window nothing ever looked at. `--status` takes # conclusions as well as statuses, so the filter runs server-side and no scan - # depth is involved: an outage of any length can't bury the anchor. A + # depth is involved: an outage of any length can't bury the anchor. + # + # Scheduled runs only. A `workflow_dispatch` run has no `.schedule` in its + # event payload, so it took the non-cron branch and covered a now-anchored 1h + # window rather than a tiled one — anchoring on it would discard the rest of + # the gap, silently, which is the same class of hole as anchoring on a + # failure. Dispatching the workflow by hand to check on a fix mid-outage is + # the natural thing to do, and is exactly when that would bite. A # partially-failed matrix run counts as a failure here, which only ever widens # the window — overlap is re-offered work the caller dedups against its own # evidence log, where a gap is silently unanalyzed. @@ -129,7 +136,7 @@ if [ -n "$cron_minute" ]; then # transient API error must not masquerade as "no successful run ever", which # would emit a confidently-worded warning naming a cause that didn't happen. if ! prev_start=$(gh_retry gh run list --workflow "$GITHUB_WORKFLOW" \ - --status success --limit 5 --json databaseId,createdAt \ + --status success --event schedule --limit 5 --json databaseId,createdAt \ --jq "[.[] | select(.databaseId != (${GITHUB_RUN_ID:-0}))] | .[0].createdAt // empty"); then echo "ERROR: 'gh run list' for the window anchor failed after retries — refusing to guess a floor that would read as a false all-clear" >&2 exit 1 @@ -150,7 +157,7 @@ if [ -n "$cron_minute" ]; then [ "$prev_intended" -lt "$COMPLETED_AFTER" ] && COMPLETED_AFTER=$prev_intended fi else - echo "WARNING: no successful '$GITHUB_WORKFLOW' run found at all. Window floored at $floor_cap_iso; anything earlier is NOT in this list. Record a coverage gap, not an all-clear." >&2 + echo "WARNING: no successful scheduled '$GITHUB_WORKFLOW' run found at all. Window floored at $floor_cap_iso; anything earlier is NOT in this list. Record a coverage gap, not an all-clear." >&2 [ "$floor_cap" -lt "$COMPLETED_AFTER" ] && COMPLETED_AFTER=$floor_cap fi fi