Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/actions/pr-require-checks/aggregate-results/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,18 @@ inputs:
write lands, and a job concluding `success` was not cancelled or timed
out. Any other conclusion with no steps listed fails closed.
default: ''
alias-jobs:
description: >-
Newline- or space-separated display names of jobs that only mirror the
gate's conclusion under an older check name while a required-check
rename rolls out (a job that needs the gate and exits with its result).
Such a job runs on contract-only events too, so the run-shape reads
ignore it: a run whose only other non-skipped job is an alias still
reads as contract-only, and a failed contract-only run with a failed
alias is still re-run by `rerun-contract-only-siblings`. A carried
`success` written by a run from before the rename, whose gate job had
the alias's name, still finds its gate step under that name.
default: ''
rate-limit-wait-seconds:
description: >-
Ceiling, in seconds, on the time this step spends waiting out GitHub API
Expand Down Expand Up @@ -308,6 +320,7 @@ runs:
YIELD_TO_FULL_RUN: ${{ inputs.yield-to-full-run }}
YIELD_WAIT_SECONDS: ${{ inputs.yield-wait-seconds }}
STEP_NAME: ${{ inputs.step-name }}
ALIAS_JOBS: ${{ inputs.alias-jobs }}
JOB_CHECK_RUN_ID: ${{ job.check_run_id }}
GH_TOKEN: ${{ inputs.token }}
REPOSITORY: ${{ inputs.repository }}
Expand Down
32 changes: 24 additions & 8 deletions .github/actions/pr-require-checks/aggregate-results/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,9 @@ verify_carried_writer() {
# branches), and every later `resolve_own_gate` answers from what that lookup
# found, so the run spends one retry budget in all.
own_gate_job=""
# Jobs that only mirror the gate under an older check name (`alias-jobs`), as
# a JSON array of names. The run-shape reads ignore them.
alias_jobs_json="$(printf '%s' "${ALIAS_JOBS:-}" | tr -s ' \t\r\n' '\n' | jq -R . | jq -sc 'map(select(length > 0))')"
own_gate_job_id=""
own_gate_step=""
own_gate_endpoint=""
Expand Down Expand Up @@ -758,9 +761,9 @@ verify_writer_step() {
# contract step can fail it over a true lanes `success`, and only the step
# tells the two apart.
gate_step_outcome() {
jq -r --arg job "$own_gate_job" --arg step "$own_gate_step" \
jq -r --arg job "$own_gate_job" --arg step "$own_gate_step" --argjson aliases "$alias_jobs_json" \
'[ .jobs[]? | select(.name == $job) ] as $gates
| [ $gates[] | .steps[]? | select(.name == $step) ]
| [ .jobs[]? | select(.name == $job or (.name as $n | $aliases | index($n))) | .steps[]? | select(.name == $step) ]
| if length == 1 then .[0] | (if .status == "completed" then (.conclusion // "") else "running" end)
elif length > 1 then "ambiguous"
elif ($gates | length) == 1 and $gates[0].status == "completed" and $gates[0].conclusion == "success" and ($gates[0].steps // [] | length) == 0 then "success"
Expand All @@ -777,13 +780,19 @@ gate_step_outcome() {
# lane still ran its change-detection job, so it reads `full`;
# unknown anything else: fewer jobs listed than `total_count`, nothing
# but the gate listed, or a body jq cannot parse.
# Jobs named in `alias-jobs` are left out of the shape: they mirror the gate.
# A run with no job named <gate-job-name> predates the rename, so there the
# alias name is the gate.
# Each caller reads `unknown` the way that fails closed for it.
job_shape() {
jq -r --arg gate "$2" \
'(.total_count // 0) as $total | [ .jobs[]? ] as $jobs | ($jobs | map(select(.conclusion != "skipped") | .name)) as $ran
jq -r --arg gate "$2" --argjson aliases "$alias_jobs_json" \
'(.total_count // 0) as $total | [ .jobs[]? ] as $jobs
| (if any($jobs[]; .name == $gate) then {gate: $gate, ignore: $aliases}
else {gate: (first($jobs[] | .name | select(. as $n | $aliases | index($n))) // $gate), ignore: []} end) as $shape
| ($jobs | map(select(.conclusion != "skipped" and (.name as $n | $shape.ignore | index($n) | not)) | .name)) as $ran
| if ($jobs | length) < $total then "unknown"
elif ($jobs | length) > 1 and $ran == [$gate] then "contract"
elif any($ran[]; . != $gate) then "full"
elif ($jobs | length) > 1 and $ran == [$shape.gate] then "contract"
elif any($ran[]; . != $shape.gate) then "full"
else "unknown" end' \
<"$1" 2>/dev/null || echo unknown
}
Expand Down Expand Up @@ -1634,7 +1643,14 @@ rerun_failed_contract_only_siblings() {
# The next poll is this read's retry, so `gh_api` sends it once.
# shellcheck disable=SC2310 # gh_api handles its own errexit; an unreadable sibling is waited for.
if GH_API_RETRY_ERRORS=false gh_api GET "repos/${REPOSITORY}/actions/runs/${id}/jobs?filter=latest&per_page=100"; then
ran="$(jq -r '[ .jobs[]? | select(.status == "in_progress" or ((.conclusion // "skipped") != "skipped")) ] | length' <"$gh_stdout")" || ran=0
# An alias job mirrors the gate, so it is not a second job that ran;
# in a run from before the rename it is the gate itself. With this
# run's gate name unresolved, aliases are left out, which can only
# make a sibling wait longer, never skip its re-run.
ran="$(jq -r --arg gate "$own_gate_job" --argjson aliases "$alias_jobs_json" \
'[ .jobs[]? ] as $jobs | (if $gate == "" or any($jobs[]; .name == $gate) then $aliases else [] end) as $ignore
| [ $jobs[] | select(.name as $n | $ignore | index($n) | not)
| select(.status == "in_progress" or ((.conclusion // "skipped") != "skipped")) ] | length' <"$gh_stdout")" || ran=0
fi
if [[ ! "$ran" =~ ^[0-9]+$ || "$ran" -lt 2 ]]; then
waiting="${waiting}${waiting:+ }${id}"
Expand Down Expand Up @@ -1666,7 +1682,7 @@ rerun_failed_contract_only_siblings() {
warn_actions_failed "repos/${REPOSITORY}/actions/runs/${id}/jobs" 'actions: write' "Not re-running run ${id}."
continue
fi
contract_shaped="$(jq -r '[ .jobs[]? | .conclusion ] | ((map(select(. != "skipped")) == ["failure"]) and (length > 1))' <"$gh_stdout")"
contract_shaped="$(jq -r --argjson aliases "$alias_jobs_json" '[ .jobs[]? | select(.name as $n | $aliases | index($n) | not) | .conclusion ] | ((map(select(. != "skipped")) == ["failure"]) and (length > 1))' <"$gh_stdout")"
Comment thread
kyle-sexton marked this conversation as resolved.
Resolved
if [[ "$contract_shaped" != true ]]; then
echo "Not re-running failed run ${id} on ${SHA}: its latest attempt is not contract-only (more than its gate ran, or the gate did not fail)."
continue
Expand Down
72 changes: 72 additions & 0 deletions .github/actions/pr-require-checks/aggregate-results/run.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1699,6 +1699,19 @@ clear_rerun_fixtures() {
"$fixtures"/POST_repos_melodic-software_ci-workflows_actions_runs_*
}

# A failed contract-only run's alias job fails with its gate. Without
# alias-jobs two failed jobs do not read as contract-only and the run stays red.
echo 'case: a full-mode success re-runs a failed contract-only sibling whose alias failed too, given alias-jobs'
clear_run_fixtures
clear_rerun_fixtures
current_run
workflow_runs "[$(completed_run 5000 failure)]"
printf '%s' '{"jobs":[{"name":"ci-status","conclusion":"failure"},{"name":"lint","conclusion":"skipped"},{"name":"old-gate","conclusion":"failure"}]}' >"$fixtures/GET_repos_melodic-software_ci-workflows_actions_runs_5000_jobs.json"
run_case 0 'success success' pass false true RERUN_CONTRACT_ONLY_SIBLINGS=true ALIAS_JOBS='old-gate'
expect_gh_call "POST repos/${repository}/actions/runs/5000/rerun-failed-jobs"
run_case 0 'success success' pass false true RERUN_CONTRACT_ONLY_SIBLINGS=true
expect_no_gh_call 'actions/runs/5000/rerun-failed-jobs'

# 5000 and 4242 (this run, completed only in this fixture) are contract-only
# shaped: one failed gate, every other job skipped. 5100 is a failed full run,
# 5500 a failed single-job run, and 5200, 5300 and 5400 did not fail.
Expand Down Expand Up @@ -1908,6 +1921,51 @@ run_case 0 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true
expect_log "Carried forward: ci-lanes is success on ${sha}"
expect_no_sleep

# A job that mirrors the gate under an older check name runs on a
# contract-only event too. Without alias-jobs it reads as a lane that ran, so
# a contract-only sibling would count as a full run in flight.
contract_only_jobs_with_alias='[{"name":"lint","conclusion":"skipped"},{"name":"test","conclusion":"skipped"},{"name":"ci-status","conclusion":null},{"name":"old-gate","conclusion":null}]'
echo 'case: yield ignores a contract-only sibling whose alias job is listed, given alias-jobs'
reset_yield_fixtures
status_list "[$(full_run_success 100)]"
workflow_runs "[$(run_entry 4300 in_progress 2026-09-05T12:00:31Z),$(run_entry 4242 in_progress 2026-09-05T12:00:30Z)]"
jobs_raw 4300 "$contract_only_jobs_with_alias"
run_case 0 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true ALIAS_JOBS='old-gate'
expect_log "Carried forward: ci-lanes is success on ${sha}"

# A success a full run wrote before the gate job was renamed names its gate
# job by the old name, which alias-jobs now carries. Without looking under that
# name too, every open pull request's carried success is discarded until a push.
echo 'case: a success written under the gate job'"'"'s old name is carried, given alias-jobs'
reset_yield_fixtures
status_list "[$(full_run_success 100)]"
printf '{"jobs":[{"name":"lint","status":"completed","conclusion":"success","steps":[]},%s]}' "$(writer_gate_job success | sed 's/"name":"ci-status"/"name":"old-gate"/')" \
>"$fixtures/GET_repos_melodic-software_ci-workflows_actions_runs_4100_jobs.json"
run_case 0 'skipped skipped' pass true true ALIAS_JOBS='old-gate'
expect_log "Carried forward: ci-lanes is success on ${sha}"
run_case 1 'skipped skipped' pass true true
expect_no_log "Carried forward: ci-lanes is success on ${sha}"
install_step_fixtures

# A contract-only sibling from before the rename has no job under the new gate
# name: its gate carries the name alias-jobs now lists. Dropping that job
# would leave nothing that ran and the shape unknown.
echo 'case: yield reads a pre-rename contract-only sibling, gated under the alias name, as contract-only'
reset_yield_fixtures
status_list "[$(full_run_success 100)]"
workflow_runs "[$(run_entry 4300 in_progress 2026-09-05T12:00:31Z),$(run_entry 4242 in_progress 2026-09-05T12:00:30Z)]"
jobs_raw 4300 '[{"name":"lint","conclusion":"skipped"},{"name":"test","conclusion":"skipped"},{"name":"old-gate","conclusion":null}]'
run_case 0 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true ALIAS_JOBS='old-gate'
expect_log "Carried forward: ci-lanes is success on ${sha}"

echo 'case: without alias-jobs the same sibling counts as a full run in flight'
reset_yield_fixtures
status_list "[$(full_run_success 100)]"
workflow_runs "[$(run_entry 4300 in_progress 2026-09-05T12:00:31Z),$(run_entry 4242 in_progress 2026-09-05T12:00:30Z)]"
jobs_raw 4300 "$contract_only_jobs_with_alias"
run_case 1 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true
expect_log '::error::superseded by full run https://github.com/melodic-software/ci-workflows/actions/runs/4300'

# Without the gate-name check, a full run whose first job has finished and whose
# lanes were skipped, before its gate job exists, has the contract-only shape
# and the older success would carry across it.
Expand Down Expand Up @@ -2526,6 +2584,20 @@ expect_rerun_after_last_listing 5000
expect_rerun_after_last_listing 5300
expect_log "Re-ran 2 failed contract-only run(s) on ${sha}."

# A contract-only sibling whose gate already failed while its alias job still
# runs. Without leaving the alias out of the in-flight count, the two jobs read
# as a full run, the wait ends, and the sibling is never re-run.
echo 'case: the in-flight wait leaves an alias job out of a contract-only sibling'"'"'s count, given alias-jobs'
reset_yield_fixtures
workflow_runs_on_call 1 "[$(completed_run 5000 failure),$(run_entry 5300 in_progress 2026-09-05T12:00:40Z)]"
workflow_runs "[$(completed_run 5000 failure),$(completed_run 5300 failure)]"
jobs_for 5000 failure skipped
printf '{"jobs":%s}' '[{"name":"lint","status":"completed","conclusion":"skipped"},{"name":"ci-status","status":"completed","conclusion":"failure"},{"name":"old-gate","status":"in_progress","conclusion":null}]' >"$fixtures/GET_repos_melodic-software_ci-workflows_actions_runs_5300_jobs.1.json"
jobs_for 5300 failure skipped
run_case 0 'success success' pass false true RERUN_CONTRACT_ONLY_SIBLINGS=true YIELD_TO_FULL_RUN=true ALIAS_JOBS='old-gate'
expect_log "Waiting 10s for in-flight run(s) 5300 on ${sha} to finish before re-running failed contract-only runs."
expect_gh_call "POST repos/${repository}/actions/runs/5300/rerun-failed-jobs"

# A carry-forward wait of 0 reads once and never polls, so it has the same race.
echo 'case: with a carry-forward wait of 0 a full-mode success also waits for an in-flight contract-only sibling'
reset_yield_fixtures
Expand Down
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,14 @@ consumer to audit it.
longer for a runner. A polling contract-only run (a wait above `0`
without yield) reads the success itself, so nothing waits for it, and
neither does a holding one (`yield-wait-seconds` above `0`).
- `alias-jobs` on the `ci-status` step, while a repository renames its
required gate job: the display names of jobs that only mirror the gate's
conclusion under the old check name (a job that needs the gate, runs
`if: always()` and exits with its result). Such a job runs on every
event, so without this input a contract-only run reads as a full run,
and a failed contract-only run whose alias failed too is not re-run. A
carried `success` written before the rename is also found under the
old gate name. Remove the input with the alias job.

A red contract-only check does not hold a merge in the meantime: GitHub's
docs do not say how several same-name check runs on one SHA are judged, and
Expand Down
Loading