Skip to content

feat(dependabot): require a named sanity check before any merge #208

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
twistedmelonman merged 2 commits into from
Oct 29, 2026
Merged

feat(dependabot): require a named sanity check before any merge #208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
94513c2
feat(dependabot): require a named sanity check before any merge
Oct 29, 2026
d4bfc9f
fix(dependabot): accept a multi-line sanity_checks value
Oct 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 45 additions & 5 deletions .github/workflows/dependabot-auto-merge.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ name: Dependabot Auto-Merge
# jobs:
# dependabot-auto-merge:
# uses: smartwatermelon/github-workflows/.github/workflows/dependabot-auto-merge.yml@dependabot-auto-merge-v3
# with:
# sanity_checks: 'tests' # checks that build or run the code; lint is not enough
#
# Do NOT add a `secrets:` block to the caller (see above).
#
Expand All @@ -68,6 +70,16 @@ name: Dependabot Auto-Merge
on:
workflow_call:
inputs:
sanity_checks:
description: >-
Comma-separated check names, as `gh pr checks` shows them, that
build or run the code (for example 'tests' or 'build'). Each must
run and pass before a merge; a matrix job 'test (3.12)' matches
'test'. Empty: nothing merges, and the PR is labeled
dependabot-blocked:no-sanity-check.
type: string
required: false
default: ''
trusted_namespaces:
description: >-
Ignored since v3: every dependency goes through the same gate.
Expand Down Expand Up @@ -141,6 +153,7 @@ jobs:
PREV_VERSION: ${{ steps.metadata.outputs.previous-version }}
NEW_VERSION: ${{ steps.metadata.outputs.new-version }}
DEP_NAMES: ${{ steps.metadata.outputs.dependency-names }}
SANITY_CHECKS: ${{ inputs.sanity_checks }}
TRUSTED_NAMESPACES: ${{ inputs.trusted_namespaces }}
ALLOW_MAJOR: ${{ inputs.allow_major }}
run: |
Expand All @@ -160,6 +173,13 @@ jobs:
echo "::notice::Not merged: empty dependency-names ($PREV_VERSION -> $NEW_VERSION)"
exit 0
fi
# Lint alone never proves the code still works (dev-env#229).
if [ -z "$(printf '%s' "${SANITY_CHECKS:-}" | tr -d ',[:space:]')" ]; then
echo "decision=blocked" >> "$GITHUB_OUTPUT"
echo "reason=no-sanity-check" >> "$GITHUB_OUTPUT"
echo "::notice::Not merged: the caller names no sanity_checks (a check that builds or runs the code)"
exit 0
fi
if [ -n "${TRUSTED_NAMESPACES:-}" ] || [ "${ALLOW_MAJOR:-false}" = "true" ]; then
echo "::notice::trusted_namespaces and allow_major are ignored since v3; remove them from the caller"
fi
Expand Down Expand Up @@ -190,6 +210,7 @@ jobs:
BASE_REF: ${{ github.event.pull_request.base.ref }}
RUN_ID: ${{ github.run_id }}
SELF_WORKFLOW: ${{ github.workflow }}
SANITY_CHECKS: ${{ inputs.sanity_checks }}
SETTLE_SECONDS: '120'
POLL_SECONDS: '30'
WAIT_SECONDS: '3600'
Expand All @@ -211,16 +232,23 @@ jobs:
| select(.state == "failure" or .state == "error") | .context') || return 1
printf '%s\n%s\n' "$runs" "$statuses"
}
sanity=$(printf '%s' "$SANITY_CHECKS" \
| jq -Rsc '[splits("[,\n]")] | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')
deadline=$(( $(date +%s) + WAIT_SECONDS ))
# Netlify checks register late: settle, then need two identical finished polls.
sleep "$SETTLE_SECONDS"
last=""
others=""
pending=""
missing=""
while :; do
if [ "$(date +%s)" -ge "$deadline" ]; then
if [ "$others" = "0" ]; then
stop blocked no-checks "only this workflow's checks ran; add a required check such as standards-check"
fi
if [ "$pending" = "0" ] && [ -n "$missing" ]; then
stop blocked no-sanity-check "sanity checks never passed: ${missing//$'\037'/, }"
fi
stop blocked timeout "checks still running after $(( WAIT_SECONDS / 60 )) min"
fi
head=$(gh pr view "$PR_URL" --json headRefOid --jq .headRefOid 2>/dev/null || true)
Expand All @@ -233,10 +261,10 @@ jobs:
fi
# Exits nonzero while pending/failing; judge the JSON.
json=$(gh pr checks "$PR_URL" --json name,workflow,bucket,link 2>/dev/null || true)
# bad_names is last: read collapses empty tab fields. Unknown buckets count as pending.
# read collapses empty tab fields, so the two name lists carry an m:/b: prefix.
# Names are joined with \u001f because job names can contain ", ".
if ! summary=$(printf '%s' "$json" | jq -er \
--arg run "/actions/runs/${RUN_ID}/" --arg wf "$SELF_WORKFLOW" '
--arg run "/actions/runs/${RUN_ID}/" --arg wf "$SELF_WORKFLOW" --argjson sanity "$sanity" '
if type != "array" then error("not an array") else . end
| (map(select((.link // "") | contains($run))) | length) as $self
| map(select(((.link // "") | contains($run)) | not)
Expand All @@ -250,11 +278,17 @@ jobs:
($bad | length),
($o | map(select(.bucket == "pass")) | length),
($o | map("\(.name)=\(.bucket)") | sort | join("|")),
($bad | map(.name) | join("\u001f")) ]
("m:" + ($sanity | map(. as $s | select(
[$o[] | select(.bucket == "pass"
and (.name == $s or (.name | startswith($s + " ("))))] | length == 0))
| join("\u001f"))),
("b:" + ($bad | map(.name) | join("\u001f"))) ]
| @tsv'); then
stop blocked cannot-evaluate "could not read the PR's checks"
fi
IFS=$'\t' read -r self others pending bad passed sig bad_names <<< "$summary"
IFS=$'\t' read -r self others pending bad passed sig missing bad_names <<< "$summary"
missing="${missing#m:}"
bad_names="${bad_names#b:}"
if [ "$self" -eq 0 ]; then
# No checks: read; seeing itself proves the list is not a permission blank.
stop blocked cannot-evaluate "this run cannot see its own check run (no checks: read on a private repo?)"
Expand All @@ -277,6 +311,12 @@ jobs:
if [ "$passed" -eq 0 ]; then
stop blocked no-checks "all $others other checks were skipped; nothing proved this head works"
fi
if [ -n "$missing" ]; then
# A late check may still register: keep polling until the deadline.
last=""
sleep "$POLL_SECONDS"
continue
fi
echo "gate=pass" >> "$GITHUB_OUTPUT"
echo "::notice::All $others other checks finished with no failure on $HEAD_SHA"
exit 0
Expand Down Expand Up @@ -325,7 +365,7 @@ jobs:
run: |
set -uo pipefail
case "${REASON:-}" in
red-check|main-red|timeout|no-checks|cannot-evaluate|merge-failed) ;;
red-check|main-red|timeout|no-checks|no-sanity-check|cannot-evaluate|merge-failed) ;;
*) REASON=cannot-evaluate ;;
esac
want="dependabot-blocked:${REASON}"
Expand Down
16 changes: 16 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,19 @@ gate covers the other risk, a bump that breaks the build.
self-visibility test turns that into a block, not a false "all green".
- The major/minor/group classification is logged, never used to decide.

### Sanity checks (required)

A green lint check proves nothing about whether the code still works.
Each caller names its sanity checks in `sanity_checks`: the checks that
build or run the code, such as `tests` or `build`. The gate merges only
when every named check ran and passed. A matrix job such as
`test (3.12)` matches the entry `test`. With no entry, nothing merges and
the PR is labeled `dependabot-blocked:no-sanity-check`.

A repo with no code to build (`.github`, `repo-template`) names
`standards-check / run-standards-check`, as a documented exception
(smartwatermelon/dev-env#229).

### Blocked PRs are labeled

When the gate does not merge a PR, it adds `dependabot-blocked` and one
Expand All @@ -162,6 +175,7 @@ these labels.
| `dependabot-blocked:red-check` | A check failed on this PR and is not failing on the base branch. |
| `dependabot-blocked:main-red` | Every failing check also fails on the base branch head. Fix `main`; the PR is probably fine. |
| `dependabot-blocked:timeout` | Checks were still running after 60 minutes. |
| `dependabot-blocked:no-sanity-check` | The caller names no `sanity_checks`, or a named check never ran or never passed. |
| `dependabot-blocked:no-checks` | No other check ran, or every other check was skipped. Add a required check such as `standards-check`. |
| `dependabot-blocked:cannot-evaluate` | Empty `dependency-names`, an unreadable PR head or check list, or the run cannot see its own check run. |
| `dependabot-blocked:merge-failed` | The gate passed but `gh pr merge` failed (head moved, approval missing, or branch protection). |
Expand Down Expand Up @@ -218,6 +232,8 @@ jobs:
# "Versioning" below), not per-action SHA pinning.
dependabot-auto-merge:
uses: smartwatermelon/github-workflows/.github/workflows/dependabot-auto-merge.yml@dependabot-auto-merge-v3
with:
sanity_checks: 'tests' # checks that build or run the code; see "Sanity checks"
```

Also add `can_approve_pull_request_reviews: true` to the repo's Actions
Expand Down
53 changes: 52 additions & 1 deletion tests/test-dependabot-auto-merge.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ policy() {
_fixture
rc=0
out="$(PREV_VERSION="$1" NEW_VERSION="$2" DEP_NAMES="$3" ALLOW_MAJOR="${4-false}" \
SANITY_CHECKS="${SANITY-tests}" \
TRUSTED_NAMESPACES="${5-}" GITHUB_OUTPUT="${STUB_DIR}/out" \
bash -c "${policy_sh}" 2>&1)" || rc=$?
decision="$(_out decision)"; reason="$(_out reason)"
Expand Down Expand Up @@ -65,6 +66,11 @@ _expect_policy "empty dependency-names -> blocked" blocked "empty dependency-nam
if [[ "${reason}" == "cannot-evaluate" ]]; then _ok "empty deps reason is cannot-evaluate"; else _bad "empty deps reason='${reason}'"; fi
policy 1.0.0 2.0.0 " "
_expect_policy "whitespace-only dependency-names -> blocked" blocked
SANITY="" policy 5.9.3 5.10.0 typescript
_expect_policy "no sanity_checks -> blocked before any wait" blocked "names no sanity_checks"
if [[ "${reason}" == "no-sanity-check" ]]; then _ok "reason is no-sanity-check"; else _bad "reason='${reason}'"; fi
SANITY=" , " policy 5.9.3 5.10.0 typescript
_expect_policy "blank sanity_checks entries -> blocked" blocked "names no sanity_checks"

# --- gate step ---------------------------------------------------------
sha="0123456789abcdef0123456789abcdef01234567"
Expand Down Expand Up @@ -93,7 +99,7 @@ gate() {
rc=0
out="$(PATH="${here}/stub-dependabot:${PATH}" GITHUB_OUTPUT="${STUB_DIR}/out" \
PR_URL=https://github.com/acme/site/pull/7 HEAD_SHA="${sha}" RUN_ID="${run_id}" \
REPO=acme/site BASE_REF=main \
REPO=acme/site BASE_REF=main SANITY_CHECKS="${SANITY-build}" \
SELF_WORKFLOW="Dependabot Auto-Merge" SETTLE_SECONDS=0 POLL_SECONDS=0.1 \
WAIT_SECONDS="${WAIT:-3}" bash -c "${gate_sh}" 2>&1)" || rc=$?
result="$(_out gate)"; reason="$(_out reason)"
Expand Down Expand Up @@ -200,6 +206,43 @@ _checks 1 self "build|skipping" "deploy|skipping"
WAIT=30 gate
_expect_gate "all other checks skipped -> no-checks" blocked no-checks "all 2 other checks were skipped"

# 15. matrix job name matches its sanity entry
_fixture; echo "${sha}" >"${STUB_DIR}/head"
_checks 1 self "test (3.11)|pass" "test (3.12)|pass" "standards|pass"
SANITY="test" gate
_expect_gate "matrix 'test (3.12)' satisfies sanity 'test'" pass "" "All 3 other checks"

# 16. all green, but the named sanity check never ran -> no-sanity-check at the deadline
_fixture; echo "${sha}" >"${STUB_DIR}/head"
_checks 1 self "standards|pass"
SANITY="build, tests" WAIT=1 gate
_expect_gate "sanity checks missing -> no-sanity-check" blocked no-sanity-check "sanity checks never passed: build, tests"

# 17. sanity check skipped -> it never passed
_fixture; echo "${sha}" >"${STUB_DIR}/head"
_checks 1 self "build|skipping" "standards|pass"
WAIT=1 gate
_expect_gate "skipped sanity check -> no-sanity-check" blocked no-sanity-check "build"

# 18. sanity check registers late -> keeps polling, then passes
_fixture; echo "${sha}" >"${STUB_DIR}/head"
_checks 1 self "standards|pass"
_checks 3 self "standards|pass" "build|pass"
WAIT=5 gate
_expect_gate "late sanity check -> pass" pass "" "All 2 other checks"

# 19. a prefix that is not a matrix suffix does not match
_fixture; echo "${sha}" >"${STUB_DIR}/head"
_checks 1 self "build-docs|pass"
WAIT=1 gate
_expect_gate "'build-docs' does not satisfy 'build'" blocked no-sanity-check "build"

# 20. a multi-line sanity_checks (YAML block scalar) splits on newlines too
_fixture; echo "${sha}" >"${STUB_DIR}/head"
_checks 1 self "build|pass" "tests|pass"
SANITY=$'build\ntests\n' gate
_expect_gate "multi-line sanity_checks -> pass" pass "" "All 2 other checks"

# --- merge step --------------------------------------------------------
merge() {
rc=0
Expand Down Expand Up @@ -246,6 +289,14 @@ else
_bad "label: rc=${rc} calls=$(<"${STUB_DIR}/calls.log")"
fi

_fixture
label no-sanity-check
if _called "--add-label dependabot-blocked,dependabot-blocked:no-sanity-check"; then
_ok "no-sanity-check is a known reason"
else
_bad "no-sanity-check label: $(<"${STUB_DIR}/calls.log")"
fi

_fixture
label 'bogus;reason'
if _called "--add-label dependabot-blocked,dependabot-blocked:cannot-evaluate"; then
Expand Down
Loading