diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml index 46b7cfa..dc1a2da 100644 --- a/.github/workflows/dependabot-auto-merge.yml +++ b/.github/workflows/dependabot-auto-merge.yml @@ -53,6 +53,8 @@ name: Dependabot Auto-Merge # jobs: # dependabot-auto-merge: # uses: smartwatermelon/github-workflows/.github/workflows/dependabot-auto-merge.yml@dependabot-auto-merge-v3 +# with: +# sanity_checks: 'tests' # checks that build or run the code; lint is not enough # # Do NOT add a `secrets:` block to the caller (see above). # @@ -68,6 +70,16 @@ name: Dependabot Auto-Merge on: workflow_call: inputs: + sanity_checks: + description: >- + Comma-separated check names, as `gh pr checks` shows them, that + build or run the code (for example 'tests' or 'build'). Each must + run and pass before a merge; a matrix job 'test (3.12)' matches + 'test'. Empty: nothing merges, and the PR is labeled + dependabot-blocked:no-sanity-check. + type: string + required: false + default: '' trusted_namespaces: description: >- Ignored since v3: every dependency goes through the same gate. @@ -141,6 +153,7 @@ jobs: PREV_VERSION: ${{ steps.metadata.outputs.previous-version }} NEW_VERSION: ${{ steps.metadata.outputs.new-version }} DEP_NAMES: ${{ steps.metadata.outputs.dependency-names }} + SANITY_CHECKS: ${{ inputs.sanity_checks }} TRUSTED_NAMESPACES: ${{ inputs.trusted_namespaces }} ALLOW_MAJOR: ${{ inputs.allow_major }} run: | @@ -160,6 +173,13 @@ jobs: echo "::notice::Not merged: empty dependency-names ($PREV_VERSION -> $NEW_VERSION)" exit 0 fi + # Lint alone never proves the code still works (dev-env#229). + if [ -z "$(printf '%s' "${SANITY_CHECKS:-}" | tr -d ',[:space:]')" ]; then + echo "decision=blocked" >> "$GITHUB_OUTPUT" + echo "reason=no-sanity-check" >> "$GITHUB_OUTPUT" + echo "::notice::Not merged: the caller names no sanity_checks (a check that builds or runs the code)" + exit 0 + fi if [ -n "${TRUSTED_NAMESPACES:-}" ] || [ "${ALLOW_MAJOR:-false}" = "true" ]; then echo "::notice::trusted_namespaces and allow_major are ignored since v3; remove them from the caller" fi @@ -190,6 +210,7 @@ jobs: BASE_REF: ${{ github.event.pull_request.base.ref }} RUN_ID: ${{ github.run_id }} SELF_WORKFLOW: ${{ github.workflow }} + SANITY_CHECKS: ${{ inputs.sanity_checks }} SETTLE_SECONDS: '120' POLL_SECONDS: '30' WAIT_SECONDS: '3600' @@ -211,16 +232,23 @@ jobs: | select(.state == "failure" or .state == "error") | .context') || return 1 printf '%s\n%s\n' "$runs" "$statuses" } + sanity=$(printf '%s' "$SANITY_CHECKS" \ + | jq -Rsc '[splits("[,\n]")] | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))') deadline=$(( $(date +%s) + WAIT_SECONDS )) # Netlify checks register late: settle, then need two identical finished polls. sleep "$SETTLE_SECONDS" last="" others="" + pending="" + missing="" while :; do if [ "$(date +%s)" -ge "$deadline" ]; then if [ "$others" = "0" ]; then stop blocked no-checks "only this workflow's checks ran; add a required check such as standards-check" fi + if [ "$pending" = "0" ] && [ -n "$missing" ]; then + stop blocked no-sanity-check "sanity checks never passed: ${missing//$'\037'/, }" + fi stop blocked timeout "checks still running after $(( WAIT_SECONDS / 60 )) min" fi head=$(gh pr view "$PR_URL" --json headRefOid --jq .headRefOid 2>/dev/null || true) @@ -233,10 +261,10 @@ jobs: fi # Exits nonzero while pending/failing; judge the JSON. json=$(gh pr checks "$PR_URL" --json name,workflow,bucket,link 2>/dev/null || true) - # bad_names is last: read collapses empty tab fields. Unknown buckets count as pending. + # read collapses empty tab fields, so the two name lists carry an m:/b: prefix. # Names are joined with \u001f because job names can contain ", ". if ! summary=$(printf '%s' "$json" | jq -er \ - --arg run "/actions/runs/${RUN_ID}/" --arg wf "$SELF_WORKFLOW" ' + --arg run "/actions/runs/${RUN_ID}/" --arg wf "$SELF_WORKFLOW" --argjson sanity "$sanity" ' if type != "array" then error("not an array") else . end | (map(select((.link // "") | contains($run))) | length) as $self | map(select(((.link // "") | contains($run)) | not) @@ -250,11 +278,17 @@ jobs: ($bad | length), ($o | map(select(.bucket == "pass")) | length), ($o | map("\(.name)=\(.bucket)") | sort | join("|")), - ($bad | map(.name) | join("\u001f")) ] + ("m:" + ($sanity | map(. as $s | select( + [$o[] | select(.bucket == "pass" + and (.name == $s or (.name | startswith($s + " ("))))] | length == 0)) + | join("\u001f"))), + ("b:" + ($bad | map(.name) | join("\u001f"))) ] | @tsv'); then stop blocked cannot-evaluate "could not read the PR's checks" fi - IFS=$'\t' read -r self others pending bad passed sig bad_names <<< "$summary" + IFS=$'\t' read -r self others pending bad passed sig missing bad_names <<< "$summary" + missing="${missing#m:}" + bad_names="${bad_names#b:}" if [ "$self" -eq 0 ]; then # No checks: read; seeing itself proves the list is not a permission blank. stop blocked cannot-evaluate "this run cannot see its own check run (no checks: read on a private repo?)" @@ -277,6 +311,12 @@ jobs: if [ "$passed" -eq 0 ]; then stop blocked no-checks "all $others other checks were skipped; nothing proved this head works" fi + if [ -n "$missing" ]; then + # A late check may still register: keep polling until the deadline. + last="" + sleep "$POLL_SECONDS" + continue + fi echo "gate=pass" >> "$GITHUB_OUTPUT" echo "::notice::All $others other checks finished with no failure on $HEAD_SHA" exit 0 @@ -325,7 +365,7 @@ jobs: run: | set -uo pipefail case "${REASON:-}" in - red-check|main-red|timeout|no-checks|cannot-evaluate|merge-failed) ;; + red-check|main-red|timeout|no-checks|no-sanity-check|cannot-evaluate|merge-failed) ;; *) REASON=cannot-evaluate ;; esac want="dependabot-blocked:${REASON}" diff --git a/README.md b/README.md index 32b3850..09d570f 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,19 @@ gate covers the other risk, a bump that breaks the build. self-visibility test turns that into a block, not a false "all green". - The major/minor/group classification is logged, never used to decide. +### Sanity checks (required) + +A green lint check proves nothing about whether the code still works. +Each caller names its sanity checks in `sanity_checks`: the checks that +build or run the code, such as `tests` or `build`. The gate merges only +when every named check ran and passed. A matrix job such as +`test (3.12)` matches the entry `test`. With no entry, nothing merges and +the PR is labeled `dependabot-blocked:no-sanity-check`. + +A repo with no code to build (`.github`, `repo-template`) names +`standards-check / run-standards-check`, as a documented exception +(smartwatermelon/dev-env#229). + ### Blocked PRs are labeled When the gate does not merge a PR, it adds `dependabot-blocked` and one @@ -162,6 +175,7 @@ these labels. | `dependabot-blocked:red-check` | A check failed on this PR and is not failing on the base branch. | | `dependabot-blocked:main-red` | Every failing check also fails on the base branch head. Fix `main`; the PR is probably fine. | | `dependabot-blocked:timeout` | Checks were still running after 60 minutes. | +| `dependabot-blocked:no-sanity-check` | The caller names no `sanity_checks`, or a named check never ran or never passed. | | `dependabot-blocked:no-checks` | No other check ran, or every other check was skipped. Add a required check such as `standards-check`. | | `dependabot-blocked:cannot-evaluate` | Empty `dependency-names`, an unreadable PR head or check list, or the run cannot see its own check run. | | `dependabot-blocked:merge-failed` | The gate passed but `gh pr merge` failed (head moved, approval missing, or branch protection). | @@ -218,6 +232,8 @@ jobs: # "Versioning" below), not per-action SHA pinning. dependabot-auto-merge: uses: smartwatermelon/github-workflows/.github/workflows/dependabot-auto-merge.yml@dependabot-auto-merge-v3 + with: + sanity_checks: 'tests' # checks that build or run the code; see "Sanity checks" ``` Also add `can_approve_pull_request_reviews: true` to the repo's Actions diff --git a/tests/test-dependabot-auto-merge.sh b/tests/test-dependabot-auto-merge.sh index 7ba40e0..b39d2ee 100755 --- a/tests/test-dependabot-auto-merge.sh +++ b/tests/test-dependabot-auto-merge.sh @@ -38,6 +38,7 @@ policy() { _fixture rc=0 out="$(PREV_VERSION="$1" NEW_VERSION="$2" DEP_NAMES="$3" ALLOW_MAJOR="${4-false}" \ + SANITY_CHECKS="${SANITY-tests}" \ TRUSTED_NAMESPACES="${5-}" GITHUB_OUTPUT="${STUB_DIR}/out" \ bash -c "${policy_sh}" 2>&1)" || rc=$? decision="$(_out decision)"; reason="$(_out reason)" @@ -65,6 +66,11 @@ _expect_policy "empty dependency-names -> blocked" blocked "empty dependency-nam if [[ "${reason}" == "cannot-evaluate" ]]; then _ok "empty deps reason is cannot-evaluate"; else _bad "empty deps reason='${reason}'"; fi policy 1.0.0 2.0.0 " " _expect_policy "whitespace-only dependency-names -> blocked" blocked +SANITY="" policy 5.9.3 5.10.0 typescript +_expect_policy "no sanity_checks -> blocked before any wait" blocked "names no sanity_checks" +if [[ "${reason}" == "no-sanity-check" ]]; then _ok "reason is no-sanity-check"; else _bad "reason='${reason}'"; fi +SANITY=" , " policy 5.9.3 5.10.0 typescript +_expect_policy "blank sanity_checks entries -> blocked" blocked "names no sanity_checks" # --- gate step --------------------------------------------------------- sha="0123456789abcdef0123456789abcdef01234567" @@ -93,7 +99,7 @@ gate() { rc=0 out="$(PATH="${here}/stub-dependabot:${PATH}" GITHUB_OUTPUT="${STUB_DIR}/out" \ PR_URL=https://github.com/acme/site/pull/7 HEAD_SHA="${sha}" RUN_ID="${run_id}" \ - REPO=acme/site BASE_REF=main \ + REPO=acme/site BASE_REF=main SANITY_CHECKS="${SANITY-build}" \ SELF_WORKFLOW="Dependabot Auto-Merge" SETTLE_SECONDS=0 POLL_SECONDS=0.1 \ WAIT_SECONDS="${WAIT:-3}" bash -c "${gate_sh}" 2>&1)" || rc=$? result="$(_out gate)"; reason="$(_out reason)" @@ -200,6 +206,43 @@ _checks 1 self "build|skipping" "deploy|skipping" WAIT=30 gate _expect_gate "all other checks skipped -> no-checks" blocked no-checks "all 2 other checks were skipped" +# 15. matrix job name matches its sanity entry +_fixture; echo "${sha}" >"${STUB_DIR}/head" +_checks 1 self "test (3.11)|pass" "test (3.12)|pass" "standards|pass" +SANITY="test" gate +_expect_gate "matrix 'test (3.12)' satisfies sanity 'test'" pass "" "All 3 other checks" + +# 16. all green, but the named sanity check never ran -> no-sanity-check at the deadline +_fixture; echo "${sha}" >"${STUB_DIR}/head" +_checks 1 self "standards|pass" +SANITY="build, tests" WAIT=1 gate +_expect_gate "sanity checks missing -> no-sanity-check" blocked no-sanity-check "sanity checks never passed: build, tests" + +# 17. sanity check skipped -> it never passed +_fixture; echo "${sha}" >"${STUB_DIR}/head" +_checks 1 self "build|skipping" "standards|pass" +WAIT=1 gate +_expect_gate "skipped sanity check -> no-sanity-check" blocked no-sanity-check "build" + +# 18. sanity check registers late -> keeps polling, then passes +_fixture; echo "${sha}" >"${STUB_DIR}/head" +_checks 1 self "standards|pass" +_checks 3 self "standards|pass" "build|pass" +WAIT=5 gate +_expect_gate "late sanity check -> pass" pass "" "All 2 other checks" + +# 19. a prefix that is not a matrix suffix does not match +_fixture; echo "${sha}" >"${STUB_DIR}/head" +_checks 1 self "build-docs|pass" +WAIT=1 gate +_expect_gate "'build-docs' does not satisfy 'build'" blocked no-sanity-check "build" + +# 20. a multi-line sanity_checks (YAML block scalar) splits on newlines too +_fixture; echo "${sha}" >"${STUB_DIR}/head" +_checks 1 self "build|pass" "tests|pass" +SANITY=$'build\ntests\n' gate +_expect_gate "multi-line sanity_checks -> pass" pass "" "All 2 other checks" + # --- merge step -------------------------------------------------------- merge() { rc=0 @@ -246,6 +289,14 @@ else _bad "label: rc=${rc} calls=$(<"${STUB_DIR}/calls.log")" fi +_fixture +label no-sanity-check +if _called "--add-label dependabot-blocked,dependabot-blocked:no-sanity-check"; then + _ok "no-sanity-check is a known reason" +else + _bad "no-sanity-check label: $(<"${STUB_DIR}/calls.log")" +fi + _fixture label 'bogus;reason' if _called "--add-label dependabot-blocked,dependabot-blocked:cannot-evaluate"; then