diff --git a/.github/scripts/ci-wait-or-fallback.sh b/.github/scripts/ci-wait-or-fallback.sh new file mode 100755 index 000000000..da5cdcddc --- /dev/null +++ b/.github/scripts/ci-wait-or-fallback.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +# +# Decide whether a self-hosted *fallback* job should DEFER to its GitHub-hosted +# *primary* job (because GitHub is running it) or SELF-RUN the tests because +# GitHub never scheduled the primary within the start deadline. +# +# This is the core of the "GitHub-first, fail over to self-hosted on a +# scheduling stall, only one actually runs the tests" pattern used by the +# ubuntu-fallback job in ci.yml. The GitHub primaries are the required checks; +# this convenience job DEFERS (exits, does nothing) as soon as GitHub starts a +# primary, and only self-runs a leg when GitHub never schedules it (a stall). +# +# Usage: +# ci-wait-or-fallback.sh "" +# +# Exact job name as it appears in the run's jobs list, +# e.g. "st-sim-a2a3 (ubuntu-latest, 3.10)". Matrix legs +# include the "(os, py)" suffix -- if the matrix changes, +# this string must change too (see RENAME_ERROR below). +# How long to wait for the primary to leave "queued" +# before declaring a stall (600 = 10 min). +# 1 if the primary has no `needs` and should appear in +# the jobs list almost immediately (e.g. `ut`); 0 if it +# is gated by `needs: detect-changes` and may appear late. +# +# Requires env: GITHUB_TOKEN (with actions:read), GITHUB_API_URL, +# GITHUB_REPOSITORY, GITHUB_RUN_ID. +# +# Prints exactly one decision token as the LAST line of stdout; all diagnostics +# go to stderr. Exit code is always 0 -- the caller branches on the token: +# DEFER primary is running or finished on GitHub: it gates itself, +# nothing for this job to do +# SELFRUN GitHub never scheduled the primary (stall): run the tests locally +# RENAME_ERROR primary never appeared though it should have (likely a +# matrix/name change broke matching): log, do not self-run +# +set -uo pipefail + +PRIMARY="${1:?primary job name required}" +START_TIMEOUT="${2:-600}" +EXPECT_FAST="${3:-0}" + +: "${GITHUB_TOKEN:?}" "${GITHUB_API_URL:?}" "${GITHUB_REPOSITORY:?}" "${GITHUB_RUN_ID:?}" + +API="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" +POLL=15 +elapsed=0 +seen=0 # have we ever observed the primary job in the jobs list? + +log() { printf '[fallback] %s\n' "$*" >&2; } + +fetch() { + curl -sf \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${API}" +} + +field() { # field (e.g. status, conclusion) + # python3 (always present on these runners) instead of jq (often absent). + printf '%s' "$1" | python3 -c ' +import json, sys +try: + data = json.load(sys.stdin) +except Exception: + sys.exit(0) +name, key = sys.argv[1], sys.argv[2] +for j in data.get("jobs", []): + if j.get("name") == name: + v = j.get(key) + print("" if v is None else v) + break +' "$2" "$3" +} + +while true; do + JSON="$(fetch || true)" + if [ -z "${JSON}" ]; then + log "jobs API fetch failed; retrying in ${POLL}s (elapsed=${elapsed}s)" + sleep "${POLL}"; elapsed=$((elapsed + POLL)); continue + fi + + STATUS="$(field "${JSON}" "${PRIMARY}" 'status')" + CONCL="$(field "${JSON}" "${PRIMARY}" 'conclusion')" + + if [ -n "${STATUS}" ] && [ "${STATUS}" != "null" ]; then + seen=1 + case "${STATUS}" in + in_progress|completed) + # GitHub is running or has finished the primary. The primary is the + # required check and gates itself (its pass/fail/skip is authoritative), + # so this convenience job has nothing to do. + log "primary status=${STATUS} (conclusion=${CONCL:-n/a}) -> DEFER (primary gates)" + echo "DEFER"; exit 0 ;; + *) + # queued / waiting / pending / requested -> not started yet. + : ;; + esac + fi + + # Primary is queued or not yet present. Enforce the start deadline. + if [ "${elapsed}" -ge "${START_TIMEOUT}" ]; then + if [ "${seen}" -eq 0 ]; then + # Never once observed. Distinguish "stall" from "matching broke". + if [ "${EXPECT_FAST}" -eq 1 ]; then + printf '::error::Fallback never observed primary job %s within %ss. It has no `needs` and should appear immediately -- the job name almost certainly drifted (matrix os/python change). Update the primary name in ci.yml.\n' "${PRIMARY}" "${START_TIMEOUT}" >&2 + echo "RENAME_ERROR"; exit 0 + fi + DC_STATUS="$(field "${JSON}" 'detect-changes' 'status')" + if [ "${DC_STATUS}" = "completed" ]; then + printf '::error::Fallback never observed primary job %s though detect-changes has completed -- likely a renamed matrix leg or a pruned primary. Refusing to self-run blindly.\n' "${PRIMARY}" >&2 + echo "RENAME_ERROR"; exit 0 + fi + # detect-changes has NOT completed (stuck or still running on GitHub), so + # the primary was never created. This fallback already confirmed via its + # own arch-diff that the work is needed, so a stalled detect-changes is + # just another GitHub scheduling stall -- take over rather than wait + # forever for a primary that will never appear. + printf '::warning::Primary %s never appeared and detect-changes has not completed (status=%s) within %ss -- detect-changes itself appears stalled; self-running on this self-hosted runner.\n' "${PRIMARY}" "${DC_STATUS:-absent}" "${START_TIMEOUT}" >&2 + echo "SELFRUN"; exit 0 + fi + printf '::warning::GitHub did not start primary job %s within %ss -- self-running the tests on this self-hosted runner.\n' "${PRIMARY}" "${START_TIMEOUT}" >&2 + echo "SELFRUN"; exit 0 + fi + + sleep "${POLL}"; elapsed=$((elapsed + POLL)) +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de8e406eb..a6ef50ac4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,8 @@ name: CI permissions: contents: read pull-requests: read + # Lets the ubuntu-fallback job poll sibling job status. + actions: read on: pull_request: @@ -720,3 +722,118 @@ jobs: DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))") PYTEST="python -m pytest examples tests/st --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa" task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "$PYTEST --pto-session-timeout 1200" + + # ---------- Self-hosted fail-over for the ubuntu-latest legs ---------- + # The GitHub primaries (ut / st-sim-a2a3 / st-sim-a5) are the REQUIRED checks + # and gate on their own results. This job is a non-required convenience for the + # rare case where GitHub's hosted pool is congested and never schedules a leg: + # it polls the three primaries in parallel and, per leg, DEFERS the moment + # GitHub starts running it (exits, freeing the sim slot in seconds) and only + # self-runs a leg that GitHub never scheduled. On such a stall the required + # primary stays pending -- admin-merge using this job's green self-run result. + # Do NOT make this the required check. GCC 15 / gtest are expected on the + # runner. Detects arch changes itself so a detect-changes stall cannot block it. + ubuntu-fallback: + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: [self-hosted, sim] + timeout-minutes: 90 + env: + PTO2_SCHEDULER_TIMEOUT_MS: "5000" + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + # Same rule as the detect-changes job; inlined so a detect-changes stall + # cannot block this fallback. + - name: Detect arch changes + id: arch + run: | + FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}) + NON_CODE='^(docs/|\.docs/|\.claude/|KNOWN_ISSUES\.md$|\.gitignore$|README\.md$|\.pre-commit-config\.yaml$)' + A5_ONLY='^(src/a5/|examples/a5/|tests/(st|device_tests)/a5/)' + A2A3_ONLY='^(src/a2a3/|examples/a2a3/|tests/(st|device_tests)/a2a3/)' + A2A3_REMAINING=$(echo "$FILES" | grep -vE "$A5_ONLY" | grep -vE "$NON_CODE" || true) + A5_REMAINING=$(echo "$FILES" | grep -vE "$A2A3_ONLY" | grep -vE "$NON_CODE" || true) + [ -n "$A2A3_REMAINING" ] && echo "a2a3=true" >> "$GITHUB_OUTPUT" || echo "a2a3=false" >> "$GITHUB_OUTPUT" + [ -n "$A5_REMAINING" ] && echo "a5=true" >> "$GITHUB_OUTPUT" || echo "a5=false" >> "$GITHUB_OUTPUT" + + # Poll each ubuntu primary in parallel; ci-wait-or-fallback.sh prints one of + # DEFER (GitHub is running it) / SELFRUN (stall) / RENAME_ERROR per leg. + - name: Decide defer vs self-run + id: decide + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + poll() { .github/scripts/ci-wait-or-fallback.sh "$1" 600 "$2" > "$3" 2> "$3.log" & } + poll "ut (ubuntu-latest, 3.10)" 1 ut.dec + [ "${{ steps.arch.outputs.a2a3 }}" = "true" ] && poll "st-sim-a2a3 (ubuntu-latest, 3.10)" 0 a2a3.dec + [ "${{ steps.arch.outputs.a5 }}" = "true" ] && poll "st-sim-a5 (ubuntu-latest, 3.10)" 0 a5.dec + wait + get() { if [ -f "$1" ]; then tail -n1 "$1"; else echo SKIP; fi; } + UT=$(get ut.dec); A2A3=$(get a2a3.dec); A5=$(get a5.dec) + for f in ut a2a3 a5; do [ -f "$f.dec.log" ] && { echo "--- $f ---"; cat "$f.dec.log"; }; done + echo "ut=$UT" >> "$GITHUB_OUTPUT" + echo "a2a3=$A2A3" >> "$GITHUB_OUTPUT" + echo "a5=$A5" >> "$GITHUB_OUTPUT" + echo "Decisions -> ut=$UT a2a3=$A2A3 a5=$A5" + + - name: Announce stalled legs + if: steps.decide.outputs.ut == 'SELFRUN' || steps.decide.outputs.a2a3 == 'SELFRUN' || steps.decide.outputs.a5 == 'SELFRUN' + run: | + { + echo "### ⚠ GitHub stalled some ubuntu legs — self-running here" + echo "Their required primaries stay pending; admin-merge once this job is green:" + if [ "${{ steps.decide.outputs.ut }}" = "SELFRUN" ]; then echo "- ut"; fi + if [ "${{ steps.decide.outputs.a2a3 }}" = "SELFRUN" ]; then echo "- st-sim-a2a3"; fi + if [ "${{ steps.decide.outputs.a5 }}" = "SELFRUN" ]; then echo "- st-sim-a5"; fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Install dependencies (any self-run) + if: steps.decide.outputs.ut == 'SELFRUN' || steps.decide.outputs.a2a3 == 'SELFRUN' || steps.decide.outputs.a5 == 'SELFRUN' + run: | + python3 -m venv --system-site-packages .venv + source .venv/bin/activate + pip install --upgrade pip + pip install torch --index-url https://download.pytorch.org/whl/cpu + pip install '.[test]' + + - name: ut self-run + if: steps.decide.outputs.ut == 'SELFRUN' + run: | + source .venv/bin/activate + python -m pytest tests/ut -m "not requires_hardware" -v + cmake -B tests/ut/cpp/build -S tests/ut/cpp + cmake --build tests/ut/cpp/build + ctest --test-dir tests/ut/cpp/build -LE requires_hardware --output-on-failure + + - name: st-sim-a2a3 self-run + if: steps.decide.outputs.a2a3 == 'SELFRUN' + run: | + source .venv/bin/activate + C="--platform a2a3sim --device 0-15 --pto-session-timeout 600 --require-pto-isa" + D="tests/st/a2a3/tensormap_and_ringbuffer/dfx" + python -m pytest examples tests/st $C -v + python -m pytest $D/dep_gen/test_dep_gen.py $C -p no:xdist --enable-dep-gen + python -m pytest $D/l2_swimlane/ $C -p no:xdist --enable-l2-swimlane --enable-dep-gen + python -m pytest $D/pmu/test_pmu.py $C -p no:xdist --enable-pmu 2 + python -m pytest $D/args_dump/test_args_dump.py $C -p no:xdist --dump-args + + - name: st-sim-a5 self-run + if: steps.decide.outputs.a5 == 'SELFRUN' + run: | + source .venv/bin/activate + python -m pytest examples tests/st --platform a5sim --device 0-15 -v \ + --pto-session-timeout 600 --require-pto-isa + + # A leg that self-ran fails this job via its own step above. RENAME_ERROR / + # ERROR only warn: this job is non-required, and a mismatched primary name + # should be noticed and fixed, not silently self-run. + - name: Warn on unresolved legs + run: | + for p in "ut=${{ steps.decide.outputs.ut }}" "a2a3=${{ steps.decide.outputs.a2a3 }}" "a5=${{ steps.decide.outputs.a5 }}"; do + case "${p#*=}" in + RENAME_ERROR|ERROR) echo "::warning::ubuntu leg ${p%%=*} -> ${p#*=} (check the primary job name in ci.yml)" ;; + esac + done