fix(tests): wait for token-gate listeners after apply

Makefile

@@ -18,6 +18,7 @@ test: lint test-suite ## Lint + the dependency-free suite (runs on macOS or Linu
 
 test-suite: ## rigforge test suite: unit + black-box, every CPU/OS profile simulated
 	bash tests/run.sh
+	bash tests/test-access-token-readiness.sh
 
 test-e2e: ## Full end-to-end run in disposable Linux containers (needs Docker)
 	bash tests/e2e/linux.sh

tests/README.md

@@ -18,7 +18,7 @@ compiles XMRig and mines. Each layer covers what the one below it has to stub.
 | **Coverage gate** | [`coverage.sh`](coverage.sh) | kcov in **Docker**. **In CI.** | Line coverage of `rigforge.sh` + `util/proposed-grub.sh` by running `run.sh` under kcov; enforces the committed floor ([`coverage-floor.txt`](coverage-floor.txt)) plus a patch-coverage gate (diff-cover) on changed lines. | `make coverage` |
 | **Release smoke (quick)** | [`smoke.sh`](smoke.sh) | Real Linux rig, **manual**. Not in CI. | The compiled binary actually starts and hashes (`xmrig --bench`, fully offline). Fast pre-tag confidence that the worker we ship runs. | `make smoke` |
 | **Release e2e (full)** | [`e2e-real.sh`](e2e-real.sh) | Real Linux rig, **manual, root**. Not in CI. | The real thing end to end: build + tune + kernel tuning + service + a real hash, the writable control path (#236) against real systemd (`control` phase), the remote-upgrade chain with real git as the root oneshot (`upgrade` phase, #322), then a clean uninstall. **The release gate.** | `make e2e-real` (see [`RELEASING.md`](../RELEASING.md)) |
-| **Release e2e (worker↔stack)** | [`e2e-pithead.sh`](e2e-pithead.sh) ([`control` phase](e2e-pithead-control.sh), [`access-tokens` phase](e2e-pithead-tokens.sh)) | Real rig + a **live Pithead stack**, **manual, root**. Not in CI (`connect`/`control`/`access-tokens` also run as a `rigforge`/`tier4-e2e` bench-ci job). | The integration contract: the `network` phase records fixed boolean summary diagnostics (`network-summary`) before enforcing metadata and the XMRig key superset, without logging response bodies or key names; `network-sequence` reproduces the nightly `connect` → `worker-api` → `api-impact` → `network` prefix in one process/snapshot and stops there (separate phase submissions restore between phases); mining round-trip against the stack, the `:8080` API posture (open/token/`restricted`), the writable control path's DONATION apply round trip (#344/#509), a named `ACCESS_TOKENS` entry authenticating `:8081`/`:8082` while staying refused on `:8080` and after revocation (#516), stratum auth accept/reject + rotation against only `PITHEAD_URL` (a temporary plain-Stratum pool preserves the primary `user`, replaces endpoint-specific settings and removes all fallbacks; URL/password enter jq through stdin with constant programs; EXIT restores the original config bytes and runtime even on failure; explicit `stratum-auth` fails with a sanitized `missing` diagnostic when `E2E_STRATUM_PASS` is absent; `all` retains its optional skip), dashboard visibility/drop-off, dev-fee independence, and the sister-API **hashrate-impact guard** (#99 must not shave H/s under polling load). | `PITHEAD_URL=host:3333 sudo -E make e2e-pithead` (see [`RELEASING.md`](../RELEASING.md)) |
+| **Release e2e (worker↔stack)** | [`e2e-pithead.sh`](e2e-pithead.sh) ([`control` phase](e2e-pithead-control.sh), [`access-tokens` phase](e2e-pithead-tokens.sh)) | Real rig + a **live Pithead stack**, **manual, root**. Not in CI (`connect`/`control`/`access-tokens` also run as a `rigforge`/`tier4-e2e` bench-ci job). | The integration contract: the `network` phase records fixed boolean summary diagnostics (`network-summary`) before enforcing metadata and the XMRig key superset, without logging response bodies or key names; `network-sequence` reproduces the nightly `connect` → `worker-api` → `api-impact` → `network` prefix in one process/snapshot and stops there (separate phase submissions restore between phases); mining round-trip against the stack, the `:8080` API posture (open/token/`restricted`), the writable control path's DONATION apply round trip (#344/#509), a named `ACCESS_TOKENS` entry authenticating `:8081`/`:8082` while staying refused on `:8080` and after revocation (#516); setup/revocation wait up to 30 s per listener for a bearer-free GET to return HTTP 401, recording elapsed time or sanitized curl/HTTP/unit diagnostics (#573), then separately require the revoked bearer to receive HTTP 401, stratum auth accept/reject + rotation against only `PITHEAD_URL` (a temporary plain-Stratum pool preserves the primary `user`, replaces endpoint-specific settings and removes all fallbacks; URL/password enter jq through stdin with constant programs; EXIT restores the original config bytes and runtime even on failure; explicit `stratum-auth` fails with a sanitized `missing` diagnostic when `E2E_STRATUM_PASS` is absent; `all` retains its optional skip), dashboard visibility/drop-off, dev-fee independence, and the sister-API **hashrate-impact guard** (#99 must not shave H/s under polling load). | `PITHEAD_URL=host:3333 sudo -E make e2e-pithead` (see [`RELEASING.md`](../RELEASING.md)) |
 
 The unit + black-box suite, the Linux container e2e, and the coverage gate run automatically on every
 push/PR (see [`.github/workflows/ci.yml`](../.github/workflows/ci.yml)). The native macOS e2e is

tests/e2e-pithead-tokens.sh

@@ -2,17 +2,47 @@
 # ACCESS_TOKENS phase for e2e-pithead.sh (#516), split out to stay under e2e-pithead.sh's file
 # budget (docs/dev/file-budget.tsv) — same convention as e2e-pithead-control.sh. Sourced by
 # e2e-pithead.sh AFTER phase()/ok()/bad()/set_cfg()/$CFG are defined; must not be run standalone.
+# Probe without a bearer or a write: HTTP 401 establishes a listening, gated service,
+# not successful revocation. The revoked bearer must still get its own HTTP 401 below.
+_access_tokens_ready() { # <api|control> <port> <setup|revocation>
+    local service="$1" port="$2" stage="$3" path=health started=$SECONDS code=000 rc=0 active=0 remaining
+    [ "$service" != control ] || path=status
+    while [ "$((SECONDS - started))" -lt 30 ]; do
+        remaining=$((30 - SECONDS + started))
+        [ "$remaining" -gt 0 ] || break # curl treats --max-time 0 as unlimited
+        [ "$remaining" -le 2 ] || remaining=2
+        rc=0
+        code=$(curl -q --noproxy '*' -s -o /dev/null -w '%{http_code}' --max-time "$remaining" \
+            "http://127.0.0.1:$port/$path" 2>/dev/null) || rc=$?
+        if [ "$rc" = 0 ] && [ "$code" = 401 ]; then
+            printf 'access-token-readiness: service=%s stage=%s elapsed_s=%s\n' "$service" "$stage" "$((SECONDS - started))"
+            return 0
+        fi
+        # Only a refused/timed-out connection is a startup window. A reachable service
+        # with unexpected auth posture, or any other curl error, is a failure immediately.
+        case "$rc:$code" in 7:000 | 28:000) ;; *) break ;; esac
+        [ "$((SECONDS - started))" -ge 30 ] || sleep 1
+    done
+    systemctl is-active --quiet "rigforge-$service.service" 2>/dev/null && active=1
+    printf 'access-token-readiness: service=%s stage=%s elapsed_s=%s transport_rc=%s http_code=%s active=%s\n' \
+        "$service" "$stage" "$((SECONDS - started))" "$rc" "$code" "$active" >&2
+    bad "access-token $service listener did not become ready after $stage within 30s (#573)"
+    return 1
+}
+
 phase_access_tokens() {
     phase "access-tokens — a non-master ACCESS_TOKENS entry authenticates :8081 and :8082 (#516)"
-    local master bench cur new port resp code cid st waited=0 body bread
+    local master bench cur new port api_port resp code cid st waited=0 body bread
     master=$(head -c 32 /dev/urandom | xxd -p -c 256)
     bench=$(head -c 32 /dev/urandom | xxd -p -c 256)
     cur=$(jq -r '.DONATION // 1' "$CFG")
     new=$(((cur + 1) % 101))
     set_cfg ".api=\"enabled\" | .control=\"enabled\" | .ACCESS_TOKEN=\"$master\" | .ACCESS_TOKENS={\"bench\":\"$bench\"} | .api_allow_from=\"127.0.0.1/32\"" soft ||
         bad "could not enable api+control with a named ACCESS_TOKENS entry"
-    sleep 3 # let the sister API + control services (restarted by apply) settle
     port=$(jq -r '.control_port // 8082' "$CFG")
+    api_port=$(jq -r '.api_port // 8081' "$CFG")
+    _access_tokens_ready api "$api_port" setup || return 1
+    _access_tokens_ready control "$port" setup || return 1
 
     # :8080 stays master-only — a named entry must not reach xmrig's own API. The refusal code is a
     # present-but-wrong Bearer, not a missing one, and varies by xmrig version (same reasoning as
@@ -47,7 +77,8 @@ phase_access_tokens() {
 
     # Revocation: drop the entry and confirm it stops on both ports (red before / green after #516).
     set_cfg '.ACCESS_TOKENS={}' soft || bad "could not revoke the bench ACCESS_TOKENS entry"
-    sleep 3
+    _access_tokens_ready api "$api_port" revocation || return 1
+    _access_tokens_ready control "$port" revocation || return 1
     code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 -H "Authorization: Bearer $bench" "http://127.0.0.1:$(jq -r '.api_port // 8081' "$CFG")/health" 2>/dev/null || true)
     [ "$code" = 401 ] && ok "a revoked ACCESS_TOKENS entry stops authenticating :8081 (#516)" || bad "revoked entry still answered $code on :8081"
     code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 -X POST -H "Authorization: Bearer $bench" -H 'Content-Type: application/json' -d '{"DONATION":1}' "http://127.0.0.1:$port/apply" 2>/dev/null || true)

tests/test-access-token-readiness.sh

@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+# Hardware-free execution of the actual readiness helper, phase and EXIT restoration.
+# shellcheck disable=SC2034,SC2317,SC2329
+set -Eeuo pipefail
+ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+TASK_TMP=$(mktemp -d)
+trap 'rm -rf "$TASK_TMP"' EXIT
+source "$ROOT/tests/e2e-pithead-tokens.sh"
+for fn in _restore_xmrig _cleanup snapshot_config; do
+    eval "$(sed -n "/^$fn()/,/^}/p" "$ROOT/tests/e2e-pithead.sh")"
+done
+check() { [ "$1" = "$2" ] || {
+    printf 'FAIL: %s (got %s, expected %s)\n' "$3" "$1" "$2" >&2
+    exit 1
+}; }
+# Every child advances a virtual clock; no network, systemd, or real sleep is used.
+sleep() { SECONDS=$((SECONDS + $1)); }
+phase() { :; }
+ok() { printf 'PASS: %s\n' "$1"; }
+bad() {
+    printf 'FAIL: %s\n' "$1" >&2
+    return 1
+}
+refresh_profile_finish() { :; }
+rigforge_stub() { :; }
+systemctl() { return 0; }
+set_cfg() {
+    local next
+    next=$(jq "$1" "$CFG")
+    printf '%s\n' "$next" >"$CFG"
+}
+curl() {
+    local url="${*: -1}" output='' bearer=0 body=0 n=0 arg
+    printf '%s\n' "$*" >>"$CASE_DIR/argv"
+    while [ "$#" -gt 0 ]; do
+        arg=$1
+        shift
+        case "$arg" in
+        -H)
+            bearer=1
+            shift
+            ;;
+        -o)
+            output=$1
+            shift
+            ;;
+        -fsS) body=1 ;;
+        esac
+    done
+    if [ "$bearer" = 0 ]; then
+        # Refuse POST/credentials/config inputs in every readiness probe.
+        case "$url" in */health | */status) ;; *) return 99 ;; esac
+        if [[ "$url" == *:8082/status ]] && [ "$(jq '.ACCESS_TOKENS | length' "$CFG")" != 0 ]; then
+            n=$(cat "$CASE_DIR/setup-probes")
+            printf '%s' "$((n + 1))" >"$CASE_DIR/setup-probes"
+            if [ "$SCENARIO" = delayed ] && [ "$n" -lt 2 ]; then
+                printf 000
+                return 7
+            fi
+            if [ "$SCENARIO" = setup-down ]; then
+                printf 000
+                return 7
+            fi
+        fi
+        if [[ "$url" == *:8082/status ]] && [ "$(jq '.ACCESS_TOKENS | length' "$CFG")" = 0 ]; then
+            n=$(cat "$CASE_DIR/probes")
+            printf '%s' "$((n + 1))" >"$CASE_DIR/probes"
+            case "$SCENARIO" in
+            delayed) [ "$n" -ge 4 ] || {
+                printf 000
+                return 7
+            } ;;
+            down)
+                printf 'synthetic-private-error\n' >&2
+                printf 000
+                return 7
+                ;;
+            timeout)
+                printf 000
+                return 28
+                ;;
+            open)
+                printf 200
+                return 0
+                ;;
+            denied)
+                printf 403
+                return 0
+                ;;
+            error)
+                printf 000
+                return 60
+                ;;
+            esac
+        fi
+        printf 401
+    elif [[ "$url" == *:8080/* ]]; then
+        printf 403
+    elif [ "$body" = 1 ]; then
+        [[ "$url" != */status\?* ]] || {
+            printf '{"status":"applied"}'
+            return
+        }
+        printf '{}'
+    elif [[ "$url" == */apply ]]; then
+        if [ "$(jq '.ACCESS_TOKENS | length' "$CFG")" = 0 ]; then
+            printf 'post\n' >>"$CASE_DIR/revoked-posts"
+            [ "$SCENARIO" != accepted ] || {
+                printf 202
+                return
+            }
+            printf 401
+        else
+            if [ "$SCENARIO" = delayed ] && [ "$(cat "$CASE_DIR/setup-probes")" -lt 3 ]; then
+                printf 000
+                return 7
+            fi
+            printf '{"change_id":"0123456789abcdef"}' >"$output"
+            printf 202
+        fi
+    else
+        printf 401
+    fi
+}
+for SCENARIO in delayed down timeout open denied error accepted setup-down; do
+    CASE_DIR="$TASK_TMP/$SCENARIO"
+    mkdir -p "$CASE_DIR"
+    CFG="$CASE_DIR/config"
+    printf '%s\n' '{"DONATION":1,"ACCESS_TOKEN":"original","ACCESS_TOKENS":{"original":"original"}}' >"$CFG"
+    cp "$CFG" "$CASE_DIR/original"
+    printf 0 >"$CASE_DIR/probes"
+    printf 0 >"$CASE_DIR/setup-probes"
+    : >"$CASE_DIR/revoked-posts"
+    : >"$CASE_DIR/argv"
+    rc=0
+    (
+        unset SECONDS # Unsetting removes Bash's wall-clock behavior before the fake clock.
+        SECONDS=0
+        SAVED_CFG='' SAVED_XMRIG_ACTIVE=0 HAMMER_PIDS='' E2E_EXIT_RC=0
+        RIG_LOCK_HOLDER="$CASE_DIR/holder" RIGFORGE=rigforge_stub
+        snapshot_config
+        phase_access_tokens
+    ) >"$CASE_DIR/log" 2>&1 || rc=$?
+    cmp -s "$CFG" "$CASE_DIR/original" || {
+        echo 'FAIL: EXIT did not restore original bytes' >&2
+        exit 1
+    }
+    posts=$(wc -l <"$CASE_DIR/revoked-posts" | tr -d ' ')
+    probes=$(cat "$CASE_DIR/probes")
+    case "$SCENARIO" in
+    delayed)
+        check "$rc:$probes:$posts" 0:5:1 'transient refusal recovers before one revocation POST'
+        check "$(cat "$CASE_DIR/setup-probes")" 3 'setup waits before authenticated POST'
+        grep -Eq 'service=control stage=setup elapsed_s=2' "$CASE_DIR/log"
+        grep -Eq 'service=control stage=revocation elapsed_s=4' "$CASE_DIR/log"
+        grep -Eq 'a revoked ACCESS_TOKENS entry stops authenticating :8082' "$CASE_DIR/log"
+        ;;
+    down | timeout)
+        check "$rc:$probes:$posts" 1:30:0 'unreachable active listener fails boundedly before POST'
+        grep -Eq 'elapsed_s=30 transport_rc=(7|28) http_code=000 active=1' "$CASE_DIR/log"
+        ! grep -Eq synthetic-private-error "$CASE_DIR/log"
+        ;;
+    open | denied | error)
+        check "$rc:$probes:$posts" 1:1:0 'unexpected HTTP/TLS error fails without retry or POST'
+        ;;
+    setup-down)
+        check "$rc:$probes:$posts" 1:0:0 'setup failure stops before auth assertions'
+        check "$(cat "$CASE_DIR/setup-probes")" 30 'setup deadline is bounded'
+        grep -Eq 'service=control stage=setup elapsed_s=30 transport_rc=7 http_code=000 active=1' "$CASE_DIR/log"
+        ;;
+    accepted)
+        check "$rc:$probes:$posts" 1:1:1 'readiness never substitutes for revoked-bearer rejection'
+        grep -Eq 'revoked entry still answered 202 on :8082' "$CASE_DIR/log"
+        ;;
+    esac
+    # Plain probes use curl's isolated config and proxy settings, with no bearer.
+    grep -E 'http://127.0.0.1:8082/status$' "$CASE_DIR/argv" | while IFS= read -r probe; do
+        [[ "$probe" == '-q --noproxy * -s -o /dev/null -w %{http_code} --max-time '* ]] || exit 1
+        [[ "$probe" != *Authorization* && "$probe" != *POST* ]] || exit 1
+    done
+    printf 'PASS: %s (including byte-identical EXIT restoration)\n' "$SCENARIO"
+done
+# Force a clock tick between the loop condition and timeout calculation. No curl
+# may run with max-time 0, which would disable its deadline entirely.
+rc=0
+(
+    unset SECONDS
+    SECONDS=0
+    set -T
+    trap 'if [[ "$BASH_COMMAND" == "remaining="* ]]; then SECONDS=30; fi' DEBUG
+    _access_tokens_ready control 8082 revocation
+) >"$TASK_TMP/deadline.log" 2>&1 || rc=$?
+check "$rc" 1 'deadline tick refuses an unlimited curl timeout'
+grep -Eq 'elapsed_s=30 transport_rc=0 http_code=000 active=1' "$TASK_TMP/deadline.log"
+printf 'PASS: deadline tick before curl\n'