From 408f6a25f9e939b0c9fa9f9eb8dc35a8ef1c3a10 Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:52:43 -0500 Subject: [PATCH 01/10] perf(api): read per-core clocks without spawning processes (#540) --- docs/operations.md | 5 +++-- rigforge.sh | 2 +- tests/test-api-refresh-liveness.sh | 6 ++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 0dcfd30..c3f4091 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -564,8 +564,9 @@ last-change provenance (`config_meta`; #254), and the last control-path outcome `{change_id, status, reason}` or `null`; #346) — as JSON (`/health`, or nested under `rigforge` in `/1/summary` and `/2/summary`). It follows XMRig's own architecture: one tiny persistent server (python3 stdlib, ~10 MB idle) ships pre-computed bytes, so a request costs -microseconds and cannot touch mining performance; a systemd timer recomputes the state every 15 -seconds at idle priority, off the request path. The timer uses an independent wall-clock cadence, +microseconds and cannot touch mining performance; a systemd timer schedules a refresh every 15 +seconds at idle priority, off the request path. The clock probe reads per-core values without +starting a process for each core. The timer uses an independent wall-clock cadence, so a failed refresh or a unit reinstall cannot strand the feed without a future run. Summary and health responses carry `generated_at` in UTC; `doctor` reports its age and the timer's next/last trigger, and warns once the payload is over a minute old. The same diff --git a/rigforge.sh b/rigforge.sh index 9992ae6..26056fe 100755 --- a/rigforge.sh +++ b/rigforge.sh @@ -5007,7 +5007,7 @@ _cpu_eff_khz() { local f v sum=0 n=0 for f in "$CPU_SYSFS"/cpu[0-9]*/cpufreq/scaling_cur_freq; do [ -r "$f" ] || continue - v=$(cat "$f" 2>/dev/null) + IFS= read -r v <"$f" || true case "$v" in '' | *[!0-9]*) continue ;; esac sum=$((sum + v)) n=$((n + 1)) diff --git a/tests/test-api-refresh-liveness.sh b/tests/test-api-refresh-liveness.sh index 7968283..588eafd 100644 --- a/tests/test-api-refresh-liveness.sh +++ b/tests/test-api-refresh-liveness.sh @@ -1,6 +1,12 @@ # shellcheck shell=bash disable=SC1090,SC2034,SC2329 # #454: doctor's diagnosis stays deterministic here. Real transient-systemd timing was checked # separately because this dependency-free suite must also run on macOS. +assert_eq "API clock probe reads every core without a process per core (#540)" "$( + source "$SCRIPT" + CPU_SYSFS="$DOC/cpu_throttle" + cat() { return 99; } + _cpu_eff_khz +)" "3000000" RFS="$(mktemp -d "$SANDBOX/refresh-status.XXXXXX")" printf '%s' '{"generated_at":"2026-09-07T04:00:00Z"}' >"$RFS/summary.json" refresh_status() { # [refresh state] From 5e22933638a50c7f369a87a97c6c70c74d75f865 Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:57:38 -0500 Subject: [PATCH 02/10] test(e2e): require fresh feed after thermal control apply (#540) --- tests/e2e-pithead-control.sh | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/e2e-pithead-control.sh b/tests/e2e-pithead-control.sh index f941bb6..913c250 100644 --- a/tests/e2e-pithead-control.sh +++ b/tests/e2e-pithead-control.sh @@ -5,13 +5,18 @@ # file declares no preflight of its own and must not be run standalone. phase_control() { phase "control — writable control path DONATION apply round trip (#344/#509)" - local tok cur new port resp code cid st waited=0 body + local tok cur new port api_port resp code cid st waited=0 body max_temp target_temp tok=$(head -c 32 /dev/urandom | xxd -p -c 256) cur=$(jq -r '.DONATION // 1' "$CFG") new=$(((cur + 1) % 101)) - set_cfg ".api=\"enabled\" | .control=\"enabled\" | .ACCESS_TOKEN=\"$tok\" | .api_allow_from=\"127.0.0.1/32\"" soft || bad "could not enable the control path" + max_temp=$(jq -r 'if .max_temp_c == null or .max_temp_c == "" then 100 else .max_temp_c end' "$CFG") + [ "$max_temp" -le 100 ] || max_temp=100 + [ "$max_temp" -gt 40 ] || bad "cannot lower the existing thermal cutoff safely" + target_temp=$((max_temp - 1)) + set_cfg ".api=\"enabled\" | .control=\"enabled\" | .ACCESS_TOKEN=\"$tok\" | .api_allow_from=\"127.0.0.1/32\" | .watchdog=\"enabled\" | .max_temp_c=$max_temp" soft || bad "could not enable the control path" sleep 3 # let rigforge-control.service (restarted by install_control) settle port=$(jq -r '.control_port // 8082' "$CFG") + api_port=$(jq -r '.api_port // 8081' "$CFG") resp="$(mktemp)" code=$(curl -s -o "$resp" -w '%{http_code}' --max-time 10 -H "Authorization: Bearer $tok" \ -H "Content-Type: application/json" -d "{\"DONATION\": $new}" "http://127.0.0.1:$port/apply" 2>/dev/null || true) @@ -31,6 +36,27 @@ phase_control() { [ "$(jq -r '.DONATION' "$CFG")" = "$new" ] && ok "config.json carries DONATION=$new (control-apply persisted it)" || bad "config.json DONATION is '$(jq -r '.DONATION' "$CFG")', expected $new" + local before feed_temp feed_stamp started + before=$(date -u +%Y-%m-%dT%H:%M:%SZ) + code=$(curl -s -o "$resp" -w '%{http_code}' --max-time 10 -H "Authorization: Bearer $tok" \ + -H "Content-Type: application/json" -d "{\"max_temp_c\":$target_temp}" "http://127.0.0.1:$port/apply" 2>/dev/null || true) + cid=$(jq -r '.change_id // empty' "$resp" 2>/dev/null || true) + rm -f "$resp" + [ "$code" = 202 ] && [ -n "$cid" ] || bad "max_temp_c apply was not accepted (HTTP $code)" + started=$SECONDS + while [ "$((SECONDS - started))" -lt 90 ]; do + body=$(curl -fsS --max-time 5 -H "Authorization: Bearer $tok" "http://127.0.0.1:$port/status?change_id=$cid" 2>/dev/null || true) + st=$(printf '%s' "$body" | jq -r '.status // empty' 2>/dev/null || true) + body=$(curl -fsS --max-time 5 -H "Authorization: Bearer $tok" "http://127.0.0.1:$api_port/1/summary" 2>/dev/null || true) + feed_temp=$(printf '%s' "$body" | jq -r '.rigforge.watchdog.max_temp_c // empty' 2>/dev/null || true) + feed_stamp=$(printf '%s' "$body" | jq -r '.generated_at // empty' 2>/dev/null || true) + [ "$st" = applied ] && [ "$feed_temp" = "$target_temp" ] && [[ "$feed_stamp" > "$before" ]] && break + sleep 5 + done + waited=$((SECONDS - started)) + [ "$st" = applied ] && [ "$feed_temp" = "$target_temp" ] && [[ "$feed_stamp" > "$before" ]] && + ok "applied max_temp_c reaches a new direct sister-API summary within ${waited}s (#540)" || + bad "max_temp_c feed stale after 90s (status=${st:-unknown}, temp=${feed_temp:-missing}, generated_at=${feed_stamp:-missing})" # No token blanking here: _cleanup's EXIT trap restores DONATION, the token and the control # state from the snapshot, and blanking ACCESS_TOKEN while control is enabled is #514's own # fail-closed abort (rigforge.sh refuses that apply) — the bug this phase must not re-enact. From 84ee1a904d346cbf9691790ca18d33dc6cd5807b Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:08:29 -0500 Subject: [PATCH 03/10] test: profile refresh probes under mining priority for #540 --- docs/operations.md | 4 ++- tests/e2e-api-refresh-profile.sh | 43 ++++++++++++++++++++++++++++++++ tests/e2e-pithead-control.sh | 6 +++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/e2e-api-refresh-profile.sh diff --git a/docs/operations.md b/docs/operations.md index c3f4091..6971a30 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -566,7 +566,9 @@ last-change provenance (`config_meta`; #254), and the last control-path outcome persistent server (python3 stdlib, ~10 MB idle) ships pre-computed bytes, so a request costs microseconds and cannot touch mining performance; a systemd timer schedules a refresh every 15 seconds at idle priority, off the request path. The clock probe reads per-core values without -starting a process for each core. The timer uses an independent wall-clock cadence, +starting a process for each core. The hardware control test checks direct feed propagation and +compares complete refresh and clock-probe timings for the old and builtin reads at idle priority, +publishing only into scratch files. The timer uses an independent wall-clock cadence, so a failed refresh or a unit reinstall cannot strand the feed without a future run. Summary and health responses carry `generated_at` in UTC; `doctor` reports its age and the timer's next/last trigger, and warns once the payload is over a minute old. The same diff --git a/tests/e2e-api-refresh-profile.sh b/tests/e2e-api-refresh-profile.sh new file mode 100644 index 0000000..02f029e --- /dev/null +++ b/tests/e2e-api-refresh-profile.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Read-only comparison for #540; invoked by the reserved control phase, never standalone. +set -euo pipefail +# shellcheck disable=SC1090 +source "$1" +TMPDIR=${TMPDIR:-/run} +RIGFORGE_API_DATA=$(mktemp -d "$TMPDIR/rigforge-refresh-profile.XXXXXX") +export RIGFORGE_API_DATA +trap 'rm -rf "$RIGFORGE_API_DATA"' EXIT +eval "$(declare -f _cpu_eff_khz | sed '1s/_cpu_eff_khz/_cpu_eff_khz_builtin/')" +_cpu_eff_khz_legacy() { + local f v sum=0 n=0 + for f in "$CPU_SYSFS"/cpu[0-9]*/cpufreq/scaling_cur_freq; do + [ -r "$f" ] || continue + v=$(cat "$f" 2>/dev/null) + case "$v" in '' | *[!0-9]*) continue ;; esac + sum=$((sum + v)) + n=$((n + 1)) + done + [ "$n" -gt 0 ] && echo $((sum / n)) || echo "" +} +# Fixed helper names only; emit durations, never arguments or probe values. +for probe in parse_config _read_api_summary _api_tune_json _api_power_json _health_json _mem_summary _msr_log_status _dmi _watchdog_json _api_config_json _api_config_meta_json _api_control_json _api_control_history_json; do + eval "$(declare -f "$probe" | sed "1s/$probe/${probe}_profile_original/")" + eval "$probe() { local started=\$SECONDS rc; ${probe}_profile_original \"\$@\"; rc=\$?; printf 'refresh-profile: %s $probe %ss\\n' \"\$mode\" \"\$((SECONDS - started))\" >&2; return \"\$rc\"; }" +done +for mode in legacy builtin builtin legacy; do + systemctl is-active --quiet "$SERVICE_NAME" + systemctl show rigforge-api-refresh.timer rigforge-api-refresh.service -p ActiveState -p SubState -p LastTriggerUSec + _cpu_eff_khz() { + local started=$SECONDS rc + "_cpu_eff_khz_$mode" + rc=$? + printf 'refresh-profile: %s clock probe %ss\n' "$mode" "$((SECONDS - started))" >&2 + return "$rc" + } + started=$SECONDS + api_refresh + jq -e '.generated_at and .rigforge.watchdog and .rigforge.health.service_active' "$RIGFORGE_API_DATA/summary.json" >/dev/null + printf 'refresh-profile: %s complete refresh %ss\n' "$mode" "$((SECONDS - started))" + systemctl is-active --quiet "$SERVICE_NAME" + systemctl show rigforge-api-refresh.timer rigforge-api-refresh.service -p ActiveState -p SubState -p LastTriggerUSec +done diff --git a/tests/e2e-pithead-control.sh b/tests/e2e-pithead-control.sh index 913c250..49412f6 100644 --- a/tests/e2e-pithead-control.sh +++ b/tests/e2e-pithead-control.sh @@ -57,6 +57,12 @@ phase_control() { [ "$st" = applied ] && [ "$feed_temp" = "$target_temp" ] && [[ "$feed_stamp" > "$before" ]] && ok "applied max_temp_c reaches a new direct sister-API summary within ${waited}s (#540)" || bad "max_temp_c feed stale after 90s (status=${st:-unknown}, temp=${feed_temp:-missing}, generated_at=${feed_stamp:-missing})" + # Bounded A/B diagnostics at the refresh service's priority, while the miner runs. Never + # replace the live feed: both implementations publish only into the scratch directory. + systemctl is-active --quiet xmrig || bad "refresh profiling requires the miner running" + systemctl show rigforge-api-refresh.timer -p ActiveState -p LastTriggerUSec + timeout 300 nice -n 19 ionice -c 3 bash "$(dirname "${BASH_SOURCE[0]}")/e2e-api-refresh-profile.sh" "$RIGFORGE" || + bad "refresh A/B diagnostics did not complete" # No token blanking here: _cleanup's EXIT trap restores DONATION, the token and the control # state from the snapshot, and blanking ACCESS_TOKEN while control is enabled is #514's own # fail-closed abort (rigforge.sh refuses that apply) — the bug this phase must not re-enact. From 657eaa9b384a3c0f73731497643a1496df523b85 Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:20:58 -0500 Subject: [PATCH 04/10] test: replay retained thermal and pools publication workloads --- docs/operations.md | 6 +- tests/README.md | 25 ++++- tests/e2e-control-replay.sh | 122 ++++++++++++++++++++ tests/e2e-pithead-control.sh | 2 +- tests/e2e-pithead.sh | 16 +-- tests/e2e-refresh-window.sh | 5 +- tests/test-api-refresh-liveness.sh | 3 + tests/test-control-replay.sh | 161 +++++++++++++++++++++++++++ tests/test-refresh-window-control.sh | 2 +- 9 files changed, 323 insertions(+), 19 deletions(-) create mode 100644 tests/e2e-control-replay.sh create mode 100644 tests/test-control-replay.sh diff --git a/docs/operations.md b/docs/operations.md index 5730da0..533d36c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -566,9 +566,9 @@ last-change provenance (`config_meta`; #254), and the last control-path outcome persistent server (python3 stdlib, ~10 MB idle) ships pre-computed bytes, so a request costs microseconds and cannot touch mining performance; a systemd timer schedules a refresh every 15 seconds at idle priority, off the request path. The clock probe reads per-core values without -starting a process for each core. The hardware control test checks direct feed propagation and -captures timer-driven helper timings around thermal apply while mining, preserving the service -policy and normal feed publication. The timer uses an independent wall-clock cadence, +starting a process for each core. The queued thermal and separate pools replay tests check direct-feed +freshness and exact-ID publication, capturing timer-driven helper timings under the runner-owned +mining fixture. They preserve the service policy and normal feed publication. The timer uses an independent wall-clock cadence, so a failed refresh or a unit reinstall cannot strand the feed without a future run. Summary and health responses carry `generated_at` in UTC; `doctor` reports its age and the timer's next/last trigger, and warns once the payload is over a minute old. The same diff --git a/tests/README.md b/tests/README.md index 086cb7a..efd5f49 100644 --- a/tests/README.md +++ b/tests/README.md @@ -42,11 +42,26 @@ the owned drop-in, reloads systemd and restores the timer’s prior active state The job fails if journal collection or measurement export fails, no profiled refresh completes successfully, or cleanup fails. Begin events from unfinished helpers are retained to locate stalled operations, including a pass stopped by cleanup. -For affected-workload acceptance, use a reserved `rigforge` `tier4-e2e` job with -`phases=["connect","control"]`: connect proves mining before control on the same reserved worker. -A `phases=["control"]` run validates diagnostic collection only; -do not invoke the profiler against a rig yourself. Routing still chooses an eligible worker: -this captures the same thermal-control/service path, but does not guarantee the original fixture. +The ordinary `phases=["connect","control"]` job proves mining and profiles a bounded thermal +apply on an eligible worker; it does not recreate the historical sequence. For #540's separate +thermal and pools replay cases, submit one declaring phase per `tier4-e2e` job with +`fixture=pithead-control`: `control-replay-thermal` or `control-replay-pools`. The deployed runner +privately supplies the operator pools probe, context and named reservation. The gate verifies that +reservation, bypasses only its own competing flock, and leaves the runner's holder marker and release +alone. Without the fixture, a replay is refused before config changes; ordinary locking is unchanged. +Both cases establish real jobs and an accepted share, retain thread/tuning and payout identity, set +recorded scalars (thermal 100, DONATION 0, watchdog interval 5), and replay thermal 101 → 100, +DONATION 1 → 0, watchdog interval 6 → 5, then the private pools probe. The thermal case verifies a 100 baseline, then applies direct 102 and requires a newer feed +carrying that cutoff and this exact terminal change ID within 90 seconds. The separate pools +case requires this exact change ID to reach `applied` in the enriched feed within the same bound, +not a later change's outcome. Both continue five-minute observations for diagnostics and preserve the +original pass/fail deadline even if a read crosses it. Full applies can stop/restart the miner; +observations record its active state rather than aborting during that expected transition. +Requests use secured scratch payloads, never credential-bearing arguments or logs. The current probe +and borrowed worker form an equivalent fixture: historical starting config/checkout, original pool +payload, threads/affinity/tuning and precise inter-apply timing remain unknown in the runner context. +No guessed delays or hardware differences are substituted. Do not invoke any replay or profiler +against a worker yourself; the runner owns reservation, restoration and baseline verification. Short passes or an injected delay do not attribute the historical stale-feed failure; #540's fix acceptance requires evidence of the same failure mechanism. Profiling adds clock-read overhead and the source remains the submitted RigForge head; no alternative production implementation is diff --git a/tests/e2e-control-replay.sh b/tests/e2e-control-replay.sh new file mode 100644 index 0000000..6dd5dec --- /dev/null +++ b/tests/e2e-control-replay.sh @@ -0,0 +1,122 @@ +# shellcheck shell=bash +# #540: only the runner's private pithead-control fixture authorizes these replay cases. +REPLAY_RUNNER_LOCK=0 +replay_lock() { + case "${E2E_PITHEAD_CONTROL_FIXTURE:-}:$1" in + :control-replay-*) die "replay requires the queued pithead-control fixture" ;; + :*) rig_lock rigforge e2e-pithead ;; + pithead-control:control-replay-thermal | pithead-control:control-replay-pools) + [[ "${E2E_RIG_LOCK_UNIT:-}" =~ ^bench-ci-riglock-[a-zA-Z0-9_-]+-[0-9]+$ ]] && + systemctl is-active --quiet "$E2E_RIG_LOCK_UNIT" || die "runner reservation is unavailable" + printf '%s' "${E2E_PITHEAD_CONTROL_CONTEXT:-}" | jq -e '.version == 1 and .fixture == "pithead-control" and .thermal.feed_deadline_s == 90 and .thermal.starting_scalars == {max_temp_c:100,DONATION:0,watchdog_interval_min:5} and (.unknown | type == "array" and length > 0)' >/dev/null || die "invalid replay context" + printf '%s' "${IT_RIG_POOLS_PROBE:-}" | jq -e 'type == "array" and length > 0 and all(.[]; type == "object" and all(.url,.user,.pass; type == "string" and length > 0))' >/dev/null || die "missing private pools probe" + REPLAY_RUNNER_LOCK=1 + ;; + *) die "fixture requires one declaring replay phase" ;; + esac +} +phase_control_replay_thermal() { phase_control_replay thermal; } +phase_control_replay_pools() { phase_control_replay pools; } + +_replay_apply() { # JSON on stdin; credentials never become process arguments or log output. + local request response code id curl_rc=0 + request=$(mktemp) || return 1 + response=$(mktemp) || { + rm -f "$request" + return 1 + } + cat >"$request" || { + rm -f "$request" "$response" + return 1 + } + code=$(_control_curl "$tok" -s -o "$response" -w '%{http_code}' --max-time 10 --max-filesize 16384 \ + -H 'Content-Type: application/json' --data-binary "@$request" "http://127.0.0.1:$port/apply" 2>/dev/null) || curl_rc=$? + id=$(jq -ser 'select(length == 1) | .[0].change_id | select(type == "string" and test("^[0-9a-f]{16}$"))' "$response" 2>/dev/null || true) + rm -f "$request" "$response" + [ "$curl_rc" = 0 ] && [ "$code" = 202 ] && [ -n "$id" ] || return 1 + printf '%s\n' "$id" +} +_replay_status() { + _control_curl "$tok" -fsS --max-time 3 --max-filesize 16384 "http://127.0.0.1:$port/status?change_id=$cid" 2>/dev/null | + jq -sr 'select(length == 1) | .[0].status | select(. == "applied" or . == "pending" or . == "accepted" or . == "rolled_back" or . == "failed" or . == "rejected")' || true +} +_replay_settle() { # Preserve the recorded 90-second apply bound, without logging payloads. + local started=$SECONDS st + while [ "$((SECONDS - started))" -lt 90 ]; do + st=$(_replay_status) + refresh_profile_state allow-inactive || return 1 + [ "$((SECONDS - started))" -lt 90 ] || break + case "$st" in + applied) return 0 ;; + rolled_back | failed | rejected) return 1 ;; + esac + sleep 5 + done + return 1 +} +_replay_window() { # -- publication of this exact ID, not a later change. + local kind="$1" started=$SECONDS observed=0 body stamp temp terminal st elapsed + while [ "$((SECONDS - started))" -lt 300 ]; do + refresh_profile_state allow-inactive || return 1 + st=$(_replay_status) + body=$(_control_curl "$tok" -fsS --max-time 3 --max-filesize 65536 "http://127.0.0.1:$api_port/1/summary" 2>/dev/null || true) + # Only bounded scalar fields are exported; arbitrary responses/config remain private. + stamp=$(printf '%s' "$body" | jq -sr 'select(length == 1) | .[0].generated_at | select(type == "string" and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"))' 2>/dev/null || true) + temp=$(printf '%s' "$body" | jq -sr 'select(length == 1) | .[0].rigforge.watchdog.max_temp_c | select(type == "number" and . >= 40 and . <= 110)' 2>/dev/null || true) + terminal=$(printf '%s' "$body" | jq -sr --arg id "$cid" 'select(length == 1) | .[0] | ([.rigforge.control] + (.rigforge.control_history // [])) | any(.[]; .change_id == $id and .status == "applied")' 2>/dev/null || true) + elapsed=$((SECONDS - started)) + printf 'replay-window: case=%s elapsed_s=%s change_id=%s status=%s generated_at=%s max_temp_c=%s published=%s\n' "$kind" "$elapsed" "$cid" "${st:-unknown}" "${stamp:-missing}" "${temp:-missing}" "${terminal:-false}" + if [ "$elapsed" -lt 90 ] && [ "$st" = applied ] && [[ "$stamp" > "$before" ]] && [ "$observed" = 0 ]; then + if [ "$terminal" = true ] && { [ "$kind" = pools ] || [ "$temp" = 102 ]; }; then observed=1; fi + fi + sleep 5 + done + [ "$observed" = 1 ] +} +_replay_setup() { + local stage + stage=$(mktemp) || return 1 + # Token and endpoint enter jq through stdin. Promote the runner-added pool without + # altering source pools, worker identities or credentials; refuse a missing bench pool. + if printf '%s\n%s\n' "$tok" "$PITHEAD_URL" | jq -Rn --slurpfile cfg "$CFG" ' + [inputs] as $i | $cfg[0] | + if any(.pools[]; .url == $i[1]) then + .pools |= (map(select(.url == $i[1])) + map(select(.url != $i[1]))) | + .api="enabled" | .control="enabled" | .ACCESS_TOKEN=$i[0] | + .api_allow_from="127.0.0.1/32" | .watchdog="enabled" | + .max_temp_c=100 | .DONATION=0 | .watchdog_interval_min=5 + else error("runner bench pool missing") end' >"$stage" && mv "$stage" "$CFG"; then + "$RIGFORGE" apply >/dev/null 2>&1 + else + rm -f "$stage" + return 1 + fi +} +phase_control_replay() { + local kind="$1" tok port api_port cid before payload + [ "$REPLAY_RUNNER_LOCK" = 1 ] || bad "replay requires verified runner reservation" + phase "control replay — $kind publication (#540)" + tok=$(head -c 32 /dev/urandom | xxd -p -c 256) + # Replay only the recorded scalars; preserve worker threads, tuning and payout identity. + _replay_setup || bad "replay setup failed" + phase_connect preserve-pools || bad "replay mining precondition failed" + port=$(jq -r '.control_port // 8082' "$CFG") + api_port=$(jq -r '.api_port // 8081' "$CFG") + refresh_profile_start || bad "could not instrument replay" + for payload in '{"max_temp_c":101}' '{"max_temp_c":100}' '{"DONATION":1}' '{"DONATION":0}' '{"watchdog_interval_min":6}' '{"watchdog_interval_min":5}'; do + cid=$(printf '%s' "$payload" | _replay_apply) || bad "replay scalar apply refused" + _replay_settle || bad "replay scalar apply did not settle within 90s" + done + before=$(date -u +%Y-%m-%dT%H:%M:%SZ) + cid=$(printf '%s' "$IT_RIG_POOLS_PROBE" | jq -c '{pools:.}' | _replay_apply) || bad "replay pools apply refused" + if [ "$kind" = thermal ]; then + _replay_settle || bad "replay pools apply did not settle within 90s" + [ "$(jq -r .max_temp_c "$CFG")" = 100 ] || bad "thermal baseline is not 100" + before=$(date -u +%Y-%m-%dT%H:%M:%SZ) + cid=$(printf '{"max_temp_c":102}' | _replay_apply) || bad "replay thermal apply refused" + fi + local rc=0 + _replay_window "$kind" || rc=1 + refresh_profile_finish || bad "replay profiling or restoration failed" + [ "$rc" = 0 ] && ok "replay $kind published within 90s (#540)" || bad "replay $kind feed stale after 90s (#540)" +} diff --git a/tests/e2e-pithead-control.sh b/tests/e2e-pithead-control.sh index 0547d64..19c21c1 100644 --- a/tests/e2e-pithead-control.sh +++ b/tests/e2e-pithead-control.sh @@ -8,7 +8,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/e2e-refresh-window.sh" _control_curl() { # Keep the temporary write token out of process arguments. local token="$1" shift - printf 'header = %s\n' "$(printf 'Authorization: Bearer %s' "$token" | jq -Rs .)" | curl --config - "$@" + printf 'header = %s\n' "$(printf 'Authorization: Bearer %s' "$token" | jq -Rs .)" | curl -q --noproxy '*' --config - "$@" } phase_control() { diff --git a/tests/e2e-pithead.sh b/tests/e2e-pithead.sh index 6752ce6..d492787 100644 --- a/tests/e2e-pithead.sh +++ b/tests/e2e-pithead.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash # Worker ↔ stack contract gate (#114): a provisioned worker against a live Pithead stack. -# PITHEAD_URL=stack-host:3333 sudo bash tests/e2e-pithead.sh all -# PITHEAD_URL (required) the stack's stratum host:port -# E2E_STRATUM_PASS opt-in: the stack's actual enforced stratum password # E2E_DASH_URL opt-in: dashboard /api/state URL (self-signed HTTPS accepted) # E2E_DASH_AUTH user:pass for the dashboard's basic auth # E2E_SHARE_TIMEOUT seconds to wait for an accepted share (default 180) @@ -17,6 +14,8 @@ RIGFORGE="$HERE/rigforge.sh" CFG="$HERE/config.json" source "$(dirname "${BASH_SOURCE[0]}")/e2e-pithead-control.sh" && source "$(dirname "${BASH_SOURCE[0]}")/e2e-pithead-tokens.sh" # phase_control (#509), phase_access_tokens (#516) +source "$(dirname "${BASH_SOURCE[0]}")/e2e-control-replay.sh" +# bench-ci-fixture: pithead-control control-replay-thermal control-replay-pools PASS=0 FAIL=0 E2E_EXIT_RC=0 ok() { PASS=$((PASS + 1)) @@ -120,7 +119,7 @@ _cleanup() { cleanup_ok=0 } [ "$cleanup_ok" != 1 ] || rm -f "$SAVED_CFG" || cleanup_ok=0 - rm -f "${RIG_LOCK_HOLDER:-${RIG_LOCK_FILE:-/var/lock/rig-e2e.lock}.holder}" || true + [ "${REPLAY_RUNNER_LOCK:-0}" = 1 ] || rm -f "${RIG_LOCK_HOLDER:-${RIG_LOCK_FILE:-/var/lock/rig-e2e.lock}.holder}" || true [ "$cleanup_ok" = 1 ] } snapshot_config() { @@ -171,7 +170,7 @@ wait_for_job() { # -> 0 when the log shows a stratum job phase_connect() { phase "connect — worker mines against the live stack ($PITHEAD_URL)" - set_cfg ".pools[0].url = \"$PITHEAD_URL\"" + [ "${1:-}" = preserve-pools ] || set_cfg ".pools[0].url = \"$PITHEAD_URL\"" systemctl is-active --quiet xmrig || "$RIGFORGE" start >/dev/null 2>&1 [ -n "$WLOG" ] || WLOG="$(find "$HERE" -path '*worker*' -name xmrig.log 2>/dev/null | head -1)" if [ -z "$WLOG" ]; then @@ -535,8 +534,8 @@ summary() { require_preflight "$@" # Validate the phase against its own `phase_` function: a typo dies with the rig untouched. # #183: the lock follows, before snapshot_config (whose _cleanup runs `apply`) and the first API touch. -[ "${1:-all}" = all ] || declare -F "phase_${1//-/_}" >/dev/null || die "unknown phase '$1' (connect|worker-api|api-impact|network|stratum-auth|dashboard|dev-fee|control|access-tokens|all)" -rig_lock rigforge e2e-pithead +[ "${1:-all}" = all ] || declare -F "phase_${1//-/_}" >/dev/null || die "unknown phase '$1' (connect|worker-api|api-impact|network|stratum-auth|dashboard|dev-fee|control|control-replay-thermal|control-replay-pools|access-tokens|all)" +replay_lock "${1:-all}" snapshot_config case "${1:-all}" in @@ -548,6 +547,7 @@ stratum-auth) phase_stratum_auth ;; dashboard) phase_dashboard ;; dev-fee) phase_dev_fee ;; control) phase_control ;; +control-replay-thermal | control-replay-pools) phase_control_replay "${1##*-}" ;; access-tokens) phase_access_tokens ;; all) for run_phase in phase_connect phase_worker_api phase_api_impact phase_network phase_stratum_auth phase_dashboard phase_dev_fee phase_control phase_access_tokens; do @@ -555,7 +555,7 @@ all) [ "$FAIL" -eq 0 ] || break done ;; -*) die "unknown phase '$1' (connect|worker-api|api-impact|network|stratum-auth|dashboard|dev-fee|control|access-tokens|all)" ;; +*) die "unknown phase '$1' (connect|worker-api|api-impact|network|stratum-auth|dashboard|dev-fee|control|control-replay-thermal|control-replay-pools|access-tokens|all)" ;; esac _cleanup || bad "pre-test config/runtime restoration failed; snapshot retained at $SAVED_CFG" trap - EXIT diff --git a/tests/e2e-refresh-window.sh b/tests/e2e-refresh-window.sh index aad6204..41678bc 100644 --- a/tests/e2e-refresh-window.sh +++ b/tests/e2e-refresh-window.sh @@ -25,7 +25,10 @@ refresh_profile_start() { } refresh_profile_state() { printf 'refresh-window: state at_ns=%s\n' "$(date +%s%N)" - systemctl is-active --quiet xmrig || return 1 + local active=0 + systemctl is-active --quiet xmrig && active=1 + printf 'refresh-window: xmrig_active=%s\n' "$active" + [ "$active" = 1 ] || [ "${1:-}" = allow-inactive ] || return 1 systemctl show rigforge-api-refresh.timer rigforge-api-refresh.service \ -p ActiveState -p SubState -p Result -p LastTriggerUSec -p NextElapseUSecRealtime \ -p ExecMainStartTimestamp -p ExecMainExitTimestamp -p ExecMainStatus diff --git a/tests/test-api-refresh-liveness.sh b/tests/test-api-refresh-liveness.sh index 3e70d5f..d73b985 100644 --- a/tests/test-api-refresh-liveness.sh +++ b/tests/test-api-refresh-liveness.sh @@ -263,3 +263,6 @@ bash "$ROOT/tests/test-refresh-profile.sh" assert_rc "timer-refresh diagnostics regression suite (#546)" "$?" "0" bash "$ROOT/tests/test-refresh-window-control.sh" assert_rc "thermal refresh-window regression suite (#546)" "$?" "0" + +bash "$ROOT/tests/test-control-replay.sh" +assert_rc "queued control replay regression suite (#540)" "$?" "0" diff --git a/tests/test-control-replay.sh b/tests/test-control-replay.sh new file mode 100644 index 0000000..bb89b02 --- /dev/null +++ b/tests/test-control-replay.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# #540: queued replay input, sequence, transport and publication checks, without hardware. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [ "$#" = 0 ]; then + REPLAY_TEST_DIR=$(mktemp -d) + export REPLAY_TEST_DIR + trap 'rm -rf "$REPLAY_TEST_DIR"' EXIT + for kind in thermal pools; do + for success in fresh history; do + bash "$0" "$kind" "$success" >"$REPLAY_TEST_DIR/$kind-$success.log" 2>&1 + grep -q "replay $kind published within 90s" "$REPLAY_TEST_DIR/$kind-$success.log" + grep -q 'finish-profile' "$REPLAY_TEST_DIR/$kind-$success.log" + done + for mode in stale late wrong-id; do + if bash "$0" "$kind" "$mode" >"$REPLAY_TEST_DIR/$kind-$mode.log" 2>&1; then + echo "replay $kind $mode unexpectedly passed" >&2 + exit 1 + fi + grep -q "feed stale after 90s" "$REPLAY_TEST_DIR/$kind-$mode.log" + done + done + for mode in missing inactive context probe other bad-id mining curl-failed; do + if bash "$0" thermal "$mode" >"$REPLAY_TEST_DIR/$mode.log" 2>&1; then + echo "replay $mode unexpectedly passed" >&2 + exit 1 + fi + done + ( + source "$ROOT/tests/e2e-refresh-window.sh" + systemctl() { [ "$1" = show ]; } + if refresh_profile_state >"$REPLAY_TEST_DIR/inactive-state"; then exit 1; fi + refresh_profile_state allow-inactive >>"$REPLAY_TEST_DIR/inactive-state" + grep -q 'xmrig_active=0' "$REPLAY_TEST_DIR/inactive-state" + ) + printf 'control replay regressions: PASS\n' + exit 0 +fi +kind="$1" mode="$2" +source "$ROOT/tests/e2e-pithead-control.sh" +source "$ROOT/tests/e2e-control-replay.sh" +CFG="$REPLAY_TEST_DIR/$kind-$mode.json" +CALLS="$REPLAY_TEST_DIR/$kind-$mode.calls" +PITHEAD_URL='fixture.invalid:3333' +RIGFORGE=/usr/bin/true +printf '{"control_port":8082,"api_port":8081,"pools":[{"url":"original.invalid:3333","user":"synthetic-user","pass":"SOURCE-SECRET"},{"url":"fixture.invalid:3333","user":"synthetic-user","pass":"BENCH-SECRET"}]}' >"$CFG" +E2E_PITHEAD_CONTROL_FIXTURE=pithead-control +E2E_RIG_LOCK_UNIT=bench-ci-riglock-synthetic-123 +E2E_PITHEAD_CONTROL_CONTEXT='{"version":1,"fixture":"pithead-control","thermal":{"feed_deadline_s":90,"starting_scalars":{"max_temp_c":100,"DONATION":0,"watchdog_interval_min":5}},"unknown":["historical timing"]}' +IT_RIG_POOLS_PROBE='[{"url":"fixture.invalid:3333","user":"synthetic-user","pass":"SECRET $(never-execute) `never-execute`"}]' +case "$mode" in +missing) E2E_PITHEAD_CONTROL_FIXTURE='' ;; +context) E2E_PITHEAD_CONTROL_CONTEXT='{}' ;; +probe) IT_RIG_POOLS_PROBE='[]' ;; +esac +die() { + echo "$1" >&2 + exit 2 +} +bad() { + echo "$1" >&2 + exit 1 +} +ok() { echo "$1"; } +phase() { :; } +rig_lock() { die 'fixture tried to compete for the rig lock'; } +systemctl() { [ "$mode" != inactive ]; } +jq() { + [[ "$*" != *synthetic-token* && "$*" != *SECRET* ]] || die 'credential-bearing jq argument' + command jq "$@" +} +export http_proxy='http://127.0.0.1:9' ALL_PROXY='http://127.0.0.1:9' NO_PROXY='' +phase_connect() { + [ "$mode" != mining ] && [ "$1" = preserve-pools ] && + [ "$(jq -r '.pools[0].url' "$CFG")" = "$PITHEAD_URL" ] && + [ "$(jq -r '.pools[1].url' "$CFG")" = original.invalid:3333 ] && + [ "$(jq -r '.pools[1].pass' "$CFG")" = SOURCE-SECRET ] +} +refresh_profile_start() { :; } +refresh_profile_state() { :; } +refresh_profile_finish() { echo finish-profile; } +head() { printf 'synthetic-token'; } +xxd() { cat; } +sleep() { SECONDS=$((SECONDS + $1)); } +date() { printf '2026-09-30T00:00:00Z\n'; } +curl() { + [ "$1 $2 $3 $4 $5" = '-q --noproxy * --config -' ] || return 1 + [[ "$*" != *SECRET* && "$*" != *synthetic-token* ]] || return 1 + grep -q '^header = "Authorization: Bearer synthetic-token"$' || return 1 + local output='' data='' url='' target=100 id stamp='2026-09-30T00:00:00Z' published_id + while [ "$#" -gt 0 ]; do + case "$1" in + -o) + output="$2" + shift + ;; + --data-binary) + data="$2" + shift + ;; + http://*) url="$1" ;; + esac + shift + done + if [ -n "$output" ]; then + [ "$(LC_ALL=C ls -ld "${data#@}" | cut -c1-10)" = '-rw-------' ] || return 1 + jq -c 'if has("pools") then {pools:"PRIVATE"} else . end' "${data#@}" >>"$CALLS" + id=$(printf '%016x' "$(wc -l <"$CALLS")") + [ "$mode" != bad-id ] || id='bad-id' + printf '{"change_id":"%s"}' "$id" >"$output" + printf 202 + [ "$mode" != curl-failed ] || return 7 + elif [[ "$url" == */status* ]]; then + printf '{"status":"applied"}' + else + id=$(printf '%016x' "$(wc -l <"$CALLS")") + published_id="$id" + [ "$mode" != wrong-id ] || published_id='ffffffffffffffff' + if [ "$mode" != stale ] && { [ "$mode" != late ] || [ "$((SECONDS - started))" -ge 95 ]; }; then + stamp='2026-09-30T00:00:01Z' + target=102 + fi + if [ "$mode" = history ]; then + printf '{"generated_at":"%s","rigforge":{"watchdog":{"max_temp_c":%s},"control":null,"control_history":[{"change_id":"%s","status":"applied"}]}}' "$stamp" "$target" "$published_id" + else + printf '{"generated_at":"%s","rigforge":{"watchdog":{"max_temp_c":%s},"control":{"change_id":"%s","status":"applied"},"control_history":[]}}' "$stamp" "$target" "$published_id" + fi + fi +} +phase_name="control-replay-$kind" +[ "$mode" != other ] || phase_name=control +replay_lock "$phase_name" +[ "$REPLAY_RUNNER_LOCK" = 1 ] || die 'reservation bypass not tracked' +phase_control_replay "$kind" +expected='{"max_temp_c":101} +{"max_temp_c":100} +{"DONATION":1} +{"DONATION":0} +{"watchdog_interval_min":6} +{"watchdog_interval_min":5} +{"pools":"PRIVATE"}' +[ "$kind" != thermal ] || expected="$expected +{\"max_temp_c\":102}" +[ "$(cat "$CALLS")" = "$expected" ] || die 'recorded apply order differs' +! grep -q 'SECRET' "$CALLS" || die 'pool credentials escaped into log' +# Exercise the real cleanup function: the runner marker survives, the ordinary marker does not. +eval "$(sed -n '/^_cleanup()/,/^}/p' "$ROOT/tests/e2e-pithead.sh")" +_restore_xmrig() { return 0; } +# shellcheck disable=SC2034 # consumed by the extracted cleanup function +RIGFORGE=/usr/bin/true HAMMER_PIDS='' +RIG_LOCK_HOLDER="$REPLAY_TEST_DIR/holder" +SAVED_CFG=$(mktemp) +cp "$CFG" "$SAVED_CFG" +printf 'runner owns release\n' >"$RIG_LOCK_HOLDER" +_cleanup +[ "$(cat "$RIG_LOCK_HOLDER")" = 'runner owns release' ] || die 'cleanup removed runner marker' +REPLAY_RUNNER_LOCK=0 +SAVED_CFG=$(mktemp) +cp "$CFG" "$SAVED_CFG" +_cleanup +[ ! -e "$RIG_LOCK_HOLDER" ] || die 'ordinary cleanup stopped removing its marker' diff --git a/tests/test-refresh-window-control.sh b/tests/test-refresh-window-control.sh index 14bdbb6..4df2d62 100644 --- a/tests/test-refresh-window-control.sh +++ b/tests/test-refresh-window-control.sh @@ -57,7 +57,7 @@ refresh_profile_state() { :; } refresh_profile_finish() { printf 'finish-window elapsed=%s\n' "$((SECONDS - started))"; } curl() { [[ "$*" != *Bearer* ]] || return 1 - [ "$1 $2" = "--config -" ] || return 1 + [ "$1 $2 $3 $4 $5" = "-q --noproxy * --config -" ] || return 1 grep -Eq '^header = "Authorization: Bearer [0-9a-f]{64}"$' || return 1 local output="" data="" url="" arg temp=100 stamp='2026-09-30T00:00:00Z' while [ "$#" -gt 0 ]; do From f93a8e09f249eccb7055ec8f54fb0b70839a89c1 Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:32:10 -0500 Subject: [PATCH 05/10] test: retain replay failure diagnostics before cleanup --- tests/README.md | 4 +++- tests/test-control-replay.sh | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index efd5f49..dec5a94 100644 --- a/tests/README.md +++ b/tests/README.md @@ -48,7 +48,9 @@ thermal and pools replay cases, submit one declaring phase per `tier4-e2e` job w `fixture=pithead-control`: `control-replay-thermal` or `control-replay-pools`. The deployed runner privately supplies the operator pools probe, context and named reservation. The gate verifies that reservation, bypasses only its own competing flock, and leaves the runner's holder marker and release -alone. Without the fixture, a replay is refused before config changes; ordinary locking is unchanged. +alone. The hardware-free `test-control-replay.sh` checks that refusal before config changes and the +unchanged ordinary locking; on failure it prints each synthetic case log's final 20 lines before +removing its sandbox, preserving the failed exit status. Both cases establish real jobs and an accepted share, retain thread/tuning and payout identity, set recorded scalars (thermal 100, DONATION 0, watchdog interval 5), and replay thermal 101 → 100, DONATION 1 → 0, watchdog interval 6 → 5, then the private pools probe. The thermal case verifies a 100 baseline, then applies direct 102 and requires a newer feed diff --git a/tests/test-control-replay.sh b/tests/test-control-replay.sh index bb89b02..a2bc7b9 100644 --- a/tests/test-control-replay.sh +++ b/tests/test-control-replay.sh @@ -5,7 +5,13 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" if [ "$#" = 0 ]; then REPLAY_TEST_DIR=$(mktemp -d) export REPLAY_TEST_DIR - trap 'rm -rf "$REPLAY_TEST_DIR"' EXIT + replay_test_cleanup() { + local rc=$? + if [ "$rc" != 0 ]; then tail -n 20 "$REPLAY_TEST_DIR"/*.log >&2 || true; fi + rm -rf "$REPLAY_TEST_DIR" + exit "$rc" + } + trap replay_test_cleanup EXIT for kind in thermal pools; do for success in fresh history; do bash "$0" "$kind" "$success" >"$REPLAY_TEST_DIR/$kind-$success.log" 2>&1 From 0524bc3ef7706cb9477a0bf91d506aad35b13fdf Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:39:53 -0500 Subject: [PATCH 06/10] test: use Bash builtin for replay apply fixture --- tests/README.md | 2 +- tests/test-control-replay.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index dec5a94..50836ff 100644 --- a/tests/README.md +++ b/tests/README.md @@ -48,7 +48,7 @@ thermal and pools replay cases, submit one declaring phase per `tier4-e2e` job w `fixture=pithead-control`: `control-replay-thermal` or `control-replay-pools`. The deployed runner privately supplies the operator pools probe, context and named reservation. The gate verifies that reservation, bypasses only its own competing flock, and leaves the runner's holder marker and release -alone. The hardware-free `test-control-replay.sh` checks that refusal before config changes and the +alone. The hardware-free `test-control-replay.sh` uses Bash builtins for its apply fixture and checks that refusal before config changes and the unchanged ordinary locking; on failure it prints each synthetic case log's final 20 lines before removing its sandbox, preserving the failed exit status. Both cases establish real jobs and an accepted share, retain thread/tuning and payout identity, set diff --git a/tests/test-control-replay.sh b/tests/test-control-replay.sh index a2bc7b9..02ee3a4 100644 --- a/tests/test-control-replay.sh +++ b/tests/test-control-replay.sh @@ -48,7 +48,7 @@ source "$ROOT/tests/e2e-control-replay.sh" CFG="$REPLAY_TEST_DIR/$kind-$mode.json" CALLS="$REPLAY_TEST_DIR/$kind-$mode.calls" PITHEAD_URL='fixture.invalid:3333' -RIGFORGE=/usr/bin/true +RIGFORGE=true # Bash's builtin fixture does not depend on a container's external true path. printf '{"control_port":8082,"api_port":8081,"pools":[{"url":"original.invalid:3333","user":"synthetic-user","pass":"SOURCE-SECRET"},{"url":"fixture.invalid:3333","user":"synthetic-user","pass":"BENCH-SECRET"}]}' >"$CFG" E2E_PITHEAD_CONTROL_FIXTURE=pithead-control E2E_RIG_LOCK_UNIT=bench-ci-riglock-synthetic-123 @@ -153,7 +153,7 @@ expected='{"max_temp_c":101} eval "$(sed -n '/^_cleanup()/,/^}/p' "$ROOT/tests/e2e-pithead.sh")" _restore_xmrig() { return 0; } # shellcheck disable=SC2034 # consumed by the extracted cleanup function -RIGFORGE=/usr/bin/true HAMMER_PIDS='' +RIGFORGE=true HAMMER_PIDS='' RIG_LOCK_HOLDER="$REPLAY_TEST_DIR/holder" SAVED_CFG=$(mktemp) cp "$CFG" "$SAVED_CFG" From 2d565c9663c3400c174ee1e9ce93542c6d6839d9 Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:22:48 -0500 Subject: [PATCH 07/10] test: account refresh CPU time and scheduler delay --- tests/README.md | 10 ++++++-- tests/e2e-api-refresh-profile.sh | 19 +++++++++++++++ tests/e2e-refresh-window.sh | 13 ++++++++--- tests/test-refresh-profile.sh | 40 ++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/tests/README.md b/tests/README.md index 50836ff..a4faf0f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -28,8 +28,14 @@ against GitHub Actions' ToS. They're a manual pre-tag gate the releaser runs. The queued `control` phase also instruments real timer-driven API refreshes around a thermal control apply (#546). A temporary runtime drop-in replaces only the refresh service's command with `e2e-api-refresh-profile.sh`; its priority, timeout, timer and live feed remain in use. -The profiler logs fixed helper names, begin/end timestamps and nanosecond durations, never -arguments or payloads. Five-second observations record timer/service state and the direct feed +The profiler logs fixed helper names, begin/end timestamps and nanosecond durations, plus full-pass +CPU ticks (the shell and waited-for children), clock ticks per second, nice level and the shell's +cumulative scheduler wait in nanoseconds. CPU-time deltas distinguish computation from elapsed +waiting; scheduler wait excludes children and can be zero when kernel scheduler statistics are +disabled, so zero does not prove absence of contention. Missing/malformed accounting or a journal +without a successful refresh and CPU begin/end from the same process fails diagnostics. +No arguments, process names or payloads are exported. +Five-second observations record timer/service state and the direct feed for five minutes around the apply; the changed ceiling must still appear within 90 seconds. Control-phase requests pass the temporary bearer token through curl's stdin configuration, keeping it out of process arguments. Each POST uses its own securely allocated response file, diff --git a/tests/e2e-api-refresh-profile.sh b/tests/e2e-api-refresh-profile.sh index 204d68d..6c70c7b 100644 --- a/tests/e2e-api-refresh-profile.sh +++ b/tests/e2e-api-refresh-profile.sh @@ -4,6 +4,23 @@ set -euo pipefail # shellcheck disable=SC1090 source "$1" +PROFILE_CLOCK_TICKS=$(getconf CLK_TCK) +[[ "$PROFILE_CLOCK_TICKS" =~ ^[1-9][0-9]*$ ]] +refresh_profile_cpu() { + local stat runtime wait slices field profile_pid="${BASHPID:-$$}" + local -a fields + IFS= read -r stat <"${REFRESH_PROFILE_STAT_FILE:-/proc/$profile_pid/stat}" || return 1 + # comm may contain spaces or parentheses; every field after its final ')' is numeric/state. + read -r -a fields <<<"${stat##*) }" + read -r runtime wait slices <"${REFRESH_PROFILE_SCHEDSTAT_FILE:-/proc/$profile_pid/schedstat}" || return 1 + for field in "${fields[11]:-}" "${fields[12]:-}" "${fields[13]:-}" "${fields[14]:-}" "$runtime" "$wait" "$slices"; do + [[ "$field" =~ ^[0-9]+$ ]] || return 1 + done + [[ "${fields[16]:-}" =~ ^-?[0-9]+$ ]] && [ "${fields[16]}" -ge -20 ] && [ "${fields[16]}" -le 19 ] || return 1 + # CPU ticks include this shell and waited-for children; scheduler delay is this shell only. + printf 'refresh-profile-cpu: pid=%s event=%s cpu_ticks=%s clock_ticks_per_s=%s nice=%s scheduler_wait_ns=%s\n' \ + "$profile_pid" "$1" "$((10#${fields[11]} + 10#${fields[12]} + 10#${fields[13]} + 10#${fields[14]}))" "$PROFILE_CLOCK_TICKS" "${fields[16]}" "$wait" >&2 +} refresh_profile_event() { printf 'refresh-profile: pid=%s helper=%s event=%s at_ns=%s duration_ns=%s rc=%s\n' \ "$$" "$1" "$2" "$3" "$4" "$5" >&2 @@ -16,8 +33,10 @@ for probe in api_refresh parse_config _read_api_summary _api_rigforge_block _api local profile_start profile_end profile_rc profile_start=\$(date +%s%N) refresh_profile_event $probe begin \"\$profile_start\" 0 0 + [ $probe != api_refresh ] || refresh_profile_cpu begin ${probe}_profile_original \"\$@\" profile_rc=\$? + [ $probe != api_refresh ] || refresh_profile_cpu end profile_end=\$(date +%s%N) refresh_profile_event $probe end \"\$profile_end\" \"\$((profile_end - profile_start))\" \"\$profile_rc\" return \"\$profile_rc\" diff --git a/tests/e2e-refresh-window.sh b/tests/e2e-refresh-window.sh index 41678bc..0e77ee8 100644 --- a/tests/e2e-refresh-window.sh +++ b/tests/e2e-refresh-window.sh @@ -34,7 +34,7 @@ refresh_profile_state() { -p ExecMainStartTimestamp -p ExecMainExitTimestamp -p ExecMainStatus } refresh_profile_finish() { - local log rc=0 + local log records rc=0 if [ -n "$REFRESH_PROFILE_STAGING" ]; then if rm -f "$REFRESH_PROFILE_STAGING"; then REFRESH_PROFILE_STAGING=""; else rc=1; fi fi @@ -48,8 +48,15 @@ refresh_profile_finish() { journalctl --sync || rc=1 journalctl -u rigforge-api-refresh.service --since "$REFRESH_PROFILE_SINCE UTC" --no-pager -o cat >"$log" || rc=1 # Publish only the fixed-schema measurements; other unit journal messages may contain secrets. - sed -nE '/^refresh-profile: pid=[0-9][0-9]* helper=[a-z_][a-z_]* event=(begin|end) at_ns=[0-9][0-9]* duration_ns=[0-9][0-9]* rc=[0-9][0-9]*$/p' "$log" || rc=1 - grep -Eq '^refresh-profile: .*helper=api_refresh event=end .*rc=0$' "$log" || rc=1 + records=$(sed -nE '/^refresh-profile: pid=[0-9][0-9]* helper=[a-z_][a-z_]* event=(begin|end) at_ns=[0-9][0-9]* duration_ns=[0-9][0-9]* rc=[0-9][0-9]*$/p; /^refresh-profile-cpu: pid=[0-9][0-9]* event=(begin|end) cpu_ticks=[0-9][0-9]* clock_ticks_per_s=[1-9][0-9]* nice=-?[0-9][0-9]* scheduler_wait_ns=[0-9][0-9]*$/p' "$log") || rc=1 + printf '%s\n' "$records" + # A completed pass needs its own CPU begin/end, not counters from another process. + printf '%s\n' "$records" | awk ' + $1 == "refresh-profile-cpu:" && $3 == "event=begin" { begun[$2] = 1 } + $1 == "refresh-profile-cpu:" && $3 == "event=end" && begun[$2] { cpu[$2] = 1 } + $1 == "refresh-profile:" && $3 == "helper=api_refresh" && $4 == "event=end" && $7 == "rc=0" && cpu[$2] { completed = 1 } + END { exit !completed } + ' || rc=1 rm -f "$log" fi if rm -f "$REFRESH_PROFILE_DROPIN" && systemctl daemon-reload; then diff --git a/tests/test-refresh-profile.sh b/tests/test-refresh-profile.sh index a59cdae..16c9574 100644 --- a/tests/test-refresh-profile.sh +++ b/tests/test-refresh-profile.sh @@ -9,8 +9,14 @@ check() { if ! "$@"; then exit 1 fi; } export TEST_PAYLOAD="$CASE_DIR/payload" +# Fixed Linux accounting fixtures keep these checks hardware-free, including on macOS. +export REFRESH_PROFILE_STAT_FILE="$CASE_DIR/stat" REFRESH_PROFILE_SCHEDSTAT_FILE="$CASE_DIR/schedstat" +stat_tail=(S 0 0 0 0 0 0 0 0 0 0 10 20 30 40 20 19) +printf '123 (synthetic (comm) name) %s\n' "${stat_tail[*]}" >"$REFRESH_PROFILE_STAT_FILE" +printf '100000 900000 3\n' >"$REFRESH_PROFILE_SCHEDSTAT_FILE" cat >"$CASE_DIR/fixture.sh" <"\$TEST_PAYLOAD"; } @@ -18,8 +24,21 @@ EOF_FIXTURE bash "$ROOT/tests/e2e-api-refresh-profile.sh" "$CASE_DIR/fixture.sh" 2>"$CASE_DIR/measurements" check grep -q '^refresh-profile: .*helper=_health_json event=begin ' "$CASE_DIR/measurements" check grep -q '^refresh-profile: .*helper=api_refresh event=end .*rc=0$' "$CASE_DIR/measurements" +check grep -q '^refresh-profile-cpu: .*event=begin cpu_ticks=100 clock_ticks_per_s=100 nice=19 scheduler_wait_ns=900000$' "$CASE_DIR/measurements" +check grep -q '^refresh-profile-cpu: .*event=end cpu_ticks=100 clock_ticks_per_s=100 nice=19 scheduler_wait_ns=900000$' "$CASE_DIR/measurements" check test "$(cat "$TEST_PAYLOAD")" = private-probe-value check bash -c '! grep -q private-probe-value "$1"' -- "$CASE_DIR/measurements" +check bash -c '! grep -q "synthetic (comm) name" "$1"' -- "$CASE_DIR/measurements" +# Malformed or unavailable counters cannot silently become zero or a successful profile. +cp "$REFRESH_PROFILE_STAT_FILE" "$CASE_DIR/stat-good" +printf '123 (synthetic) S\n' >"$REFRESH_PROFILE_STAT_FILE" +if bash "$ROOT/tests/e2e-api-refresh-profile.sh" "$CASE_DIR/fixture.sh" 2>"$CASE_DIR/bad-stat"; then exit 1; fi +check bash -c '! grep -q "refresh-profile-cpu:" "$1"' -- "$CASE_DIR/bad-stat" +cp "$CASE_DIR/stat-good" "$REFRESH_PROFILE_STAT_FILE" +if TEST_CLOCK_TICKS=unknown bash "$ROOT/tests/e2e-api-refresh-profile.sh" "$CASE_DIR/fixture.sh" 2>"$CASE_DIR/bad-clock"; then exit 1; fi +mv "$REFRESH_PROFILE_SCHEDSTAT_FILE" "$CASE_DIR/schedstat-good" +if bash "$ROOT/tests/e2e-api-refresh-profile.sh" "$CASE_DIR/fixture.sh" 2>"$CASE_DIR/missing-schedstat"; then exit 1; fi +mv "$CASE_DIR/schedstat-good" "$REFRESH_PROFILE_SCHEDSTAT_FILE" # A 50ms probe must be visible below one second, rather than rounded to zero as in job 1876. duration=$(sed -n 's/.*helper=_health_json event=end .*duration_ns=\([0-9]*\) rc=0/\1/p' "$CASE_DIR/measurements") check test "$duration" -ge 1000000 @@ -52,6 +71,7 @@ check test ! -e "$owned" check test -z "$REFRESH_PROFILE_DROPIN" check grep -q 'stop rigforge-api-refresh.service' "$CASE_DIR/calls" check grep -q 'helper=_health_json event=end' "$CASE_DIR/collected" +check grep -q '^refresh-profile-cpu: .*event=end cpu_ticks=100 ' "$CASE_DIR/collected" check bash -c '! grep -q sensitive "$1"' -- "$CASE_DIR/collected" # Timer must be quiesced before the service; restore it only after override removal/reload. check awk ' @@ -187,6 +207,26 @@ journalctl() { :; } if refresh_profile_finish >"$CASE_DIR/missing"; then exit 1; fi check test ! -e "$owned" # Even when journal collection fails, remove our override and propagate failure. +for invalid in missing-begin wrong-pid malformed; do + refresh_profile_start + journalctl() { + [ "$1" != --sync ] || return 0 + case "$invalid" in + missing-begin) command sed '/^refresh-profile-cpu: .*event=begin /d' "$CASE_DIR/measurements" ;; + wrong-pid) command sed '/^refresh-profile-cpu:/s/pid=[0-9]*/pid=0/' "$CASE_DIR/measurements" ;; + malformed) command sed '/^refresh-profile-cpu:/s/cpu_ticks=100/cpu_ticks=invalid/' "$CASE_DIR/measurements" ;; + esac + } + if refresh_profile_finish >"$CASE_DIR/$invalid"; then exit 1; fi + check test ! -e "$owned" +done +refresh_profile_start +journalctl() { + [ "$1" != --sync ] || return 0 + command sed '/^refresh-profile-cpu:/d' "$CASE_DIR/measurements" +} +if refresh_profile_finish >"$CASE_DIR/missing-cpu"; then exit 1; fi +check test ! -e "$owned" refresh_profile_start journalctl() { return 1; } if refresh_profile_finish >"$CASE_DIR/journal-failed"; then exit 1; fi From c9c4ce036c7251fe03b2ba5575bd3a871b989bbb Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:35:04 -0500 Subject: [PATCH 08/10] fix: give API refresh normal CPU scheduling --- docs/operations.md | 5 +++-- docs/pithead-integration.md | 4 +++- rigforge.sh | 2 +- systemd/rigforge-api-refresh.service.template | 6 +++--- tests/run.sh | 2 +- tests/test-api-refresh-liveness.sh | 6 ------ 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 533d36c..6ab1ffa 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -565,8 +565,9 @@ last-change provenance (`config_meta`; #254), and the last control-path outcome `rigforge` in `/1/summary` and `/2/summary`). It follows XMRig's own architecture: one tiny persistent server (python3 stdlib, ~10 MB idle) ships pre-computed bytes, so a request costs microseconds and cannot touch mining performance; a systemd timer schedules a refresh every 15 -seconds at idle priority, off the request path. The clock probe reads per-core values without -starting a process for each core. The queued thermal and separate pools replay tests check direct-feed +seconds off the request path. Refreshes use normal CPU priority (`Nice=0`) to share CPU time with +the miner when publishing control changes; disk I/O stays at idle priority and +the cached HTTP server retains `Nice=19`. The queued thermal and separate pools replay tests check direct-feed freshness and exact-ID publication, capturing timer-driven helper timings under the runner-owned mining fixture. They preserve the service policy and normal feed publication. The timer uses an independent wall-clock cadence, so a failed refresh or a unit reinstall cannot strand the feed without a future run. Summary and diff --git a/docs/pithead-integration.md b/docs/pithead-integration.md index 7fa932a..1ba7900 100644 --- a/docs/pithead-integration.md +++ b/docs/pithead-integration.md @@ -111,7 +111,9 @@ pool, `:8081` exists exactly while enabled, and no response byte ever contains ` pool `pass`. `:8080` stays the canonical Pithead summary probe; `:8081` is additive. Port/bind are `api_port`/`api_bind`. Architecture mirrors XMRig's own API: a tiny persistent server ships pre-computed state (a request costs microseconds — polling cannot shave hashrate), refreshed every -15s by an idle-priority wall-clock timer. Consumers should still age `generated_at`: it distinguishes +15s by a wall-clock timer. Refresh work uses normal CPU priority with idle disk I/O to give +control-change publication a normal CPU share during mining; the cached HTTP server remains at idle CPU +priority. Consumers should still age `generated_at`: it distinguishes a current report from a cached response when a rig or its refresh job is unhealthy. ### Stratum over TLS (optional) diff --git a/rigforge.sh b/rigforge.sh index 69feefe..580a2c4 100755 --- a/rigforge.sh +++ b/rigforge.sh @@ -4991,7 +4991,7 @@ _cpu_eff_khz() { local f v sum=0 n=0 for f in "$CPU_SYSFS"/cpu[0-9]*/cpufreq/scaling_cur_freq; do [ -r "$f" ] || continue - IFS= read -r v <"$f" || true + v=$(cat "$f" 2>/dev/null) case "$v" in '' | *[!0-9]*) continue ;; esac sum=$((sum + v)) n=$((n + 1)) diff --git a/systemd/rigforge-api-refresh.service.template b/systemd/rigforge-api-refresh.service.template index acb4271..3f08d2f 100644 --- a/systemd/rigforge-api-refresh.service.template +++ b/systemd/rigforge-api-refresh.service.template @@ -3,12 +3,12 @@ Description=RigForge sister API refresh (probe pass -> /run/rigforge-api, off th [Service] Type=oneshot -# Root for the probes (dmidecode / rdmsr / RAPL); idle priority — a background refresh has no -# latency requirement, so it must never compete with the miner. +# Root for the probes (dmidecode / rdmsr / RAPL). Give control-change publication a normal +# CPU share during mining; disk I/O remains idle priority. Environment=RIGFORGE_OPERATOR=$RIGFORGE_OPERATOR ExecStart=$SCRIPT_DIR/rigforge.sh api-refresh TimeoutStartSec=300 -Nice=19 +Nice=0 IOSchedulingClass=idle RuntimeDirectory=rigforge-api RuntimeDirectoryPreserve=yes diff --git a/tests/run.sh b/tests/run.sh index 6dd7c0b..73ba950 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -7275,7 +7275,7 @@ assert_contains "server ExecStart points at the credential copy (#sec)" "$(cat " assert_eq "credential var survives the envsubst render un-expanded (#sec)" "$(grep -c 'CREDENTIALS_DIRECTORY' "$APS/systemd/rigforge-api.service")" "2" assert_contains "server caps request-arrival time (slowloris) (#sec)" "$(cat "$ROOT/util/api-server.py")" "Handler.timeout" assert_contains "token compare is constant-time (#sec)" "$(cat "$ROOT/util/api-server.py")" "hmac.compare_digest" -assert_eq "refresh runs at idle priority off the request path (#164)" "$(grep -c '^IOSchedulingClass=idle$' "$APS/systemd/rigforge-api-refresh.service")" "1" +assert_eq "refresh gets normal CPU scheduling with idle disk I/O (#540)" "$(grep -E '^(Nice|IOSchedulingClass)=' "$APS/systemd/rigforge-api-refresh.service")" $'Nice=0\nIOSchedulingClass=idle' assert_contains "refresh cannot mask a frozen feed indefinitely (#476)" "$(cat "$APS/systemd/rigforge-api-refresh.service")" "TimeoutStartSec=300" assert_contains "refresh timer has an independent 15s wall-clock cadence (#454)" "$(cat "$APS/systemd/rigforge-api-refresh.timer")" "OnCalendar=*:*:0/15" assert_contains "refresh timer does not replay missed probes after downtime (#454)" "$(cat "$APS/systemd/rigforge-api-refresh.timer")" "Persistent=false" diff --git a/tests/test-api-refresh-liveness.sh b/tests/test-api-refresh-liveness.sh index d73b985..44fc62d 100644 --- a/tests/test-api-refresh-liveness.sh +++ b/tests/test-api-refresh-liveness.sh @@ -1,12 +1,6 @@ # shellcheck shell=bash disable=SC1090,SC2034,SC2329 # #454: doctor's diagnosis stays deterministic here. Real transient-systemd timing was checked # separately because this dependency-free suite must also run on macOS. -assert_eq "API clock probe reads every core without a process per core (#540)" "$( - source "$SCRIPT" - CPU_SYSFS="$DOC/cpu_throttle" - cat() { return 99; } - _cpu_eff_khz -)" "3000000" RFS="$(mktemp -d "$SANDBOX/refresh-status.XXXXXX")" printf '%s' '{"generated_at":"2026-09-07T04:00:00Z"}' >"$RFS/summary.json" refresh_status() { # [refresh state] From 0b888dd357f5e223e7ece55b895bec73392d15f4 Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:14:01 -0500 Subject: [PATCH 09/10] test: capture bounded mining and apply failure diagnostics --- tests/README.md | 3 ++ tests/e2e-control-replay.sh | 44 +++++++++++++++++++++++++++++- tests/e2e-pithead.sh | 6 ++-- tests/test-control-replay.sh | 53 ++++++++++++++++++++++++++++++++++-- 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/tests/README.md b/tests/README.md index a4faf0f..3787d60 100644 --- a/tests/README.md +++ b/tests/README.md @@ -65,6 +65,9 @@ case requires this exact change ID to reach `applied` in the enriched feed withi not a later change's outcome. Both continue five-minute observations for diagnostics and preserve the original pass/fail deadline even if a read crosses it. Full applies can stop/restart the miner; observations record its active state rather than aborting during that expected transition. +Each apply settlement emits only elapsed seconds and its validated status. Mining-precondition +failures emit fixed-schema systemd state plus counts of jobs, shares and classified XMRig errors; +unknown service values fail the diagnostic, and no journal text, endpoint or config value is printed. Requests use secured scratch payloads, never credential-bearing arguments or logs. The current probe and borrowed worker form an equivalent fixture: historical starting config/checkout, original pool payload, threads/affinity/tuning and precise inter-apply timing remain unknown in the runner context. diff --git a/tests/e2e-control-replay.sh b/tests/e2e-control-replay.sh index 6dd5dec..1caebce 100644 --- a/tests/e2e-control-replay.sh +++ b/tests/e2e-control-replay.sh @@ -18,6 +18,46 @@ replay_lock() { phase_control_replay_thermal() { phase_control_replay thermal; } phase_control_replay_pools() { phase_control_replay pools; } +_xmrig_diagnostics() { # Fixed-schema service and log counts; never export journal/config text. + local raw key value active sub result status log_state=missing counts='0 0 0 0 0 0 0 0' + local jobs accepted dns network tls auth config other + raw=$(systemctl show xmrig --property=ActiveState --property=SubState --property=Result --property=ExecMainStatus 2>/dev/null) || return 1 + while IFS='=' read -r key value; do + case "$key" in + ActiveState) active=$value ;; + SubState) sub=$value ;; + Result) result=$value ;; + ExecMainStatus) status=$value ;; + esac + done <<<"$raw" + case "${active:-}" in active | reloading | inactive | failed | activating | deactivating | maintenance) ;; *) return 1 ;; esac + case "${sub:-}" in dead | start-pre | start | start-post | running | exited | reload | stop | stop-watchdog | stop-sigterm | stop-sigkill | stop-post | final-sigterm | final-sigkill | failed | auto-restart | clean) ;; *) return 1 ;; esac + case "${result:-}" in success | resources | timeout | exit-code | signal | core-dump | watchdog | start-limit-hit | protocol | oom-kill | exec-condition | skipped) ;; *) return 1 ;; esac + [[ "${status:-}" =~ ^[0-9]{1,3}$ ]] && [ "$status" -le 255 ] || return 1 + if [ -n "${WLOG:-}" ] && [ -f "$WLOG" ]; then + log_state=present + counts=$(awk 'BEGIN { j=a=d=n=t=u=c=o=0 } + { line=tolower($0) } + line ~ /new job from/ { j++ } + line ~ /accepted \(/ { a++ } + line ~ /dns error|name or service not known|temporary failure in name resolution/ { d++; next } + line ~ /tls error|failed to verify server certificate fingerprint/ { t++; next } + line ~ /permission denied|login error|unauthorized/ { u++; next } + line ~ /configuration error|config error|failed to parse config|invalid config/ { c++; next } + line ~ /connect error|connection refused|connection timed out|network error|no active pools/ { n++; next } + line ~ /error|failed|refused|denied|invalid/ { o++ } + END { print j, a, d, n, t, u, c, o }' "$WLOG" 2>/dev/null) || return 1 + [[ "$counts" =~ ^[0-9]+\ [0-9]+\ [0-9]+\ [0-9]+\ [0-9]+\ [0-9]+\ [0-9]+\ [0-9]+$ ]] || return 1 + fi + read -r jobs accepted dns network tls auth config other <<<"$counts" + printf 'xmrig-diagnostic: active_state=%s sub_state=%s result=%s exec_main_status=%s log_state=%s jobs=%s accepted=%s dns_errors=%s network_errors=%s tls_errors=%s auth_errors=%s config_errors=%s other_errors=%s\n' \ + "$active" "$sub" "$result" "$status" "$log_state" "$jobs" "$accepted" "$dns" "$network" "$tls" "$auth" "$config" "$other" +} +_connect_bad() { + _xmrig_diagnostics || printf 'xmrig-diagnostic: unavailable\n' >&2 + bad "$1" +} + _replay_apply() { # JSON on stdin; credentials never become process arguments or log output. local request response code id curl_rc=0 request=$(mktemp) || return 1 @@ -41,9 +81,11 @@ _replay_status() { jq -sr 'select(length == 1) | .[0].status | select(. == "applied" or . == "pending" or . == "accepted" or . == "rolled_back" or . == "failed" or . == "rejected")' || true } _replay_settle() { # Preserve the recorded 90-second apply bound, without logging payloads. - local started=$SECONDS st + local started=$SECONDS st elapsed while [ "$((SECONDS - started))" -lt 90 ]; do st=$(_replay_status) + elapsed=$((SECONDS - started)) + printf 'replay-settle: elapsed_s=%s status=%s\n' "$elapsed" "${st:-unknown}" refresh_profile_state allow-inactive || return 1 [ "$((SECONDS - started))" -lt 90 ] || break case "$st" in diff --git a/tests/e2e-pithead.sh b/tests/e2e-pithead.sh index d492787..e8fe4fe 100644 --- a/tests/e2e-pithead.sh +++ b/tests/e2e-pithead.sh @@ -174,7 +174,7 @@ phase_connect() { systemctl is-active --quiet xmrig || "$RIGFORGE" start >/dev/null 2>&1 [ -n "$WLOG" ] || WLOG="$(find "$HERE" -path '*worker*' -name xmrig.log 2>/dev/null | head -1)" if [ -z "$WLOG" ]; then - bad "could not find the worker's xmrig.log" + _connect_bad "could not find the worker's xmrig.log" return 0 fi : >"$WLOG" || true # truncate so every assertion below is about THIS stack, not an old pool @@ -182,7 +182,7 @@ phase_connect() { if wait_for_job 60; then ok "connected — stratum job from $(grep -oE 'new job from [^ ]+' "$WLOG" | tail -1 | awk '{print $NF}')" else - bad "no stratum job from $PITHEAD_URL within 60s — is the stack up and reachable?" + _connect_bad "no stratum job from $PITHEAD_URL within 60s — is the stack up and reachable?" fi local share_to="${E2E_SHARE_TIMEOUT:-180}" waited=0 while [ "$waited" -lt "$share_to" ] && ! grep -q 'accepted (' "$WLOG" 2>/dev/null; do @@ -192,7 +192,7 @@ phase_connect() { if grep -q 'accepted (' "$WLOG" 2>/dev/null; then ok "share accepted by the stack ($(grep -c 'accepted (' "$WLOG") so far)" else - bad "no accepted share within ${share_to}s" + _connect_bad "no accepted share within ${share_to}s" fi # Discovery contract: the dashboard identifies workers by the stratum `user` label. local user diff --git a/tests/test-control-replay.sh b/tests/test-control-replay.sh index 02ee3a4..bfaf96c 100644 --- a/tests/test-control-replay.sh +++ b/tests/test-control-replay.sh @@ -26,12 +26,16 @@ if [ "$#" = 0 ]; then grep -q "feed stale after 90s" "$REPLAY_TEST_DIR/$kind-$mode.log" done done - for mode in missing inactive context probe other bad-id mining curl-failed; do + for mode in missing inactive context probe other bad-id mining curl-failed status-missing status-malformed; do if bash "$0" thermal "$mode" >"$REPLAY_TEST_DIR/$mode.log" 2>&1; then echo "replay $mode unexpectedly passed" >&2 exit 1 fi done + for mode in status-missing status-malformed; do + grep -q 'replay-settle: elapsed_s=.* status=unknown' "$REPLAY_TEST_DIR/$mode.log" + ! grep -q 'PRIVATE\|never-execute' "$REPLAY_TEST_DIR/$mode.log" + done ( source "$ROOT/tests/e2e-refresh-window.sh" systemctl() { [ "$1" = show ]; } @@ -39,6 +43,47 @@ if [ "$#" = 0 ]; then refresh_profile_state allow-inactive >>"$REPLAY_TEST_DIR/inactive-state" grep -q 'xmrig_active=0' "$REPLAY_TEST_DIR/inactive-state" ) + ( + source "$ROOT/tests/e2e-control-replay.sh" + WLOG="$REPLAY_TEST_DIR/xmrig.log" + printf '%s\n' 'nEw JoB fRoM fixture' 'AcCePtEd (1/0)' 'DnS ErRoR' 'CoNnEcT ErRoR' 'TlS ErRoR' 'LoGiN ErRoR' 'CoNfIg ErRoR' 'FaIlEd for another reason' >"$WLOG" + systemctl() { printf '%s\n' 'ActiveState=active' 'SubState=running' 'Result=success' 'ExecMainStatus=0'; } + _xmrig_diagnostics >"$REPLAY_TEST_DIR/xmrig-diagnostic" + grep -qx 'xmrig-diagnostic: active_state=active sub_state=running result=success exec_main_status=0 log_state=present jobs=1 accepted=1 dns_errors=1 network_errors=1 tls_errors=1 auth_errors=1 config_errors=1 other_errors=1' "$REPLAY_TEST_DIR/xmrig-diagnostic" + WLOG="$REPLAY_TEST_DIR/absent-xmrig.log" + _xmrig_diagnostics | grep -q 'log_state=missing jobs=0 accepted=0' + systemctl() { printf '%s\n' 'ActiveState=active' 'SubState=running' 'Result=success'; } + ! _xmrig_diagnostics + systemctl() { printf '%s\n' 'ActiveState=PRIVATE' 'SubState=running' 'Result=success' 'ExecMainStatus=0'; } + ! _xmrig_diagnostics + systemctl() { printf '%s\n' 'ActiveState=active' 'SubState=running' 'Result=success' 'ExecMainStatus=0'; } + WLOG="$REPLAY_TEST_DIR/xmrig.log" + awk() { + printf 'PRIVATE PATH' >&2 + return 1 + } + ! _xmrig_diagnostics 2>"$REPLAY_TEST_DIR/xmrig-diagnostic-error" + [ ! -s "$REPLAY_TEST_DIR/xmrig-diagnostic-error" ] + ) + ( + source "$ROOT/tests/e2e-control-replay.sh" + eval "$(sed -n '/^phase_connect()/,/^}/p' "$ROOT/tests/e2e-pithead.sh")" + phase() { :; } + set_cfg() { :; } + find() { :; } + bad() { + printf 'bad:%s\n' "$1" + exit 1 + } + systemctl() { + [ "$1" = show ] && printf '%s\n' 'ActiveState=active' 'SubState=running' 'Result=success' 'ExecMainStatus=0' + } + # shellcheck disable=SC2034 # consumed by the extracted phase_connect function + HERE="$REPLAY_TEST_DIR" RIGFORGE=true WLOG='' GEN_CFG='' PITHEAD_URL='fixture.invalid:3333' + if (phase_connect) >"$REPLAY_TEST_DIR/connect-diagnostic" 2>&1; then exit 1; fi + grep -q 'xmrig-diagnostic: .* log_state=missing jobs=0 accepted=0' "$REPLAY_TEST_DIR/connect-diagnostic" + grep -q "bad:could not find the worker's xmrig.log" "$REPLAY_TEST_DIR/connect-diagnostic" + ) printf 'control replay regressions: PASS\n' exit 0 fi @@ -117,7 +162,11 @@ curl() { printf 202 [ "$mode" != curl-failed ] || return 7 elif [[ "$url" == */status* ]]; then - printf '{"status":"applied"}' + case "$mode" in + status-missing) printf '{}' ;; + status-malformed) printf '{"status":"PRIVATE $(never-execute)"}' ;; + *) printf '{"status":"applied"}' ;; + esac else id=$(printf '%016x' "$(wc -l <"$CALLS")") published_id="$id" From bfabfadc81f1bd1e259cde5d204ba03717783f48 Mon Sep 17 00:00:00 2001 From: "pithead-developer[bot]" <4933481+pithead-developer[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:06:05 -0500 Subject: [PATCH 10/10] Reject failed HTTP reads in control replay assertions --- tests/README.md | 5 ++++- tests/e2e-control-replay.sh | 13 ++++++++++--- tests/test-control-replay.sh | 31 ++++++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/tests/README.md b/tests/README.md index 3787d60..1543050 100644 --- a/tests/README.md +++ b/tests/README.md @@ -65,7 +65,10 @@ case requires this exact change ID to reach `applied` in the enriched feed withi not a later change's outcome. Both continue five-minute observations for diagnostics and preserve the original pass/fail deadline even if a read crosses it. Full applies can stop/restart the miner; observations record its active state rather than aborting during that expected transition. -Each apply settlement emits only elapsed seconds and its validated status. Mining-precondition +Status and summary GETs consume JSON only after a successful transport; failed-read bytes are +discarded and a fixed transport-failure marker is emitted. Valid JSON followed by a nonzero +transport exit cannot satisfy settlement or publication. Each apply settlement emits only +elapsed seconds and its validated status. Mining-precondition failures emit fixed-schema systemd state plus counts of jobs, shares and classified XMRig errors; unknown service values fail the diagnostic, and no journal text, endpoint or config value is printed. Requests use secured scratch payloads, never credential-bearing arguments or logs. The current probe diff --git a/tests/e2e-control-replay.sh b/tests/e2e-control-replay.sh index 1caebce..2094f86 100644 --- a/tests/e2e-control-replay.sh +++ b/tests/e2e-control-replay.sh @@ -77,8 +77,12 @@ _replay_apply() { # JSON on stdin; credentials never become process arguments or printf '%s\n' "$id" } _replay_status() { - _control_curl "$tok" -fsS --max-time 3 --max-filesize 16384 "http://127.0.0.1:$port/status?change_id=$cid" 2>/dev/null | - jq -sr 'select(length == 1) | .[0].status | select(. == "applied" or . == "pending" or . == "accepted" or . == "rolled_back" or . == "failed" or . == "rejected")' || true + local body + if ! body=$(_control_curl "$tok" -fsS --max-time 3 --max-filesize 16384 "http://127.0.0.1:$port/status?change_id=$cid" 2>/dev/null); then + printf 'replay-read: status transport failed\n' >&2 + return 0 + fi + printf '%s' "$body" | jq -sr 'select(length == 1) | .[0].status | select(. == "applied" or . == "pending" or . == "accepted" or . == "rolled_back" or . == "failed" or . == "rejected")' || true } _replay_settle() { # Preserve the recorded 90-second apply bound, without logging payloads. local started=$SECONDS st elapsed @@ -101,7 +105,10 @@ _replay_window() { # -- publication of this exact ID, not a late while [ "$((SECONDS - started))" -lt 300 ]; do refresh_profile_state allow-inactive || return 1 st=$(_replay_status) - body=$(_control_curl "$tok" -fsS --max-time 3 --max-filesize 65536 "http://127.0.0.1:$api_port/1/summary" 2>/dev/null || true) + if ! body=$(_control_curl "$tok" -fsS --max-time 3 --max-filesize 65536 "http://127.0.0.1:$api_port/1/summary" 2>/dev/null); then + body='' + printf 'replay-read: summary transport failed\n' >&2 + fi # Only bounded scalar fields are exported; arbitrary responses/config remain private. stamp=$(printf '%s' "$body" | jq -sr 'select(length == 1) | .[0].generated_at | select(type == "string" and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"))' 2>/dev/null || true) temp=$(printf '%s' "$body" | jq -sr 'select(length == 1) | .[0].rigforge.watchdog.max_temp_c | select(type == "number" and . >= 40 and . <= 110)' 2>/dev/null || true) diff --git a/tests/test-control-replay.sh b/tests/test-control-replay.sh index bfaf96c..1dc21f5 100644 --- a/tests/test-control-replay.sh +++ b/tests/test-control-replay.sh @@ -18,24 +18,47 @@ if [ "$#" = 0 ]; then grep -q "replay $kind published within 90s" "$REPLAY_TEST_DIR/$kind-$success.log" grep -q 'finish-profile' "$REPLAY_TEST_DIR/$kind-$success.log" done - for mode in stale late wrong-id; do + for mode in stale late wrong-id summary-failed; do if bash "$0" "$kind" "$mode" >"$REPLAY_TEST_DIR/$kind-$mode.log" 2>&1; then echo "replay $kind $mode unexpectedly passed" >&2 exit 1 fi grep -q "feed stale after 90s" "$REPLAY_TEST_DIR/$kind-$mode.log" + if [ "$mode" = summary-failed ]; then + grep -q '^replay-read: summary transport failed$' "$REPLAY_TEST_DIR/$kind-$mode.log" + ! grep -q 'published=true' "$REPLAY_TEST_DIR/$kind-$mode.log" + fi done done - for mode in missing inactive context probe other bad-id mining curl-failed status-missing status-malformed; do + for mode in missing inactive context probe other bad-id mining curl-failed status-missing status-malformed status-failed; do if bash "$0" thermal "$mode" >"$REPLAY_TEST_DIR/$mode.log" 2>&1; then echo "replay $mode unexpectedly passed" >&2 exit 1 fi done - for mode in status-missing status-malformed; do + for mode in status-missing status-malformed status-failed; do grep -q 'replay-settle: elapsed_s=.* status=unknown' "$REPLAY_TEST_DIR/$mode.log" ! grep -q 'PRIVATE\|never-execute' "$REPLAY_TEST_DIR/$mode.log" done + grep -q '^replay-read: status transport failed$' "$REPLAY_TEST_DIR/status-failed.log" + ! grep -q 'status=applied' "$REPLAY_TEST_DIR/status-failed.log" + ( + source "$ROOT/tests/e2e-control-replay.sh" + tok=synthetic port=8082 api_port=8081 cid=0123456789abcdef before=2026-09-30T00:00:00Z + refresh_profile_state() { :; } + sleep() { SECONDS=$((SECONDS + $1)); } + _control_curl() { + if [[ "${*: -1}" == */status* ]]; then + printf '{"status":"applied"}' + return 18 + fi + printf '{"generated_at":"2026-09-30T00:00:01Z","rigforge":{"watchdog":{"max_temp_c":102},"control":{"change_id":"0123456789abcdef","status":"applied"}}}' + } + for kind in thermal pools; do + if _replay_window "$kind" >"$REPLAY_TEST_DIR/$kind-status-window.log" 2>&1; then exit 1; fi + grep -q 'status=unknown.*published=true' "$REPLAY_TEST_DIR/$kind-status-window.log" + done + ) ( source "$ROOT/tests/e2e-refresh-window.sh" systemctl() { [ "$1" = show ]; } @@ -167,6 +190,7 @@ curl() { status-malformed) printf '{"status":"PRIVATE $(never-execute)"}' ;; *) printf '{"status":"applied"}' ;; esac + [ "$mode" != status-failed ] || return 18 else id=$(printf '%016x' "$(wc -l <"$CALLS")") published_id="$id" @@ -180,6 +204,7 @@ curl() { else printf '{"generated_at":"%s","rigforge":{"watchdog":{"max_temp_c":%s},"control":{"change_id":"%s","status":"applied"},"control_history":[]}}' "$stamp" "$target" "$published_id" fi + [ "$mode" != summary-failed ] || return 18 fi } phase_name="control-replay-$kind"