From a7ad00ddd8b687f1c2e0031b0f134f2d3edcc678 Mon Sep 17 00:00:00 2001 From: Vijit Singh Date: Fri, 17 Jul 2026 19:14:26 -0500 Subject: [PATCH 1/3] refactor(doctor): a dedicated check_tor_running verdict for a tor-only outage (#621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows up #619. That fix made BOTH egress checks fail when tor was down while mining ran — two FAIL lines for one root cause. Replace that with a single dedicated check_tor_running() that owns the "tor down while the stack runs" verdict (OK when up, FAIL when down+mining, info-skip on a clean down), called first in doctor's Network section; the two egress checks revert to a plain info-skip when tor is down. One loud line, not two. doctor still exits non-zero on the outage (the #618 fault_tor_down assertion holds). tor is unconditional (no compose profile gates it), so this never false-fires on a legitimate tor-absent deployment. Co-authored-by: Claude Opus 4.8 --- pithead | 36 ++++++++++++++++++++++++++---------- tests/stack/run.sh | 15 ++++++++++----- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/pithead b/pithead index a51c293b..50eca345 100755 --- a/pithead +++ b/pithead @@ -859,6 +859,7 @@ doctor() { echo "Network:" check_stratum_exposure doctor check_stratum_listening + check_tor_running check_egress_firewall_installed check_tor_clearnet_egress # Clearnet initial sync (#183): a deliberate, privacy-relevant opt-in. Warn whenever it's on so @@ -1763,6 +1764,24 @@ mining_stack_running() { container_is_running p2pool || container_is_running monerod || container_is_running xmrig-proxy } +# doctor (#563): the tor container is unconditional (no compose profile gates it — it's the +# Tor-first backbone in every deployment, remote-node mode included). So "revenue containers up +# but tor down" is never a legitimate state: it means tor crashed or was stopped individually +# while the stack kept mining, leaving clearnet dials no longer fail-closed and every off-box +# connection (Healthchecks, Telegram, XvB, p2pool/Tari peers) without its Tor path. FAIL loudly so +# doctor can't report all-clear on a silent privacy outage. A clean `down` (nothing running) is +# fine — nothing to guard. +check_tor_running() { + if container_is_running tor; then + dr_ok "Tor container is running — the privacy backbone is up." + elif mining_stack_running; then + dr_fail "The Tor container is DOWN while the mining stack is still running — the privacy backbone is dead: clearnet dials are no longer fail-closed and off-box connections (Healthchecks, Telegram, XvB, peers) have lost their Tor path. Restart it ('./pithead restart tor'; set tor.auto_heal:true to self-heal), or bring the stack down ('./pithead down')." + else + dr_info "Tor container isn't running — the stack is down (expected after './pithead down')." + fi + return 0 +} + # doctor (#383): verify the #270 fail-closed egress rules are ACTUALLY installed while the stack # runs. `down` removes them, and a host reboot silently drops them while `restart: unless-stopped` # brings every container back — that reboot gap is the state this catches. Read-only: `sudo -n` @@ -1776,11 +1795,10 @@ check_egress_firewall_installed() { return 0 fi if ! container_is_running tor; then - if mining_stack_running; then - dr_fail "The Tor container is DOWN while the mining stack is still running — the privacy backbone is dead and clearnet dials are no longer fail-closed. Restart Tor ('./pithead restart tor'), or bring the stack down ('./pithead down')." - else - dr_info "Tor-egress firewall check skipped — the stack isn't running (rules are removed at 'down')." - fi + # tor down while the stack runs is caught by check_tor_running (a dedicated, loud verdict); + # here it's just an info-skip either way — a clean `down` removed the rules, and a tor-only + # outage is already being FAILed above. + dr_info "Tor-egress firewall check skipped — the tor container isn't running." return 0 fi if ! command -v iptables >/dev/null 2>&1; then @@ -1859,11 +1877,9 @@ check_dashboard_answers() { # must not fail cron health gates on a slow circuit. check_tor_clearnet_egress() { if ! container_is_running tor; then - if mining_stack_running; then - dr_fail "The Tor container is DOWN while the mining stack runs — every off-box connection (Healthchecks, Telegram, XvB, p2pool/Tari peers) has lost its Tor path. Restart Tor ('./pithead restart tor'; set tor.auto_heal:true to self-heal), or bring the stack down." - else - dr_info "Tor clearnet-egress check skipped — the tor container isn't running." - fi + # A tor-only outage is FAILed by check_tor_running; this test can't probe egress without a + # live tor either way, so it just info-skips. + dr_info "Tor clearnet-egress check skipped — the tor container isn't running." return 0 fi if ! command -v curl >/dev/null 2>&1; then diff --git a/tests/stack/run.sh b/tests/stack/run.sh index 0e8b4d7d..ab088890 100755 --- a/tests/stack/run.sh +++ b/tests/stack/run.sh @@ -202,12 +202,17 @@ rm -f "$SANDBOX/.env" # Stack down (tor not running) -> info skip: rules are legitimately absent after 'down'. out="$(RUNNING_CONTAINERS="" PATH="$DRBIN:$PATH" run_sourced "$SANDBOX" check_egress_firewall_installed 2>&1)" assert_contains "egress check: stack down -> info" "$out" "isn't running" -# tor DOWN while the mining stack runs -> FAIL, not a silent skip: the privacy backbone is dead but -# revenue containers are live (tor crashed / stopped individually). doctor must not report all-clear. +# The tor-down-while-mining verdict lives in the dedicated check_tor_running (#563), so the egress +# checks just info-skip when tor is down — they don't double-FAIL the same root cause. out="$(RUNNING_CONTAINERS="p2pool" PATH="$DRBIN:$PATH" run_sourced "$SANDBOX" check_egress_firewall_installed 2>&1)" -assert_contains "egress check: tor down + mining up -> FAIL" "$out" "Tor container is DOWN" -out="$(RUNNING_CONTAINERS="p2pool" PATH="$DRBIN:$PATH" run_sourced "$SANDBOX" check_tor_clearnet_egress 2>&1)" -assert_contains "clearnet-egress check: tor down + mining up -> FAIL" "$out" "Tor container is DOWN" +assert_contains "egress check: tor down -> info skip (dedicated check owns the verdict)" "$out" "isn't running" +# check_tor_running: the loud, dedicated privacy-outage verdict. +out="$(RUNNING_CONTAINERS="tor" PATH="$DRBIN:$PATH" run_sourced "$SANDBOX" check_tor_running 2>&1)" +assert_contains "tor-running check: tor up -> OK" "$out" "privacy backbone is up" +out="$(RUNNING_CONTAINERS="" PATH="$DRBIN:$PATH" run_sourced "$SANDBOX" check_tor_running 2>&1)" +assert_contains "tor-running check: whole stack down -> info" "$out" "stack is down" +out="$(RUNNING_CONTAINERS="p2pool" PATH="$DRBIN:$PATH" run_sourced "$SANDBOX" check_tor_running 2>&1)" +assert_contains "tor-running check: tor down + mining up -> FAIL" "$out" "Tor container is DOWN" # Running + tagged rules present -> OK. out="$(RUNNING_CONTAINERS="tor" IPT_TAGGED=1 PATH="$DRBIN:$PATH" run_sourced "$SANDBOX" check_egress_firewall_installed 2>&1)" assert_contains "egress check: rules installed -> OK" "$out" "installed" From 4b96d2d877d5bbf27df269faa7d5804b83d9ea34 Mon Sep 17 00:00:00 2001 From: Vijit Singh Date: Fri, 17 Jul 2026 19:40:09 -0500 Subject: [PATCH 2/3] fix(#622): one-click upgrade rides out a 502/503/504 from the proxy (#625) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The self-upgrade recreates the dashboard container itself. While it restarts, caddy (the reverse proxy) stays up and answers 502/503/504 — the upstream is briefly gone, not failed. pollResult already rode out a dropped connection (proxy down) but treated a gateway 5xx (proxy up, no upstream) as terminal, so the modal jumped to "failed / HTTP 502" even though the upgrade landed and the durable control result said "upgraded". Ride out 502/503/504 like a dropped connection; the durable control result is the real outcome and the poll cap is the backstop. A genuine backend 500 (upstream up, erroring) still fast-fails. Regression test: a 502 for the first N polls then the terminal "upgraded" result must resolve to done, not failed. Co-authored-by: Claude Opus 4.8 --- CHANGELOG.md | 11 +++++++++++ .../mining_dashboard/web/static/configview.mjs | 10 ++++++++-- .../dashboard/tests/frontend/configview.test.mjs | 16 ++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 815786dc..20bd5a3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ per the process in [`docs/releasing.md`](docs/releasing.md). ## [Unreleased] +### Fixed + +- **One-click upgrade no longer shows a false "HTTP 502 — did not complete" on a + successful upgrade (#622).** The self-upgrade recreates the dashboard container + itself; while it restarts, caddy (the reverse proxy) stays up and answers + 502/503/504 because the upstream is briefly gone. The result poller already rode + out a dropped connection but treated a gateway 5xx as terminal, so the modal + jumped to "failed" even though the upgrade landed. Gateway 502/503/504 are now + ridden out like a dropped connection; the durable control result is the real + outcome. A genuine backend 500 still fast-fails. + ## [1.8.0] - 2026-07-17 **Config UX round 2.** The Configuration view is regrouped around logical diff --git a/build/dashboard/mining_dashboard/web/static/configview.mjs b/build/dashboard/mining_dashboard/web/static/configview.mjs index e21d8ffb..82f7082f 100644 --- a/build/dashboard/mining_dashboard/web/static/configview.mjs +++ b/build/dashboard/mining_dashboard/web/static/configview.mjs @@ -43,8 +43,9 @@ const UPGRADE_POLL_MAX = 450; // 15 minutes — an upgrade pulls a whole release // Poll /api/control/result until a terminal result lands; shared by the Configuration view and // the Upgrade button (#59). `skip` ignores an intermediate status under the same id (the // still-present "previewed" result while a commit runs; "running" while an upgrade runs). Both -// flows recreate the dashboard container itself, so a fetch here can transiently fail (connection -// refused mid-restart) — ride it out and keep polling until the result file answers. +// flows recreate the dashboard container itself, so a fetch here can transiently fail — either a +// dropped connection (proxy down) or a 502/503/504 (proxy up, upstream mid-restart, #622). Ride +// both out and keep polling until the result file answers. async function pollResult(id, skip, max = POLL_MAX) { for (let i = 0; i < max; i++) { await new Promise((r) => setTimeout(r, POLL_MS)); @@ -55,6 +56,11 @@ async function pollResult(id, skip, max = POLL_MAX) { continue; } if (res.status === 202) continue; + // Both flows recreate the dashboard container itself; while it restarts, the reverse proxy + // (caddy) stays up and answers 502/503/504 — the upstream is briefly gone, not failed. Ride + // these out like a dropped connection (#59/#622); the durable control result is the real + // outcome and `max` is the backstop. A real backend 500 (upstream up, erroring) still throws. + if (res.status === 502 || res.status === 503 || res.status === 504) continue; if (!res.ok) throw new Error(`HTTP ${res.status}`); const out = await res.json(); if (out.status === skip) continue; diff --git a/build/dashboard/tests/frontend/configview.test.mjs b/build/dashboard/tests/frontend/configview.test.mjs index 08dd9491..e3dce34d 100644 --- a/build/dashboard/tests/frontend/configview.test.mjs +++ b/build/dashboard/tests/frontend/configview.test.mjs @@ -94,6 +94,22 @@ test("runUpgrade posts the seen version, skips 'running', rides out the restart, assert.equal(posted.headers["X-Pithead-Control"], "1"); // CSRF guard rides every mutation }); +test("runUpgrade rides out a 502/503/504 from the proxy (upstream mid-restart), not just a dropped connection (#622)", async () => { + let polls = 0; + const fetchStub = async (url) => { + if (url === "/api/control/upgrade") { + return { status: 202, ok: false, json: async () => ({ id: ID, status: "pending" }) }; + } + polls++; + // caddy stays up and answers a gateway error while the dashboard upstream is recreated — + // the common self-upgrade state, and the one the old poller misclassified as terminal. + if (polls <= 3) return { status: 502, ok: false, json: async () => ({}) }; + return okResult({ status: "upgraded", version: "v9.9.9" }); + }; + const out = await withFastPoll(fetchStub, () => runUpgrade("v9.9.9")); + assert.equal(out.status, "upgraded"); // rode out the 502s to the durable result, no throw +}); + test("runUpgrade surfaces a host-side rejection as the outcome, not a throw", async () => { const fetchStub = async (url) => url === "/api/control/upgrade" From ac9b7230448a6753f4112b2d477707fb3b9dc609 Mon Sep 17 00:00:00 2001 From: Vijit Singh Date: Fri, 17 Jul 2026 19:47:53 -0500 Subject: [PATCH 3/3] =?UTF-8?q?release:=20prep=20v1.8.1=20=E2=80=94=20one-?= =?UTF-8?q?click=20upgrade=20502=20fix=20(#622)=20(#626)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch release over v1.8.0: the one-click upgrade poller now rides out a 502/503/504 from the reverse proxy during the dashboard's own restart, so a successful upgrade no longer surfaces a false "HTTP 502 — did not complete" modal (#622). VERSION + dashboard pyproject/uv.lock bumped in lockstep. Co-authored-by: Claude Opus 4.8 --- CHANGELOG.md | 2 ++ VERSION | 2 +- build/dashboard/pyproject.toml | 2 +- build/dashboard/uv.lock | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20bd5a3e..84f69c02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ per the process in [`docs/releasing.md`](docs/releasing.md). ## [Unreleased] +## [1.8.1] - 2026-07-18 + ### Fixed - **One-click upgrade no longer shows a false "HTTP 502 — did not complete" on a diff --git a/VERSION b/VERSION index afa2b351..b9268dae 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.0 \ No newline at end of file +1.8.1 \ No newline at end of file diff --git a/build/dashboard/pyproject.toml b/build/dashboard/pyproject.toml index f715df9a..1e6a3570 100644 --- a/build/dashboard/pyproject.toml +++ b/build/dashboard/pyproject.toml @@ -7,7 +7,7 @@ name = "mining-dashboard" # Keep in lockstep with the top-level VERSION file — the single source of truth for the stack version # (#44). A shell test (tests/stack/run.sh) fails if these drift; the dashboard *displays* the version # from VERSION (baked in as PITHEAD_VERSION, #58), so this is packaging metadata only. -version = "1.8.0" +version = "1.8.1" description = "Monitoring dashboard and XvB switching engine for Pithead" readme = "README.md" requires-python = ">=3.11" diff --git a/build/dashboard/uv.lock b/build/dashboard/uv.lock index 20da352f..9ace3f43 100644 --- a/build/dashboard/uv.lock +++ b/build/dashboard/uv.lock @@ -782,7 +782,7 @@ wheels = [ [[package]] name = "mining-dashboard" -version = "1.8.0" +version = "1.8.1" source = { editable = "." } dependencies = [ { name = "aiofiles" },