From 1058c0c32963b5f0749ee3e35fc9e5b185e32640 Mon Sep 17 00:00:00 2001 From: Vijit Singh Date: Tue, 21 Jul 2026 10:17:48 -0500 Subject: [PATCH 01/13] feat(#196): expose blocks/disk_growth/xvb_history on /api/state (Tier-1) (#707) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(#196): expose blocks/disk_growth/xvb_history on /api/state (Tier-1) The #196 telemetry backbone (PR #600) added persisted SQLite tables and capture hooks for five series but shipped without surfacing any of them to the client. This adds the missing /api/state exposure for the three Tier-1 series — blocks, disk_growth, xvb_history — range-filtered like share_stats and bucket-averaged past the existing 700-point chart cap for the two higher-cadence series. network_history and worker_history (Tier-2) are untouched. Co-Authored-By: Claude Opus 4.8 * refactor(#196): fold disk_growth/xvb_history formatters into one gauge-series helper ponytail-review: build_disk_growth and build_xvb_history duplicated the same filter -> downsample -> {x, ...cols} shape, differing only in which columns they carry. Shared _gauge_series helper, net -13 lines. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- CHANGELOG.md | 13 ++ build/dashboard/mining_dashboard/web/views.py | 77 ++++++++ build/dashboard/tests/web/test_views.py | 177 +++++++++++++++++- docs/dashboard.md | 5 + 4 files changed, 271 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd9bd68d..eddf014c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ Pithead ships as **one product, one version** — the version lives in the top-l [`VERSION`](VERSION) file and every released image is tagged with it. Releases are cut per the process in [`docs/dev/releasing.md`](docs/dev/releasing.md). +## [Unreleased] + +### Added + +- **`/api/state` exposure for three of the #196 telemetry-backbone series (Tier-1).** The backbone + PR (#600) added five persisted SQLite tables with capture, storage, and retention, but shipped + without surfacing them to the client. This slice exposes three — `blocks` (pool block-found + events), `disk_growth` (hourly monerod-DB-size + host-disk-usage samples), and `xvb_history` + (~5-min XvB-credited scalar samples) — as range-filtered arrays under those same keys, bounded + at the existing 700-point chart cap for the two higher-cadence series. `network_history` and + `worker_history` are a separate (Tier-2) slice, not touched here. No chart renders any of these + series yet — that's a further follow-up. + ## [1.10.0] - 2026-07-20 ### Added diff --git a/build/dashboard/mining_dashboard/web/views.py b/build/dashboard/mining_dashboard/web/views.py index e0c2cf24..64d3d03b 100644 --- a/build/dashboard/mining_dashboard/web/views.py +++ b/build/dashboard/mining_dashboard/web/views.py @@ -335,6 +335,77 @@ def _window_reject_pct(rows, seconds): return "—" if pct is None else f"{pct:.2f}%" +# -------------------------------------------------------------------------------------- +# #196 Tier-1 telemetry backbone: blocks / disk_growth / xvb_history surfaced on /api/state. +# The backbone (capture + storage + retention, PR #600) shipped without this exposure step — +# these three formatters are it. network_history and worker_history are Tier-2 (a separate +# slice of the epic) and are not exposed here. +# -------------------------------------------------------------------------------------- + + +def _downsample_gauge_rows(rows, value_cols, target=_MAX_CHART_POINTS): + """Bucket-average arbitrary point-in-time (gauge) columns down to ``target`` points. + + Mirrors ``_downsample_share_stats``, but averages instead of summing: these rows are + periodic READINGS (disk size, XvB credited averages), not per-interval deltas, so summing + them would inflate the series instead of thinning it. A no-op when already at/under target.""" + if len(rows) <= target: + return rows + chunk_size = len(rows) / target + out = [] + for i in range(target): + chunk = rows[int(i * chunk_size) : int((i + 1) * chunk_size)] + if not chunk: + continue + bucket = {"ts": chunk[len(chunk) // 2]["ts"]} + for col in value_cols: + vals = [r.get(col, 0) or 0 for r in chunk] + bucket[col] = round(sum(vals) / len(vals), 2) + out.append(bucket) + return out + + +def build_blocks(blocks, range_arg, window=None): + """Persisted P2Pool block-found events (#196) as chart-ready points, restricted to the + selected range/window (``_filter_events`` bounds any ts-keyed list, so this table reuses + it as-is). A handful of rows a week — no downsampling needed.""" + return [ + { + "x": int(b["ts"] * 1000), + "height": b.get("height", 0), + "difficulty": b.get("difficulty", 0), + } + for b in _filter_events(blocks, range_arg, window) + ] + + +def _gauge_series(rows, range_arg, window, value_cols): + """Shared shape for a persisted gauge series (#196): filter to the selected range/window, + bucket-average past ``_MAX_CHART_POINTS`` like ``share_stats``, and key each row's own + ``value_cols`` under ``x`` (ms epoch). ``build_disk_growth``/``build_xvb_history`` are this + with their own column set — the only thing that differs between them.""" + filtered = _downsample_gauge_rows(_filter_events(rows, range_arg, window), value_cols) + return [{"x": int(r["ts"] * 1000), **{c: r.get(c, 0) for c in value_cols}} for r in filtered] + + +def build_disk_growth(rows, range_arg, window=None): + """Persisted hourly monerod-DB-size + host-disk-usage samples (#196) as chart-ready points — + the table keeps every row (no retention prune), so a long-lived install can otherwise pass + the chart-point cap ``_gauge_series`` bounds it at.""" + return _gauge_series( + rows, range_arg, window, ("monero_db_bytes", "disk_used_gb", "disk_total_gb") + ) + + +def build_xvb_history(rows, range_arg, window=None): + """Persisted ~5-minute XvB-credited scalar samples (#196) as chart-ready points — the 30-day + retention at this cadence is ~8.6k rows, well past the chart-point cap ``_gauge_series`` + bounds it at.""" + return _gauge_series( + rows, range_arg, window, ("avg_1h", "avg_24h", "fail_count", "donation_fraction") + ) + + def _window_duration(filtered_history, range_arg, window): """Seconds the chart currently spans — drives adaptive resolution/smoothing. From the window if zoomed, else the preset length, else (``all``/unknown) the actual data extent.""" @@ -1673,6 +1744,12 @@ def build_state(data, state_mgr, range_arg, window=None, avg_window=DEFAULT_HASH # proxy_summary so its (cumulative) shape stays unchanged for existing clients. "share_stats": build_share_stats(share_stats, range_arg, window), "reject_pct_24h": _window_reject_pct(share_stats, 24 * 3600), + # #196 Tier-1 telemetry backbone exposure: block-found events, hourly disk-growth + # samples, and ~5-min XvB-credited samples. No chart renders these yet — that's the + # deliberate next slice — the payload just carries the persisted series. + "blocks": build_blocks(state_mgr.get_blocks(), range_arg, window), + "disk_growth": build_disk_growth(state_mgr.get_disk_growth(), range_arg, window), + "xvb_history": build_xvb_history(state_mgr.get_xvb_history(), range_arg, window), "egress": egress, "topology": topology, "chart": build_chart( diff --git a/build/dashboard/tests/web/test_views.py b/build/dashboard/tests/web/test_views.py index 0872ea02..33faccec 100644 --- a/build/dashboard/tests/web/test_views.py +++ b/build/dashboard/tests/web/test_views.py @@ -30,8 +30,10 @@ _target_points, _window_reject_pct, build_badges, + build_blocks, build_cadence, build_chart, + build_disk_growth, build_earnings, build_energy, build_hashrate, @@ -46,6 +48,7 @@ build_worker_detail, build_workers, build_xvb_calc, + build_xvb_history, canonical_window, get_shell_html, host_display_addr, @@ -1393,6 +1396,137 @@ def test_short_series_is_untouched(self): assert len(pts) == 3 # under the cap -> native resolution, no bucketing +class TestBlocksDiskGrowthXvbHistorySeries: + """#196 Tier-1: the persisted blocks/disk_growth/xvb_history backbone surfaced on + /api/state. network_history and worker_history are Tier-2 and out of scope here.""" + + def test_build_blocks_shape_and_ms_epoch(self): + now = time.time() + rows = [{"ts": now - 60, "height": 42, "difficulty": 123.0}] + pts = build_blocks(rows, "all") + assert pts == [{"x": int((now - 60) * 1000), "height": 42, "difficulty": 123.0}] + + def test_build_blocks_range_filters_old_rows(self): + now = time.time() + rows = [ + {"ts": now - 8 * 24 * 3600, "height": 1, "difficulty": 1.0}, + {"ts": now - 60, "height": 2, "difficulty": 2.0}, + ] + assert len(build_blocks(rows, "1w")) == 1 + + def test_build_blocks_empty(self): + assert build_blocks([], "all") == [] + + def test_build_disk_growth_shape_and_ms_epoch(self): + now = time.time() + rows = [ + { + "ts": now - 3600, + "monero_db_bytes": 85_000_000_000, + "disk_used_gb": 120.5, + "disk_total_gb": 500.0, + } + ] + pts = build_disk_growth(rows, "all") + assert pts == [ + { + "x": int((now - 3600) * 1000), + "monero_db_bytes": 85_000_000_000, + "disk_used_gb": 120.5, + "disk_total_gb": 500.0, + } + ] + + def test_build_disk_growth_range_filters_old_rows(self): + now = time.time() + rows = [ + { + "ts": now - 8 * 24 * 3600, + "monero_db_bytes": 1, + "disk_used_gb": 1, + "disk_total_gb": 1, + }, + {"ts": now - 60, "monero_db_bytes": 2, "disk_used_gb": 2, "disk_total_gb": 2}, + ] + assert len(build_disk_growth(rows, "1w")) == 1 + + def test_build_disk_growth_long_series_is_bounded_and_bucket_averaged(self): + # Hourly, permanent (no retention prune) -> a long-lived install can pass the cap. + now = time.time() + rows = [ + { + "ts": now - i * 3600, + "monero_db_bytes": 100, + "disk_used_gb": 10.0, + "disk_total_gb": 500.0, + } + for i in range(10_000, 0, -1) # ascending ts, like the DB returns them + ] + pts = build_disk_growth(rows, "all") + assert len(pts) <= _MAX_CHART_POINTS + # Every source row carries the same constant values, so the bucket average is exact. + assert all(p["monero_db_bytes"] == 100 and p["disk_used_gb"] == 10.0 for p in pts) + + def test_build_xvb_history_shape_and_ms_epoch(self): + now = time.time() + rows = [ + { + "ts": now - 300, + "avg_1h": 1000.0, + "avg_24h": 900.0, + "fail_count": 0, + "donation_fraction": 0.5, + } + ] + pts = build_xvb_history(rows, "all") + assert pts == [ + { + "x": int((now - 300) * 1000), + "avg_1h": 1000.0, + "avg_24h": 900.0, + "fail_count": 0, + "donation_fraction": 0.5, + } + ] + + def test_build_xvb_history_range_filters_old_rows(self): + now = time.time() + rows = [ + { + "ts": now - 40 * 24 * 3600, + "avg_1h": 1, + "avg_24h": 1, + "fail_count": 0, + "donation_fraction": 0, + }, + { + "ts": now - 300, + "avg_1h": 2, + "avg_24h": 2, + "fail_count": 0, + "donation_fraction": 0, + }, + ] + assert len(build_xvb_history(rows, "1m")) == 1 + + def test_build_xvb_history_long_series_is_bounded_and_bucket_averaged(self): + # 30 days at ~5-min cadence is ~8.6k rows, well past the chart cap. + now = time.time() + rows = [ + { + "ts": now - i * 300, + "avg_1h": 1000.0, + "avg_24h": 900.0, + "fail_count": 0, + "donation_fraction": 0.5, + } + for i in range(8_640, 0, -1) # ascending ts, like the DB returns them + ] + pts = build_xvb_history(rows, "all") + assert len(pts) <= _MAX_CHART_POINTS + assert all(p["avg_1h"] == 1000.0 for p in pts) + + # --- pool/network passthrough --------------------------------------------------------- @@ -1749,7 +1883,14 @@ def test_empty_estimates_available_false_no_crash(self): # ponytail: this _state_mgr()/_data() pair looks near-duplicated with the ones in test_metrics.py, # but the per-module defaults differ on purpose (e.g. tari_sync, the get_tiers/xvb shapes). A shared # builder would need enough params that it reads worse than the local copy — left duplicated. -def _state_mgr(history=None, mode="P2POOL", share_stats=None): +def _state_mgr( + history=None, + mode="P2POOL", + share_stats=None, + blocks=None, + disk_growth=None, + xvb_history=None, +): sm = MagicMock() sm.get_history.return_value = history or [] sm.get_xvb_stats.return_value = {"current_mode": mode} @@ -1758,6 +1899,10 @@ def _state_mgr(history=None, mode="P2POOL", share_stats=None): sm.get_share_stats.return_value = share_stats or [] sm.get_raffle_wins.return_value = [] sm.is_db_healthy.return_value = True + # #196 Tier-1 telemetry backbone exposure. + sm.get_blocks.return_value = blocks or [] + sm.get_disk_growth.return_value = disk_growth or [] + sm.get_xvb_history.return_value = xvb_history or [] return sm @@ -1804,6 +1949,9 @@ def test_has_all_sections(self): "proxy_summary", "share_stats", "reject_pct_24h", + "blocks", + "disk_growth", + "xvb_history", "egress", "topology", "chart", @@ -1881,6 +2029,33 @@ def test_share_stats_series_and_24h_rate_surfaced(self): empty = build_state(_data(), _state_mgr(), "all") assert empty["share_stats"] == [] and empty["reject_pct_24h"] == "—" + def test_blocks_disk_growth_xvb_history_surfaced(self): + # #196 Tier-1: the three backbone series ride on /api/state, each sourced from its own + # StateManager getter. + now = time.time() + sm = _state_mgr( + blocks=[{"ts": now - 60, "height": 5, "difficulty": 10.0}], + disk_growth=[ + {"ts": now - 60, "monero_db_bytes": 1, "disk_used_gb": 2.0, "disk_total_gb": 3.0} + ], + xvb_history=[ + { + "ts": now - 60, + "avg_1h": 1, + "avg_24h": 2, + "fail_count": 0, + "donation_fraction": 0.1, + } + ], + ) + st = build_state(_data(), sm, "all") + assert st["blocks"][0]["height"] == 5 + assert st["disk_growth"][0]["monero_db_bytes"] == 1 + assert st["xvb_history"][0]["avg_1h"] == 1 + # Fresh install -> empty series, not a crash. + empty = build_state(_data(), _state_mgr(), "all") + assert empty["blocks"] == [] and empty["disk_growth"] == [] and empty["xvb_history"] == [] + def test_db_unhealthy_surfaces_field_and_badge(self): # When persistence is broken, /api/state must carry db_healthy=False and a loud badge (#131). sm = _state_mgr() diff --git a/docs/dashboard.md b/docs/dashboard.md index 27e45dbd..e3175673 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -154,6 +154,11 @@ a fresh database, and keeps running. A `db_reset` alert (Telegram and the other history before that point was cleared. Payout and XvB state rebuild from the chain and the live feed; only the historical charts reset. +The database also keeps three smaller series: pool block-found events, hourly monerod-DB-size and +host-disk-usage samples, and XvB-credited scalar samples taken roughly every 5 minutes. `/api/state` +serves them as `blocks`, `disk_growth`, and `xvb_history`, range-filtered the same way as +`share_stats`. No chart reads them yet — this is persistence and API exposure only. + While a node is down, the dashboard rejects workers so they fail over to the backup pools you've configured, rather than sitting idle on a stack that can't mine. A sustained outage stops the `xmrig-proxy` container (a `Workers rejected` badge shows) and a confirmed recovery restarts it. From c5a3b1de51ae383d9aa97af9d6d1b2da69600cf6 Mon Sep 17 00:00:00 2001 From: Vijit Singh Date: Tue, 21 Jul 2026 10:42:51 -0500 Subject: [PATCH 02/13] feat(#530): audit out-of-band config + rig changes, grouped by time (#711) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(#530): audit out-of-band config changes, group the trail by time Detects two kinds of config drift the #33 control-channel audit never saw: config.json changing without a matching dashboard commit (host-edit), and a rig applying a control-API change the dashboard never sent (rig-edit). Both append key-names-only rows to a new durable audit_events table, which also mirrors control.log so the Security panel's new hour/day/month grouping can drill deeper than the log's own trimmed tail. Co-Authored-By: Claude Opus 4.8 * chore(#530): drop unearned "reuse wherever" framing from a single-use CSS class ponytail-review pass: .card-header-row has exactly one caller today (the audit trail's grouping select) — the comment claimed forward-looking genericity it hasn't earned yet. Co-Authored-By: Claude Opus 4.8 * fix(#530): bound rig-edit audit rows, re-mask host config, pin correlation window Review follow-up on PR #711: - HIGH: rig-edit re-fired every ~30s poll into a permanent, never-pruned table — a rig keeps reporting its last terminal change_id, and the random row id defeated INSERT OR IGNORE. Fix: deterministic id `rig-edit--` (bounds the table to distinct real edits, survives restart) plus an in-memory (worker, change_id) guard that skips the redundant per-poll DB work. A rogue unauthenticated rig can no longer flood the trail with one bogus change_id. Tested across two consecutive polls and across a simulated restart. - MEDIUM: _read_host_config now re-applies the SECRET_PATHS mask (mirroring control_service.read_config's defense-in-depth pass), so a host-side masking regression can't leave a raw secret value resident in _last_host_config. The mount is already the pre-masked copy (#440) — the raw config.json is never mounted — so this is belt-and-suspenders, not a live leak; docstring corrected to state that accurately. - LOW: pinned the ≤1s "explained by a fresh commit" correlation window with a boundary test and named the ceiling in a ponytail comment. Co-Authored-By: Claude Opus 4.8 * fix(#530): sanitize all audit-row fields, add accessible name to grouping select Security review follow-up on PR #711: - _record_audit_event only ran actor/keys through audit_service._clean, despite the docstring's claim every field gets the same whitelist treatment. action/ status/source were code-controlled closed-enum values already, so this closes the gap defense-in-depth (no behavior change on any real input). - The hour/day/month onGroupChange(e.target.value)}> + + + + + ` + : null + } + ${ audit.length === 0 ? html`

No config changes have gone through the dashboard yet.

` @@ -75,15 +128,14 @@ const AuditCard = ({ audit }) => { - ${audit.map( - (e) => html` - - - - - - `, - )} + ${groupAuditEntries(audit, group).flatMap((g) => [ + g.bucket !== null + ? html` + + ` + : null, + ...g.entries.map(AuditRow), + ])}
Time (UTC)UserActionOutcomeSettings
${e.ts}${e.actor}${e.action}${e.status}${e.keys}
${g.bucket} (${g.entries.length})
` @@ -94,7 +146,10 @@ const AuditCard = ({ audit }) => { export class SecurityPanel extends Component { constructor(props) { super(props); - this.state = { access: null, audit: null, error: null }; + // auditGroup: "flat" (today's plain newest-first list) is the default so existing behavior + // doesn't change until the operator opts into grouping (#530). + this.state = { access: null, audit: null, auditGroup: "flat", error: null }; + this.setAuditGroup = (group) => this.setState({ auditGroup: group }); } async componentDidMount() { @@ -110,11 +165,11 @@ export class SecurityPanel extends Component { } render() { - const { access, audit, error } = this.state; + const { access, audit, auditGroup, error } = this.state; if (error) return html`

${error}

`; return html`
<${AccessCard} access=${access} /> - <${AuditCard} audit=${audit} /> + <${AuditCard} audit=${audit} group=${auditGroup} onGroupChange=${this.setAuditGroup} />
`; } } diff --git a/build/dashboard/tests/frontend/securityview.test.mjs b/build/dashboard/tests/frontend/securityview.test.mjs index bd49fb64..a8bff9ed 100644 --- a/build/dashboard/tests/frontend/securityview.test.mjs +++ b/build/dashboard/tests/frontend/securityview.test.mjs @@ -11,14 +11,19 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { fmtEpoch, SecurityPanel } from "../../mining_dashboard/web/static/securityview.mjs"; +import { + bucketKey, + fmtEpoch, + groupAuditEntries, + SecurityPanel, +} from "../../mining_dashboard/web/static/securityview.mjs"; import { renderToString } from "./helpers/render.mjs"; // Render the panel with its state pre-set (the render probe runs no effects, so the fetches in // componentDidMount never fire — state is injected directly, like the configview tests). function renderPanel(state) { const panel = new SecurityPanel({}); - panel.state = { access: null, audit: null, error: null, ...state }; + panel.state = { access: null, audit: null, auditGroup: "flat", error: null, ...state }; return renderToString(panel.render()); } @@ -100,3 +105,92 @@ test("fetch error surfaces as a message, not a blank panel", () => { const out = renderPanel({ error: "TypeError: fetch failed" }); assert.match(out, /fetch failed/); }); + +// #530: hour/day/month grouping for the audit trail. + +test("bucketKey: hour/day/month slice the shared ts format; unknown granularity ignored by the caller", () => { + const ts = "2026-07-20T14:35:00Z"; + assert.equal(bucketKey(ts, "hour"), "2026-07-20T14"); + assert.equal(bucketKey(ts, "day"), "2026-07-20"); + assert.equal(bucketKey(ts, "month"), "2026-07"); + assert.equal(bucketKey(42, "day"), ""); // non-string ts never throws +}); + +test("groupAuditEntries: flat/unknown granularity returns one ungrouped run", () => { + const entries = [{ ts: "2026-07-20T00:00:00Z" }, { ts: "2026-07-19T00:00:00Z" }]; + assert.deepEqual(groupAuditEntries(entries, "flat"), [{ bucket: null, entries }]); + assert.deepEqual(groupAuditEntries(entries, undefined), [{ bucket: null, entries }]); +}); + +test("groupAuditEntries: contiguous same-day entries collapse into one bucket", () => { + const entries = [ + { ts: "2026-07-20T14:00:00Z", actor: "a" }, + { ts: "2026-07-20T09:00:00Z", actor: "b" }, + { ts: "2026-07-19T23:00:00Z", actor: "c" }, + ]; + const groups = groupAuditEntries(entries, "day"); + assert.equal(groups.length, 2); + assert.equal(groups[0].bucket, "2026-07-20"); + assert.equal(groups[0].entries.length, 2); + assert.equal(groups[1].bucket, "2026-07-19"); + assert.equal(groups[1].entries.length, 1); +}); + +test("groupAuditEntries: month grouping spans multiple days in one bucket", () => { + const entries = [ + { ts: "2026-07-20T00:00:00Z" }, + { ts: "2026-07-01T00:00:00Z" }, + { ts: "2026-06-30T00:00:00Z" }, + ]; + const groups = groupAuditEntries(entries, "month"); + assert.deepEqual( + groups.map((g) => [g.bucket, g.entries.length]), + [ + ["2026-07", 2], + ["2026-06", 1], + ], + ); +}); + +test("audit card: default flat grouping shows no group-header row (unchanged row output)", () => { + const out = renderPanel({ + access: { available: true, entries: [] }, + audit: [{ ts: "2026-07-20T12:00:00Z", actor: "admin", action: "commit", status: "applied", keys: "XVB_ENABLED" }], + }); + assert.doesNotMatch(out, /audit-group-header/); + assert.match(out, /XVB_ENABLED/); +}); + +test("audit card: grouping select appears once there are entries, with the current group selected", () => { + const out = renderPanel({ + access: { available: true, entries: [] }, + audit: [{ ts: "2026-07-20T12:00:00Z", actor: "admin", action: "commit", status: "applied", keys: "XVB_ENABLED" }], + auditGroup: "day", + }); + assert.match(out, / a ``host-edit`` + audit row. Real StateManager throughout, like TestReconcileWorkerConfig — every assertion + reads back the persisted row.""" + + def _svc(self): + from mining_dashboard.service.storage_service import StateManager + + sm = StateManager(db_path=":memory:") + svc = DataService(sm, MagicMock(), MagicMock()) + return svc, sm + + def _write_config(self, path, doc): + path.write_text(json.dumps(doc)) + + async def test_control_disabled_is_a_noop(self, monkeypatch): + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", False) + svc, sm = self._svc() + try: + await svc._watch_host_config() + assert sm.get_audit_events() == [] + assert svc._last_host_config is None + finally: + sm.close() + + async def test_first_poll_only_baselines(self, tmp_path, monkeypatch): + cfg = tmp_path / "config.json" + self._write_config(cfg, {"xvb": {"enabled": True}}) + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(cfg)) + svc, sm = self._svc() + try: + await svc._watch_host_config() + assert sm.get_audit_events() == [] + assert svc._last_host_config == {"xvb": {"enabled": True}} + finally: + sm.close() + + async def test_unexplained_change_is_recorded_host_edit(self, tmp_path, monkeypatch): + cfg, log = tmp_path / "config.json", tmp_path / "control.log" + self._write_config(cfg, {"xvb": {"enabled": True}}) + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(cfg)) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + svc, sm = self._svc() + try: + await svc._watch_host_config() # baseline + self._write_config(cfg, {"xvb": {"enabled": False}}) # changed out of band + await svc._watch_host_config() + events = sm.get_audit_events() + assert len(events) == 1 + assert events[0]["source"] == "host-edit" + assert events[0]["keys"] == "xvb.enabled" + assert events[0]["status"] == "detected" + finally: + sm.close() + + async def test_change_explained_by_a_fresh_commit_is_quiet(self, tmp_path, monkeypatch): + cfg, log = tmp_path / "config.json", tmp_path / "control.log" + self._write_config(cfg, {"xvb": {"enabled": True}}) + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(cfg)) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + svc, sm = self._svc() + try: + await svc._watch_host_config() # baseline + self._write_config(cfg, {"xvb": {"enabled": False}}) + # A commit landed AFTER the baseline check — it explains the change. + log.write_text( + json.dumps( + { + "ts": _iso_now(), + "id": "11111111-1111-4111-8111-111111111111", + "actor": "admin", + "action": "commit", + "status": "applied", + "keys": "XVB_ENABLED", + } + ) + + "\n" + ) + await svc._watch_host_config() + assert sm.get_audit_events() == [] + finally: + sm.close() + + async def test_commit_of_one_key_does_not_swallow_a_concurrent_host_edit( + self, tmp_path, monkeypatch + ): + # #530 review MEDIUM: correlate BY KEY, not just by time. A fresh dashboard commit of key A + # landing in the same window as a host-side hand-edit of key B must NOT suppress B — the + # out-of-band change the feature exists to catch. Fails on the old time-only `explained = + # any(...)` logic, which swallowed the whole diff on ANY fresh commit. + cfg, log = tmp_path / "config.json", tmp_path / "control.log" + # A: xvb.enabled (committable, maps to XVB_ENABLED). B: dashboard.tari_required (maps to + # TARI_REQUIRED) — hand-edited on the host, NOT named by the commit below. + self._write_config(cfg, {"xvb": {"enabled": True}, "dashboard": {"tari_required": True}}) + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(cfg)) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + svc, sm = self._svc() + try: + await svc._watch_host_config() # baseline + # Both keys change; only A was actually committed through the dashboard. + self._write_config( + cfg, {"xvb": {"enabled": False}, "dashboard": {"tari_required": False}} + ) + log.write_text( + json.dumps( + { + "ts": _iso_now(), + "id": "11111111-1111-4111-8111-111111111111", + "actor": "admin", + "action": "commit", + "status": "applied", + "keys": "XVB_ENABLED", + } + ) + + "\n" + ) + await svc._watch_host_config() + events = sm.get_audit_events() + # B is recorded out-of-band; A (explained by the commit) is NOT double-recorded. + assert len(events) == 1 + assert events[0]["source"] == "host-edit" + assert events[0]["keys"] == "dashboard.tari_required" + finally: + sm.close() + + async def test_energy_commit_explains_an_energy_subkey_by_prefix(self, tmp_path, monkeypatch): + # #530: dashboard.energy.* is config.json-only and audits under the synthetic + # DASHBOARD_ENERGY name, which env_key_config_paths maps to the whole `dashboard.energy` + # block by prefix — so a committed energy sub-key change stays quiet even though its dotted + # diff path (dashboard.energy.cost_per_kwh) isn't the literal committed name. + cfg, log = tmp_path / "config.json", tmp_path / "control.log" + self._write_config(cfg, {"dashboard": {"energy": {"cost_per_kwh": 0.10}}}) + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(cfg)) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + svc, sm = self._svc() + try: + await svc._watch_host_config() # baseline + self._write_config(cfg, {"dashboard": {"energy": {"cost_per_kwh": 0.20}}}) + log.write_text( + json.dumps( + { + "ts": _iso_now(), + "id": "11111111-1111-4111-8111-111111111111", + "actor": "admin", + "action": "commit", + "status": "applied", + "keys": "DASHBOARD_ENERGY", + } + ) + + "\n" + ) + await svc._watch_host_config() + assert sm.get_audit_events() == [] + finally: + sm.close() + + async def test_stale_commit_before_the_last_check_does_not_explain(self, tmp_path, monkeypatch): + cfg, log = tmp_path / "config.json", tmp_path / "control.log" + self._write_config(cfg, {"xvb": {"enabled": True}}) + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(cfg)) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + svc, sm = self._svc() + try: + # An old commit, already accounted for before this watcher ever ran, then a NEW + # out-of-band change — the stale commit must not explain it away. + log.write_text( + json.dumps( + { + "ts": "2020-01-01T00:00:00Z", + "id": "11111111-1111-4111-8111-111111111111", + "actor": "admin", + "action": "commit", + "status": "applied", + "keys": "XVB_ENABLED", + } + ) + + "\n" + ) + await svc._watch_host_config() # baseline + self._write_config(cfg, {"xvb": {"enabled": False}}) + await svc._watch_host_config() + events = sm.get_audit_events() + assert len(events) == 1 + assert events[0]["source"] == "host-edit" + finally: + sm.close() + + async def test_missing_mount_is_a_quiet_noop(self, monkeypatch): + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", "/nonexistent/config.json") + svc, sm = self._svc() + try: + await svc._watch_host_config() + assert sm.get_audit_events() == [] + finally: + sm.close() + + async def test_a_raw_secret_value_never_reaches_the_audit_row(self, tmp_path, monkeypatch): + # Defense-in-depth (#530 review MEDIUM): even if the host masking regressed and left a RAW + # secret in the mounted copy, the re-mask keeps its value out of the persisted snapshot and + # out of any audit row. Change a non-secret key alongside the raw secret; the row names the + # non-secret key only, and the secret value appears nowhere. + cfg, log = tmp_path / "config.json", tmp_path / "control.log" + self._write_config( + cfg, {"dashboard": {"auth": {"password": "s3cr3t-raw"}}, "xvb": {"enabled": True}} + ) + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(cfg)) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + svc, sm = self._svc() + try: + await svc._watch_host_config() # baseline (secret already masked in the snapshot) + assert svc._last_host_config["dashboard"]["auth"]["password"] == {"__secret__": True} + self._write_config( + cfg, + {"dashboard": {"auth": {"password": "s3cr3t-changed"}}, "xvb": {"enabled": False}}, + ) + await svc._watch_host_config() + events = sm.get_audit_events() + assert len(events) == 1 + assert events[0]["keys"] == "xvb.enabled" # the secret masks to a sentinel both sides + blob = json.dumps(events) + json.dumps(svc._last_host_config) + assert "s3cr3t-raw" not in blob and "s3cr3t-changed" not in blob + finally: + sm.close() + + async def test_explained_window_is_at_most_one_second(self, tmp_path, monkeypatch): + # Boundary (#530 review LOW): the "explained by a fresh commit" check floors `since` to + # `_last_host_check - 1` to absorb the audit log's whole-second ts truncation. That grace + # is exactly 1s wide — a commit 1s before the last check still explains (truncation), one + # 2s before does not. Pinned here so the honest ceiling can't silently widen. + cfg, log = tmp_path / "config.json", tmp_path / "control.log" + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(cfg)) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + check_epoch = _parse_audit_ts("2026-07-20T12:00:05Z") + + async def _run_with_commit_ts(commit_ts): + svc, sm = self._svc() + svc._last_host_config = {"xvb": {"enabled": True}} + svc._last_host_check = check_epoch + self._write_config(cfg, {"xvb": {"enabled": False}}) + log.write_text( + json.dumps( + { + "ts": commit_ts, + "id": "11111111-1111-4111-8111-111111111111", + "actor": "admin", + "action": "commit", + "status": "applied", + "keys": "XVB_ENABLED", + } + ) + + "\n" + ) + try: + await svc._watch_host_config() + return len(sm.get_audit_events()) + finally: + sm.close() + + # 1s before the last check: still explains (truncation grace) -> no host-edit row. + assert await _run_with_commit_ts("2026-07-20T12:00:04Z") == 0 + # 2s before: outside the grace, correctly NOT explained -> host-edit recorded. + assert await _run_with_commit_ts("2026-07-20T12:00:03Z") == 1 + + +class TestReadHostConfig: + """#530 review MEDIUM: the mounted copy is pre-masked host-side, but _read_host_config + re-applies the SECRET_PATHS mask (like control_service.read_config) so a host masking + regression can't leave a raw secret resident in the long-lived config snapshot.""" + + def test_secret_value_is_remasked(self, tmp_path, monkeypatch): + cfg = tmp_path / "config.json" + cfg.write_text(json.dumps({"dashboard": {"auth": {"password": "leaked"}}})) + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(cfg)) + out = _read_host_config() + assert out["dashboard"]["auth"]["password"] == {"__secret__": True} + + def test_missing_or_bad_file_is_none(self, tmp_path, monkeypatch): + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", "/nonexistent/config.json") + assert _read_host_config() is None + bad = tmp_path / "config.json" + bad.write_text("{not json") + monkeypatch.setattr(ds_mod.config, "HOST_CONFIG_PATH", str(bad)) + assert _read_host_config() is None + + +class TestMirrorControlAudit: + """#530: opportunistically copies the #33 log's recent entries into the durable + ``audit_events`` table so the Security panel can group deeper than the log's own trimmed + tail. ``audit_service.recent_changes()`` output is already sanitized — nothing new to clean + here, only to persist.""" + + def _svc(self): + from mining_dashboard.service.storage_service import StateManager + + sm = StateManager(db_path=":memory:") + svc = DataService(sm, MagicMock(), MagicMock()) + return svc, sm + + def _log_line(self, **over): + entry = { + "ts": "2026-07-10T12:00:00Z", + "id": "11111111-1111-4111-8111-111111111111", + "actor": "admin", + "action": "commit", + "status": "applied", + "keys": "XVB_ENABLED", + } + entry.update(over) + return json.dumps(entry) + + async def test_disabled_is_a_noop(self, monkeypatch): + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", False) + svc, sm = self._svc() + try: + await svc._mirror_control_audit() + assert sm.get_audit_events() == [] + finally: + sm.close() + + async def test_mirrors_log_entries(self, tmp_path, monkeypatch): + log = tmp_path / "control.log" + log.write_text(self._log_line() + "\n") + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + svc, sm = self._svc() + try: + await svc._mirror_control_audit() + events = sm.get_audit_events() + assert len(events) == 1 + assert events[0]["source"] == "control" + assert events[0]["actor"] == "admin" + assert events[0]["keys"] == "XVB_ENABLED" + finally: + sm.close() + + async def test_re_mirroring_is_idempotent(self, tmp_path, monkeypatch): + log = tmp_path / "control.log" + log.write_text(self._log_line() + "\n") + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + svc, sm = self._svc() + try: + await svc._mirror_control_audit() + await svc._mirror_control_audit() + assert len(sm.get_audit_events()) == 1 + finally: + sm.close() + + async def test_entries_without_an_id_are_skipped(self, tmp_path, monkeypatch): + log = tmp_path / "control.log" + log.write_text(self._log_line(id="") + "\n") + monkeypatch.setattr(ds_mod.config, "DASHBOARD_CONTROL_ENABLED", True) + monkeypatch.setattr(ds_mod.audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + svc, sm = self._svc() + try: + await svc._mirror_control_audit() + assert sm.get_audit_events() == [] + finally: + sm.close() diff --git a/build/dashboard/tests/service/test_storage_service.py b/build/dashboard/tests/service/test_storage_service.py index e171cc65..7b2fcb7d 100644 --- a/build/dashboard/tests/service/test_storage_service.py +++ b/build/dashboard/tests/service/test_storage_service.py @@ -1315,3 +1315,134 @@ def test_write_error_flags_db_unhealthy(self, state_manager): def test_reads_and_writes_after_close_are_safe(self, state_manager): state_manager.close() state_manager.reconcile_worker_config_status("cid-1", "rolled_back") # must not raise + + +class TestWorkerConfigChangeKnown: + """#530: whether a change_id was ever spooled by THIS dashboard — the rig-edit detector's + only question. Only ``add_worker_config_version`` (the dashboard's own worker-apply write) + ever populates ``worker_config``, so an unknown change_id means the RIG applied it.""" + + def test_known_change_id_is_true(self, state_manager): + state_manager.add_worker_config_version("rig1", "cid-1", "accepted", {}, None) + assert state_manager.worker_config_change_known("cid-1") is True + + def test_unknown_change_id_is_false(self, state_manager): + assert state_manager.worker_config_change_known("no-such-id") is False + + def test_empty_change_id_is_false(self, state_manager): + assert state_manager.worker_config_change_known("") is False + assert state_manager.worker_config_change_known(None) is False + + def test_after_close_is_false(self, state_manager): + state_manager.close() + assert state_manager.worker_config_change_known("cid-1") is False + + def test_lookup_error_fails_open_true(self, state_manager): + # A DB hiccup during the lookup must not manufacture a false rig-edit report — fail toward + # treating the change_id as known (quiet), not toward flagging it. + with state_manager._db_lock: + state_manager._conn.execute("DROP TABLE worker_config") + assert state_manager.worker_config_change_known("cid-1") is True + + +class TestAuditEvents: + """#530: the durable audit_events table backing the Security panel — mirrored control.log + rows plus the out-of-band host-edit/rig-edit detections.""" + + def test_add_and_get_round_trips(self, state_manager): + state_manager.add_audit_event( + id="ev-1", + ts="2026-07-20T12:00:00Z", + source="host-edit", + actor="", + action="host-edit", + status="detected", + keys="xvb.enabled", + ) + events = state_manager.get_audit_events() + assert len(events) == 1 + assert events[0]["id"] == "ev-1" + assert events[0]["source"] == "host-edit" + assert events[0]["keys"] == "xvb.enabled" + + def test_insert_or_ignore_is_idempotent_on_id(self, state_manager): + # Re-mirroring the same control.log row (or re-detecting the same out-of-band event) + # must not duplicate it. + for _ in range(3): + state_manager.add_audit_event( + id="dup-1", + ts="2026-07-20T12:00:00Z", + source="control", + actor="admin", + action="commit", + status="applied", + keys="XVB_ENABLED", + ) + assert len(state_manager.get_audit_events()) == 1 + + def test_newest_first_by_ts(self, state_manager): + state_manager.add_audit_event( + id="a", + ts="2026-07-01T00:00:00Z", + source="control", + actor="", + action="commit", + status="applied", + keys="", + ) + state_manager.add_audit_event( + id="b", + ts="2026-07-20T00:00:00Z", + source="host-edit", + actor="", + action="host-edit", + status="detected", + keys="", + ) + events = state_manager.get_audit_events() + assert [e["id"] for e in events] == ["b", "a"] + + def test_limit_applies(self, state_manager): + for i in range(5): + state_manager.add_audit_event( + id=f"ev-{i}", + ts=f"2026-07-{i + 1:02d}T00:00:00Z", + source="control", + actor="", + action="commit", + status="applied", + keys="", + ) + assert len(state_manager.get_audit_events(limit=2)) == 2 + + def test_write_error_flags_db_unhealthy(self, state_manager): + with state_manager._db_lock: + state_manager._conn.execute("DROP TABLE audit_events") + state_manager.add_audit_event( + id="x", + ts="2026-07-20T00:00:00Z", + source="control", + actor="", + action="commit", + status="applied", + keys="", + ) + assert state_manager.is_db_healthy() is False + + def test_reads_and_writes_after_close_are_safe(self, state_manager): + state_manager.close() + state_manager.add_audit_event( + id="x", + ts="2026-07-20T00:00:00Z", + source="control", + actor="", + action="commit", + status="applied", + keys="", + ) # must not raise + assert state_manager.get_audit_events() == [] + + def test_read_error_returns_empty_list(self, state_manager): + with state_manager._db_lock: + state_manager._conn.execute("DROP TABLE audit_events") + assert state_manager.get_audit_events() == [] diff --git a/build/dashboard/tests/web/test_server.py b/build/dashboard/tests/web/test_server.py index 0d11747c..b4f626d0 100644 --- a/build/dashboard/tests/web/test_server.py +++ b/build/dashboard/tests/web/test_server.py @@ -496,6 +496,143 @@ async def test_audit_route_failure_is_sanitized(self, control_client, monkeypatc assert resp.status == 500 assert "secret" not in json.dumps(await resp.json()) + async def test_audit_route_merges_db_only_entries(self, control_client, monkeypatch): + # #530: an out-of-band host-edit/rig-edit row lives only in audit_events, never in + # control.log — it must still appear in the served feed. + monkeypatch.setattr(audit_service.config, "CONTROL_AUDIT_LOG", "/nonexistent/control.log") + state_mgr = control_client.app["state_manager"] + state_mgr.add_audit_event( + id="hostedit-1", + ts="2026-07-20T12:00:00Z", + source="host-edit", + actor="", + action="host-edit", + status="detected", + keys="xvb.enabled", + ) + resp = await control_client.get("/api/audit") + assert resp.status == 200 + entries = (await resp.json())["entries"] + assert len(entries) == 1 + assert entries[0]["source"] == "host-edit" + assert entries[0]["keys"] == "xvb.enabled" + + async def test_audit_route_shows_a_fresh_commit_before_it_is_mirrored( + self, control_client, tmp_path, monkeypatch + ): + # A commit that just landed in control.log, before the next poll cycle mirrors it to the + # DB, must still show up immediately — no regression from #530's DB merge. + log = tmp_path / "control.log" + log.write_text( + json.dumps( + { + "ts": "2026-07-20T12:00:00Z", + "id": "11111111-1111-4111-8111-111111111111", + "actor": "admin", + "action": "commit", + "status": "applied", + "keys": "XVB_ENABLED", + } + ) + + "\n" + ) + monkeypatch.setattr(audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + resp = await control_client.get("/api/audit") + entries = (await resp.json())["entries"] + assert len(entries) == 1 + assert entries[0]["keys"] == "XVB_ENABLED" + + async def test_audit_route_deduplicates_a_mirrored_row( + self, control_client, tmp_path, monkeypatch + ): + # The same control.log row, present both live (log tail) and mirrored (DB) — one row out, + # not two. + log = tmp_path / "control.log" + log.write_text( + json.dumps( + { + "ts": "2026-07-20T12:00:00Z", + "id": "22222222-2222-4222-8222-222222222222", + "actor": "admin", + "action": "commit", + "status": "applied", + "keys": "XVB_ENABLED", + } + ) + + "\n" + ) + monkeypatch.setattr(audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + state_mgr = control_client.app["state_manager"] + state_mgr.add_audit_event( + id="22222222-2222-4222-8222-222222222222", + ts="2026-07-20T12:00:00Z", + source="control", + actor="admin", + action="commit", + status="applied", + keys="XVB_ENABLED", + ) + resp = await control_client.get("/api/audit") + entries = (await resp.json())["entries"] + assert len(entries) == 1 + + async def test_audit_route_sorts_newest_first_across_sources( + self, control_client, tmp_path, monkeypatch + ): + log = tmp_path / "control.log" + log.write_text( + json.dumps( + { + "ts": "2026-07-01T00:00:00Z", + "id": "33333333-3333-4333-8333-333333333333", + "actor": "admin", + "action": "commit", + "status": "applied", + "keys": "XVB_ENABLED", + } + ) + + "\n" + ) + monkeypatch.setattr(audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + state_mgr = control_client.app["state_manager"] + state_mgr.add_audit_event( + id="hostedit-2", + ts="2026-07-20T00:00:00Z", + source="host-edit", + actor="", + action="host-edit", + status="detected", + keys="xvb.enabled", + ) + resp = await control_client.get("/api/audit") + entries = (await resp.json())["entries"] + assert [e["id"] for e in entries] == ["hostedit-2", "33333333-3333-4333-8333-333333333333"] + + async def test_audit_route_shows_a_no_id_log_row_live( + self, control_client, tmp_path, monkeypatch + ): + # A pre-auth "invalid"/"refused" control.log row (#33) has no id — never mirrored to the + # DB, but still shown live from the log tail. + log = tmp_path / "control.log" + log.write_text( + json.dumps( + { + "ts": "2026-07-20T12:00:00Z", + "id": "", + "actor": "", + "action": "invalid", + "status": "refused-oversize", + "keys": "", + } + ) + + "\n" + ) + monkeypatch.setattr(audit_service.config, "CONTROL_AUDIT_LOG", str(log)) + resp = await control_client.get("/api/audit") + entries = (await resp.json())["entries"] + assert len(entries) == 1 + assert entries[0]["status"] == "refused-oversize" + class TestSecurityHeaders: async def test_security_headers_present(self, client): diff --git a/docs/dashboard.md b/docs/dashboard.md index e3175673..b1738d4d 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -691,6 +691,31 @@ stripped to a safe character set before it is served. See [Operations › Watching for intruders](operations.md#watching-for-intruders) for the log paths, size bounds, and rotation steps. +### Catching changes made outside the dashboard + +The audit trail above only sees requests that went through the control channel. Two things can +change the stack without it: a hand-edit (or a `pithead apply` run from the host CLI) to +`config.json`, and a config change applied directly to a rig's own control API instead of through +Worker Inspect ([#530](https://github.com/p2pool-starter-stack/pithead/issues/530)). The dashboard +watches for both on its normal poll cycle and appends them to the SAME audit trail: + +- **`host-edit`** — `config.json` changed since the last poll and no control-channel commit + explains it. The row names the changed setting paths (e.g. `xvb.donation_level`); it never + records a value. +- **`rig-edit`** — a worker's control API reports a config change this dashboard never sent. The + row names the worker and the rig's own change id; RigForge's status feed reports only the + outcome of a change, not a per-key diff, so unlike `host-edit` this can't name which setting + moved — inspect the rig directly to see what changed. + +Either kind is worth treating like a rotate-now signal in the same spirit as +[Operations › Watching for intruders](operations.md#watching-for-intruders): if you didn't make +the change, someone or something with host or rig access did. + +The audit trail is no longer only a log tail: entries — both mirrored from `control.log` and the +two out-of-band kinds above — persist to the dashboard's own database, so the card's grouping +selector (hour/day/month) can drill back further than the log's own trimmed window. Pick "All" for +the flat newest-first view, or a coarser grouping to scan a longer history at a glance. + ## Upgrading from the dashboard With `dashboard.control.enabled: true` (the same flag as the Configuration view) and a newer diff --git a/docs/dev/testing-strategy.md b/docs/dev/testing-strategy.md index b4c738c9..e1a520db 100644 --- a/docs/dev/testing-strategy.md +++ b/docs/dev/testing-strategy.md @@ -116,6 +116,7 @@ The deploy-time axes — each changes a real runtime path. Full table and assert | `doctor` runtime verdicts (#383): egress firewall, stratum listening, dashboard answers | real box | 1 ✅ (stubbed toolchain) · 4 ▶ (`--check`) | | Control channel (#33): `apply --dry-run` preview, runner claim/validate/commit, fail-closed flag, rw/ro spool mounts | spool files / sourced fns | 1 ✅ (shell + pytest + compose) · 4 (systemd path unit on a real box — not yet a matrix row) | | Audit + access logs (#349): key-names-not-values audit entries, bounded log growth, Caddyfile log block, hostile log content served inert | spool/log fixtures | 1 ✅ (shell + pytest + node) · 4 (real Caddy writes over Tor — covered by the same onion matrix row) | +| Out-of-band audit detection + persistence (#530): a `config.json` change with no matching commit (`host-edit`) or a rig reporting a change_id the dashboard never spooled (`rig-edit`) both append to the durable `audit_events` table (mirrored `control.log` rows + these two kinds); Security panel hour/day/month grouping | poll-loop diff / real StateManager | 1 ✅ (`test_data_service.py::TestWatchHostConfig`/`TestRigEditDetection`/`TestMirrorControlAudit`, `test_storage_service.py::TestAuditEvents`, `securityview.test.mjs` grouping) · 4 (deferred — the underlying rig-side-edit-visible-in-the-enriched-feed mechanism is already proven live by the #516 row below; a real box producing a `host-edit`/`rig-edit` audit row end-to-end is not yet its own matrix leg) | ### H. Host / infrastructure (real-only) diff --git a/docs/operations.md b/docs/operations.md index 2672e6df..015fcce0 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -194,6 +194,19 @@ change staged through the dashboard and who committed it. A change you didn't ma rotate-now signal. Secret rotation beyond the dashboard password is tracked in [#378](https://github.com/p2pool-starter-stack/pithead/issues/378). +`control.log` only sees requests that went through the dashboard. The dashboard's own poll loop +additionally watches for two changes it did NOT make and appends them to the same audit trail +([#530](https://github.com/p2pool-starter-stack/pithead/issues/530), +[details](dashboard.md#catching-changes-made-outside-the-dashboard)): a `config.json` edit with no +matching commit (`host-edit`), and a worker's control API reporting a change the dashboard never +sent (`rig-edit`). Both name only what changed — settings or a worker, never a secret value — and +both are the same rotate-now signal as an unexplained `control.log` entry. + +Unlike `control.log`, which the writer trims to bound its size, the audit trail served by the +dashboard persists to its own database — mirrored `control.log` rows plus the two out-of-band +kinds above — so the Security panel's hour/day/month grouping can look back further than the log's +own trimmed window. + --- ## Updating the stack From 028e932383baef459b21c63b6ccf110f9e24d974 Mon Sep 17 00:00:00 2001 From: Vijit Singh Date: Tue, 21 Jul 2026 10:51:43 -0500 Subject: [PATCH 03/13] feat(#490): opt-in fail-closed miner hold on unrecoverable dashboard health failures (#708) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(#490): opt-in fail-closed miner hold on unrecoverable dashboard health failures Adds dashboard.fail_closed (default false). Off, an unrecoverable failure only alerts (existing #131/#337 paths) and mining continues, since the dashboard is an observability layer independent of the xmrig-proxy -> p2pool -> monerod datapath. On, it reuses the #35 sync gate's own hold (stop/start SYNC_GATE_CONTAINERS) rather than a new path, but — unlike that one-way latch — re-checks every cycle and releases once the condition clears. Scoped narrowly to genuine, non-transient failures: the DB auto-heal (#489) rebuild itself failing (StateManager.is_db_unrecoverable, distinct from the existing db_healthy flag, which also covers an ordinary transient write error that must never gate), or the dashboard container crash-looping (ContainerHealthMonitor.is_bad("dashboard")). XvB/payout state-integrity loss already safes to plain p2pool via algo_service's existing fail_count>=3 guard and isn't duplicated here. Registered in config.reference.json and both drift guards (pithead's CONTROL_DASHBOARD_EDITABLE_KEYS, control_service.py's EDITABLE_ENV_KEY_PATHS) so it round-trips the #33 control-approval path the same as dashboard.tari_required. Co-Authored-By: Claude Opus 4.8 * refactor: share the SYNC_GATE_CONTAINERS stop/start loop between the sync gate and the new fail-closed gate ponytail-review finding: _apply_sync_gate and _apply_fail_closed_gate duplicated the same stop-every-container / start-every-container-tracking-ok loops. Extract _stop_gate_containers(quiet) / _start_gate_containers() and use them from both, so the two holds over the same container set can't drift out of step. Co-Authored-By: Claude Opus 4.8 * fix(#490): gate fail-closed on a debounce-CONFIRMED bad verdict, not a first-sighting seed Coordinator finding 1: is_bad("dashboard") read the raw level, which ContainerHealthMonitor.update() seeds to "bad" on a first-sighting unhealthy snapshot WITHOUT clearing the 120s continuous-unhealthy debounce a known container must pass. That seed is intentionally silent for alerting, but the fail-closed gate read it as a live truth, so a dashboard reported unhealthy at first sighting could hold the fleet on an unconfirmed, possibly-transient state — contradicting the "only confirmed, non-transient failures gate" intent. Fix: rename is_bad -> is_confirmed_bad and require c["alerted"] (True only when the debounce produced an alert edge — crash loop or unhealthy past the threshold), so a silently-seeded baseline never gates. One-line predicate change; the alerted flag already existed. Finding 2: added classification-layer tests pinning transient-never-gates — seed not confirmed, single/early unhealthy not confirmed before 120s, confirmed only after the debounce (test_container_health); the disabled-sinks alert path still feeds the tracker and confirms through the 120s debounce (test_alert_service); and the real trigger expression (is_db_unrecoverable OR is_confirmed_bad) built from real objects does not gate on a seed (test_data_service). The is_db_unrecoverable-only-after-failed-rebuild and transient-write-error-does-not vs. gate cases were already covered in test_storage_service. Co-Authored-By: Claude Opus 4.8 * docs(#490): add CHANGELOG entry + fix testing-strategy method name - CHANGELOG: open the [Unreleased] section (consumed into [1.10.0] at the last release-prep) with the dashboard.fail_closed Added entry, matching the per-feature-PR convention #596/#597 followed. - testing-strategy.md: the shipped accessor is is_confirmed_bad, not is_bad; also mark tier-3 deferred (· 3 ▶) to match the sibling docker-control stop/start rows and the PR's own deferred-to-gouda note. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- CHANGELOG.md | 11 ++ .../mining_dashboard/config/config.py | 9 ++ .../mining_dashboard/service/alert_service.py | 10 ++ .../service/container_health.py | 11 ++ .../service/control_service.py | 1 + .../mining_dashboard/service/data_service.py | 93 +++++++++++- .../service/storage_service.py | 19 ++- .../web/static/configlogic.mjs | 1 + build/dashboard/mining_dashboard/web/views.py | 11 ++ build/dashboard/tests/config/test_config.py | 13 ++ .../tests/service/test_alert_service.py | 61 ++++++++ .../tests/service/test_container_health.py | 52 +++++++ .../tests/service/test_data_service.py | 142 ++++++++++++++++++ .../tests/service/test_storage_service.py | 31 ++++ build/dashboard/tests/web/test_views.py | 10 ++ config.reference.json | 1 + docker-compose.yml | 6 + docs/architecture.md | 2 +- docs/configuration.md | 1 + docs/dashboard.md | 10 ++ docs/dev/testing-strategy.md | 1 + pithead | 19 ++- tests/stack/run.sh | 15 ++ 23 files changed, 519 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eddf014c..98fe2539 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,17 @@ per the process in [`docs/dev/releasing.md`](docs/dev/releasing.md). ### Added +- **Opt-in fail-closed miner hold on an unrecoverable dashboard health failure** (#490). New + `dashboard.fail_closed` toggle, default **off**. The dashboard is an observability layer, not the + mining datapath (`xmrig-proxy` → `p2pool` → `monerod` runs independently of it), so by default an + unhealthy condition only alerts (Telegram/Healthchecks/webhook) and shows a badge while mining + continues. Set it `true` and a genuinely unrecoverable failure — the SQLite database failing to + rebuild after its own auto-heal attempt (disk full, permissions), or the `dashboard` container + itself crash-looping past the #337 debounce — holds `p2pool` and `xmrig-proxy` using the same + #35 sync-gate stop/start mechanism, with a `Miner held (fail-closed)` badge. Unlike the sync + gate's one-way latch it re-checks every cycle and releases on its own once the condition clears. + A transient write blip, a slow query, or a single failed external fetch never trips it — those + still only alert. Gated by the #33 control-approval path like other `dashboard.*` toggles. - **`/api/state` exposure for three of the #196 telemetry-backbone series (Tier-1).** The backbone PR (#600) added five persisted SQLite tables with capture, storage, and retention, but shipped without surfacing them to the client. This slice exposes three — `blocks` (pool block-found diff --git a/build/dashboard/mining_dashboard/config/config.py b/build/dashboard/mining_dashboard/config/config.py index 1408024b..6b2c9ed4 100644 --- a/build/dashboard/mining_dashboard/config/config.py +++ b/build/dashboard/mining_dashboard/config/config.py @@ -429,6 +429,15 @@ def _nonneg(v): if c.strip() ] +# Opt-in fail-closed miner hold on an UNRECOVERABLE health failure (#490), `dashboard.fail_closed`, +# default false. The dashboard is an observability layer, not the mining datapath (xmrig-proxy -> +# p2pool -> monerod is independent of it), so the default posture is alert-only: a cosmetic +# dashboard fault must never idle the fleet. `true` reuses the #35 sync-gate's own mechanism +# (stop/start SYNC_GATE_CONTAINERS) to hold the miner until the failure clears — see +# DataService._apply_fail_closed_gate for what counts as "unrecoverable" (narrowly: DB recovery +# itself failing, or the dashboard container crash-looping — never a transient blip). +DASHBOARD_FAIL_CLOSED = os.environ.get("DASHBOARD_FAIL_CLOSED", "false").strip().lower() == "true" + # Debounce: a node must be unreachable this long before it's declared DOWN, and reachable # this long before recovery — so a single transient timeout or a brief restart doesn't # kick every miner to their backups (and back) on a blip. diff --git a/build/dashboard/mining_dashboard/service/alert_service.py b/build/dashboard/mining_dashboard/service/alert_service.py index 9b100c5a..a40a68b1 100644 --- a/build/dashboard/mining_dashboard/service/alert_service.py +++ b/build/dashboard/mining_dashboard/service/alert_service.py @@ -765,6 +765,16 @@ async def process(self, **signals): self._wallet_edges(signals.get("observed_wallet", "")) except Exception as exc: # never let the tripwire break the data loop logger.debug("Wallet baseline update failed (%s)", type(exc).__name__) + # Keep the container-health debounce state current even with every sink off (#490): + # `dashboard.fail_closed` reads `self.containers.is_confirmed_bad("dashboard")` off + # the same tracker this alerting path would otherwise be the only feeder for. No + # alert fires here — just the state update the alerting branch below does anyway. + containers = signals.get("containers") + if containers is not None: + try: + self.containers.update(containers, now=signals.get("now")) + except Exception as exc: # never let a tracker bug break the data loop + logger.debug("Container-health update failed (%s)", type(exc).__name__) return [] try: alerts = self.evaluate(**signals) diff --git a/build/dashboard/mining_dashboard/service/container_health.py b/build/dashboard/mining_dashboard/service/container_health.py index deeca415..63a56f73 100644 --- a/build/dashboard/mining_dashboard/service/container_health.py +++ b/build/dashboard/mining_dashboard/service/container_health.py @@ -69,6 +69,17 @@ def __init__( # ok_since : when the current continuous-clean streak began self._containers = {} + def is_confirmed_bad(self, name): + """True only if `name`'s bad state was CONFIRMED by the debounce — a crash loop or a + continuous-unhealthy streak past ``unhealthy_after`` — i.e. it produced an alert edge + (``alerted``). A first-sighting silently-seeded baseline (already unhealthy/restarting at + the monitor's first look) is deliberately NOT confirmed: it skipped the debounce a KNOWN + container must pass, exactly as it skips the alert. `dashboard.fail_closed`'s miner hold + (#490) reads this rather than the raw level, so it holds the fleet only on a confirmed, + non-transient failure. Unknown/never-seen container reads as not bad.""" + c = self._containers.get(name) + return bool(c and c["state"] == "bad" and c["alerted"]) + def update(self, states, now=None): """Feed this cycle's ``{name: state}`` snapshot; return the debounced edges.""" now = self._clock() if now is None else now diff --git a/build/dashboard/mining_dashboard/service/control_service.py b/build/dashboard/mining_dashboard/service/control_service.py index b8c61e04..5b4f9f92 100644 --- a/build/dashboard/mining_dashboard/service/control_service.py +++ b/build/dashboard/mining_dashboard/service/control_service.py @@ -105,6 +105,7 @@ def _deep_merge(base, override): "XVB_ENABLED": ("xvb.enabled",), "XVB_DONATION_LEVEL": ("xvb.donation_level",), "TARI_REQUIRED": ("dashboard.tari_required",), + "DASHBOARD_FAIL_CLOSED": ("dashboard.fail_closed",), "DASHBOARD_CHECK_UPDATES": ("dashboard.check_for_updates",), "DASHBOARD_TZ": ("dashboard.timezone",), "MONERO_MEM_LIMIT": ("monero.mem_limit",), diff --git a/build/dashboard/mining_dashboard/service/data_service.py b/build/dashboard/mining_dashboard/service/data_service.py index 5c752814..37023194 100644 --- a/build/dashboard/mining_dashboard/service/data_service.py +++ b/build/dashboard/mining_dashboard/service/data_service.py @@ -43,6 +43,7 @@ CHECK_FOR_UPDATES, CLEARNET_STATE_DIR, DASHBOARD_ENERGY, + DASHBOARD_FAIL_CLOSED, ENABLE_XVB, GITHUB_RELEASES_API, GITHUB_RIGFORGE_RELEASES_API, @@ -574,6 +575,7 @@ def __init__(self, state_manager, proxy_client, xvb_client): "workers_rejected": False, "miner_released": False, "miner_held": False, + "fail_closed_held": False, "timestamp": 0, } @@ -642,6 +644,11 @@ def __init__(self, state_manager, proxy_client, xvb_client): self.miner_released = False self.miner_held = False + # Opt-in fail-closed miner hold on an UNRECOVERABLE health failure (#490), dashboard. + # fail_closed, default false — see `_apply_fail_closed_gate`. Transient like `miner_held`, + # not persisted: a restart re-derives it from the current health signals. + self.fail_closed_held = False + # Restore persistent state from DB to prevent empty dashboard on service restart loaded_snapshot = self.state_manager.load_snapshot() if loaded_snapshot and isinstance(loaded_snapshot, dict): @@ -692,6 +699,19 @@ async def _apply_worker_rejection(self, monero_down, tari_down): if await self.docker_control.start(REJECT_WORKERS_CONTAINER): self.workers_rejected = False + async def _stop_gate_containers(self, quiet): + """Stop every ``SYNC_GATE_CONTAINERS`` container; shared by the #35 sync gate and the + #490 fail-closed gate, the two holds that stop the same container set.""" + for container in SYNC_GATE_CONTAINERS: + await self.docker_control.stop(container, quiet=quiet) + + async def _start_gate_containers(self): + """Start every ``SYNC_GATE_CONTAINERS`` container; True only if every start succeeded.""" + ok = True + for container in SYNC_GATE_CONTAINERS: + ok = (await self.docker_control.start(container)) and ok + return ok + async def _apply_sync_gate(self, gate_satisfied): """ Hold p2pool + xmrig-proxy stopped until the required chain(s) have fully synced once, @@ -716,10 +736,7 @@ async def _apply_sync_gate(self, gate_satisfied): return if gate_satisfied: - ok = True - for container in SYNC_GATE_CONTAINERS: - ok = (await self.docker_control.start(container)) and ok - if ok: + if await self._start_gate_containers(): self.miner_released = True self.miner_held = False logger.info( @@ -730,8 +747,7 @@ async def _apply_sync_gate(self, gate_satisfied): # Still syncing: keep the miner held. Log the human-facing notice only on the first # cycle of a hold; the per-cycle re-assert stops are quiet to avoid flooding the log. - for container in SYNC_GATE_CONTAINERS: - await self.docker_control.stop(container, quiet=self.miner_held) + await self._stop_gate_containers(quiet=self.miner_held) if not self.miner_held: self.miner_held = True logger.info( @@ -739,6 +755,53 @@ async def _apply_sync_gate(self, gate_satisfied): f"until synced." ) + async def _apply_fail_closed_gate(self, unrecoverable): + """ + Opt-in (`dashboard.fail_closed`, default False) miner hold on an UNRECOVERABLE health + failure (#490) — reuses the #35 sync gate's own mechanism (stop/start + ``SYNC_GATE_CONTAINERS`` through ``docker_control``) rather than a new hold path. + + "Unrecoverable" is scoped narrowly by the caller to genuine, non-transient failures: a DB + whose auto-heal rebuild itself failed (``StateManager.is_db_unrecoverable``), or the + dashboard container itself crash-looping / stuck unhealthy past the #337 debounce + (``AlertService.containers.is_confirmed_bad("dashboard")`` — a debounce-CONFIRMED verdict, + never a first-sighting seed). A transient write blip, a slow query, a single failed + external fetch, or a container merely reported unhealthy on one poll is never + "unrecoverable" — those already alert (#131/#337) and must never gate; a false positive + here idles the fleet and costs revenue. + + Unlike the sync gate's one-way latch, this re-checks every cycle and releases once + ``unrecoverable`` clears — the failures it watches (disk full, a crash-looping container) + are the kind an operator fixes without a full stack restart, and the miner should resume + on its own once they do. Only engages once the sync gate has actually released the miner; + holding before that is already #35's job. + + Default False is alert-only: `dashboard.fail_closed` off means these same signals keep + alerting (unchanged) but this method is a no-op, so a cosmetic dashboard fault never idles + the fleet — the mining datapath (xmrig-proxy -> p2pool -> monerod) is independent of the + dashboard by design. + """ + if not DASHBOARD_FAIL_CLOSED or not self.miner_released: + return + + if unrecoverable: + await self._stop_gate_containers(quiet=self.fail_closed_held) + if not self.fail_closed_held: + self.fail_closed_held = True + logger.error( + f"Unrecoverable health failure with dashboard.fail_closed enabled — holding " + f"{', '.join(SYNC_GATE_CONTAINERS)} until it clears." + ) + return + + if self.fail_closed_held and await self._start_gate_containers(): + self.fail_closed_held = False + logger.info( + f"Unrecoverable health failure cleared — starting " + f"{', '.join(SYNC_GATE_CONTAINERS)}; mining can resume." + ) + # On a partial-start failure stay held so the next cycle retries. + async def _sync_xvb_stats(self): """ Fetch XvB's reported averages (avg_1h/avg_24h/fail_count) over Tor and persist them. @@ -1365,9 +1428,14 @@ async def run(self): ) # Per-container restart/health snapshot for the crash-loop/unhealthy alert # (#337) — 9 inspect calls against the read-only docker-proxy, skipped - # entirely while Telegram is off (same cost discipline as alert_metrics). + # entirely while Telegram is off AND dashboard.fail_closed is off (same cost + # discipline as alert_metrics). fail_closed needs it even with Telegram off: + # it's the only source for "is the dashboard container itself crash-looping" + # (#490). container_states = ( - await get_container_health() if self.alert_service.enabled else {} + await get_container_health() + if (self.alert_service.enabled or DASHBOARD_FAIL_CLOSED) + else {} ) await self.alert_service.process( monero_down=monero_down, @@ -1418,6 +1486,14 @@ async def run(self): # the read-only docker-proxy. containers=container_states, ) + # 5b. Fail-closed miner hold (#490), opt-in via dashboard.fail_closed. Reads + # the DB auto-heal outcome and the dashboard's OWN crash-loop state — both + # narrow, non-transient "unrecoverable" signals — off the trackers `process` + # above just fed (see `_apply_fail_closed_gate` for what counts and why). + await self._apply_fail_closed_gate( + self.state_manager.is_db_unrecoverable() + or self.alert_service.containers.is_confirmed_bad("dashboard") + ) # Once-daily status digest, reusing the metrics built above (only when the bot # is on, which is also the only time maybe_daily_summary would send). await self.alert_service.maybe_daily_summary( @@ -1466,6 +1542,7 @@ async def run(self): "workers_rejected": self.workers_rejected, "miner_released": self.miner_released, "miner_held": self.miner_held, + "fail_closed_held": self.fail_closed_held, "clearnet_sync": self.clearnet_sync_state, "system": { "disk": disk_usage, diff --git a/build/dashboard/mining_dashboard/service/storage_service.py b/build/dashboard/mining_dashboard/service/storage_service.py index 2f1f2b79..c2b1a1ee 100644 --- a/build/dashboard/mining_dashboard/service/storage_service.py +++ b/build/dashboard/mining_dashboard/service/storage_service.py @@ -117,6 +117,14 @@ def __init__(self, db_path: str = None): None # {"ts", "reason", "quarantine"} of the most recent reset, for the alert ) + # True only when the auto-heal RECOVERY ITSELF just failed (disk full, permissions) — + # distinct from ``db_healthy``, which also flips false on an ordinary transient write + # error (a locked DB, a momentary I/O hiccup) that must never be treated as unrecoverable. + # This is the narrow signal `dashboard.fail_closed` (#490) gates on: a DB that a corruption + # was DETECTED for and whose rebuild then failed, not merely "a write failed once". Cleared + # on the next recovery attempt that succeeds. + self.db_unrecoverable = False + # Per-table "last successful write" health signal for the v1.7 telemetry backbone (#196 # Wave-0), mirroring db_healthy above but per table: DataService's whole poll loop is one # big try/except, so a capture hook that starts silently raising would otherwise stop @@ -219,6 +227,7 @@ def _recover_corrupt_db(self, reason: str): self._conn.row_factory = sqlite3.Row self._apply_schema() self.db_healthy = True + self.db_unrecoverable = False # a later successful attempt clears an earlier failure self.db_reset_count += 1 self.last_db_reset = {"ts": time.time(), "reason": reason, "quarantine": quarantine} self.logger.error( @@ -228,7 +237,9 @@ def _recover_corrupt_db(self, reason: str): quarantine or "(in-memory, nothing to quarantine)", ) except (sqlite3.Error, OSError) as e: - # Recovery itself failed (disk full, permissions) — leave persistence flagged unhealthy. + # Recovery itself failed (disk full, permissions) — this is the unrecoverable case + # #490's fail-closed gate watches for, distinct from an ordinary transient write error. + self.db_unrecoverable = True self._db_error("DB Recovery Error", e) def _prune_quarantined(self): @@ -260,6 +271,12 @@ def is_db_healthy(self) -> bool: """True unless a DB init or write has failed — drives the dashboard persistence badge (#131).""" return self.db_healthy + def is_db_unrecoverable(self) -> bool: + """True only when the auto-heal rebuild itself just failed (#489/#490) — narrower than + ``is_db_healthy() is False``, which also covers an ordinary transient write error. Feeds + `dashboard.fail_closed`'s miner hold; a transient blip must never trip it.""" + return self.db_unrecoverable + def _create_tables(self): """Creates necessary tables if they don't exist.""" # Per-window hashrate columns (#168) are appended so a fresh DB starts with them; existing diff --git a/build/dashboard/mining_dashboard/web/static/configlogic.mjs b/build/dashboard/mining_dashboard/web/static/configlogic.mjs index de4707c9..52eeee85 100644 --- a/build/dashboard/mining_dashboard/web/static/configlogic.mjs +++ b/build/dashboard/mining_dashboard/web/static/configlogic.mjs @@ -128,6 +128,7 @@ export const LOGICAL_GROUPS = [ "proxy", "xvb", "dashboard.tari_required", + "dashboard.fail_closed", ], }, { name: "Workers", prefixes: ["workers"] }, diff --git a/build/dashboard/mining_dashboard/web/views.py b/build/dashboard/mining_dashboard/web/views.py index 64d3d03b..a6dbef22 100644 --- a/build/dashboard/mining_dashboard/web/views.py +++ b/build/dashboard/mining_dashboard/web/views.py @@ -1295,6 +1295,17 @@ def build_badges(data, metrics, mode_variant, db_healthy=True, wallet_change=Non "title": "p2pool and xmrig-proxy are held until the required chains finish syncing", } ) + # Fail-closed miner hold on an unrecoverable health failure (Issue #490), opt-in via + # dashboard.fail_closed. Distinct from the sync-gate badge above — this fires post-sync. + if data.get("fail_closed_held"): + badges.append( + { + "text": "Miner held (fail-closed)", + "variant": "bad", + "title": "dashboard.fail_closed is on and an unrecoverable health failure is " + "holding p2pool and xmrig-proxy until it clears", + } + ) # Non-blocking Tari (Issue #51): stay operational, surface a top-bar badge with the live # percentage once known (omitted early so it isn't a stale "0%"). if data.get("tari_syncing_passive"): diff --git a/build/dashboard/tests/config/test_config.py b/build/dashboard/tests/config/test_config.py index 34e59d8d..75aebdc0 100644 --- a/build/dashboard/tests/config/test_config.py +++ b/build/dashboard/tests/config/test_config.py @@ -41,6 +41,19 @@ def test_monero_prune_accepts_falsy_forms(self): with patch.dict(os.environ, {"MONERO_PRUNE": v}): assert _reload_config().MONERO_PRUNE is False, f"{v!r} should be full" + def test_fail_closed_defaults_off(self): + # #490: the dashboard is an observability layer, so a cosmetic fault must never idle the + # fleet by default — absent DASHBOARD_FAIL_CLOSED must read False. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DASHBOARD_FAIL_CLOSED", None) + assert _reload_config().DASHBOARD_FAIL_CLOSED is False + + def test_fail_closed_env_override(self): + with patch.dict(os.environ, {"DASHBOARD_FAIL_CLOSED": "true"}): + assert _reload_config().DASHBOARD_FAIL_CLOSED is True + with patch.dict(os.environ, {"DASHBOARD_FAIL_CLOSED": "false"}): + assert _reload_config().DASHBOARD_FAIL_CLOSED is False + def test_update_interval_tolerates_bad_values(self): # A malformed override must fall back to the default, not crash the dashboard at import. for v, expected in [("2", 2), ("2.5", 2), ("", 30), ("nonsense", 30)]: diff --git a/build/dashboard/tests/service/test_alert_service.py b/build/dashboard/tests/service/test_alert_service.py index 14625585..42fbc59a 100644 --- a/build/dashboard/tests/service/test_alert_service.py +++ b/build/dashboard/tests/service/test_alert_service.py @@ -672,6 +672,44 @@ async def test_disabled_notifier_still_persists_wallet_baseline(self): assert store["payout_wallet_prev8"] == _W_A[:8] assert float(store["payout_wallet_changed_ts"]) > 0 + async def test_disabled_notifier_still_updates_container_health_tracker(self): + # #490: dashboard.fail_closed reads svc.containers.is_confirmed_bad("dashboard") off the + # SAME tracker this alerting path would otherwise be the only feeder for — it must stay fed + # every cycle even with every sink off (the default stack), or a fail_closed operator with + # Telegram off would never see the dashboard crash/unhealthy signal used to gate the hold. + # `now` rides the signals through to containers.update, so a healthy→unhealthy streak past + # the 120s debounce confirms through the disabled path — proving it's genuinely fed (a + # never-fed tracker would stay not-confirmed). + svc = _svc(notifier=_FakeNotifier(enabled=False)) + base = dict( + monero_down=False, + tari_down=False, + tari_required=True, + miner_released=True, + workers=[], + workers_expected=False, + ) + + def snap(health, now): + return dict( + base, + now=now, + containers={ + "dashboard": { + "running": True, + "restarting": False, + "restart_count": 0, + "health": health, + } + }, + ) + + await svc.process(**snap("healthy", 1000)) # known, healthy baseline + await svc.process(**snap("unhealthy", 1000)) # streak starts + assert svc.containers.is_confirmed_bad("dashboard") is False # too young to gate + await svc.process(**snap("unhealthy", 1130)) # past 120s -> confirmed + assert svc.containers.is_confirmed_bad("dashboard") is True + async def test_disabled_path_swallows_wallet_baseline_error(self): # A broken kv store must not break the data loop, Telegram on or off. def boom(_k, _v=None): @@ -689,6 +727,29 @@ def boom(_k, _v=None): ) assert out == [] + async def test_disabled_path_swallows_container_health_update_error(self): + # Same "never break the data loop" contract as the wallet baseline above, for the #490 + # container-health feed. + svc = _svc(notifier=_FakeNotifier(enabled=False)) + svc.containers.update = lambda *_a, **_kw: (_ for _ in ()).throw(RuntimeError("boom")) + out = await svc.process( + monero_down=False, + tari_down=False, + tari_required=True, + miner_released=True, + workers=[], + workers_expected=False, + containers={ + "dashboard": { + "running": True, + "restarting": False, + "restart_count": 0, + "health": None, + } + }, + ) + assert out == [] + async def test_enabled_notifier_dispatches(self): notifier = _FakeNotifier() svc = _svc(notifier=notifier) diff --git a/build/dashboard/tests/service/test_container_health.py b/build/dashboard/tests/service/test_container_health.py index 33a59eaf..0cc20ab0 100644 --- a/build/dashboard/tests/service/test_container_health.py +++ b/build/dashboard/tests/service/test_container_health.py @@ -230,3 +230,55 @@ def test_stop_while_unhealthy_pending_resets_the_streak(self): clock.advance(120) # Back up and unhealthy again: the streak starts fresh, so no immediate edge. assert m.update({"monerod": _s(health="unhealthy")}) == [] + + +class TestIsConfirmedBad: + """Confirmed-level read (#490) — dashboard.fail_closed gates on a debounce-CONFIRMED bad + verdict, never on a first-sighting seed that skipped the debounce. A false positive here idles + the fleet, so the bar is the same 120s / crash-loop debounce a KNOWN container must pass.""" + + def test_unknown_container_is_not_confirmed_bad(self): + m, _clock = _monitor() + assert m.is_confirmed_bad("dashboard") is False + + def test_first_sighting_unhealthy_seed_is_not_confirmed(self): + # The seed sets state="bad" silently (no alert edge) — it must ALSO not read as confirmed, + # or fail_closed would hold the fleet on a container the debounce never vetted (#490 F1). + m, _clock = _monitor() + m.update({"dashboard": _s(health="unhealthy")}) + assert m.is_confirmed_bad("dashboard") is False + + def test_transient_unhealthy_on_known_container_not_confirmed_before_debounce(self): + # A KNOWN (healthy-first) container reported unhealthy on a single poll must NOT confirm + # until the streak clears unhealthy_after (120s) — the transient-never-gates property. + m, clock = _monitor() + m.update({"dashboard": _s(health="healthy")}) # known, healthy baseline + clock.advance(30) + assert m.update({"dashboard": _s(health="unhealthy")}) == [] + assert m.is_confirmed_bad("dashboard") is False # streak too young to gate + clock.advance(60) # 90s total < 120s + m.update({"dashboard": _s(health="unhealthy")}) + assert m.is_confirmed_bad("dashboard") is False + + def test_unhealthy_past_debounce_is_confirmed(self): + m, clock = _monitor() + m.update({"dashboard": _s(health="healthy")}) + m.update({"dashboard": _s(health="unhealthy")}) + clock.advance(121) # past unhealthy_after + assert m.update({"dashboard": _s(health="unhealthy")}) == [("dashboard", "unhealthy")] + assert m.is_confirmed_bad("dashboard") is True + + def test_crash_loop_is_confirmed_until_recovered(self): + m, clock = _monitor() + m.update({"dashboard": _s(restart_count=0)}) + clock.advance(60) + m.update({"dashboard": _s(restart_count=1)}) + clock.advance(60) + m.update({"dashboard": _s(restart_count=2)}) + clock.advance(60) + assert m.update({"dashboard": _s(restart_count=3)}) == [("dashboard", "crash_loop")] + assert m.is_confirmed_bad("dashboard") is True + # Clean streak past recovery_after clears it. + clock.advance(121) + m.update({"dashboard": _s(restart_count=3)}) + assert m.is_confirmed_bad("dashboard") is False diff --git a/build/dashboard/tests/service/test_data_service.py b/build/dashboard/tests/service/test_data_service.py index be74ec6e..7319f477 100644 --- a/build/dashboard/tests/service/test_data_service.py +++ b/build/dashboard/tests/service/test_data_service.py @@ -711,6 +711,148 @@ async def test_rehold_stops_quietly_after_first_cycle(self): assert second.kwargs.get("quiet") is True +class TestFailClosedGate: + """Opt-in (dashboard.fail_closed, #490) miner hold on an unrecoverable health failure — + reuses the #35 sync gate's own stop/start mechanism over SYNC_GATE_CONTAINERS, but (unlike + the sync gate) is not a one-way latch: it re-checks every cycle and releases once the + unrecoverable condition clears.""" + + def _svc(self, released=True): + sm = MagicMock() + sm.load_snapshot.return_value = None + svc = DataService(sm, MagicMock(), MagicMock()) + svc.docker_control = MagicMock() + svc.docker_control.stop = AsyncMock(return_value=True) + svc.docker_control.start = AsyncMock(return_value=True) + svc.miner_released = released + return svc + + def _enabled(self, on=True): + return patch.object(ds_mod, "DASHBOARD_FAIL_CLOSED", on) + + async def test_default_off_never_touches_containers(self): + # dashboard.fail_closed defaults False — an unrecoverable failure must only ever alert + # (elsewhere), never hold. A cosmetic dashboard fault must not idle the fleet. + svc = self._svc() + with ( + self._enabled(False), + patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool", "xmrig-proxy"]), + ): + await svc._apply_fail_closed_gate(unrecoverable=True) + svc.docker_control.stop.assert_not_called() + svc.docker_control.start.assert_not_called() + assert svc.fail_closed_held is False + + async def test_noop_before_sync_gate_releases_the_miner(self): + # Holding before the sync gate has released is already #35's job — engaging here too + # would just be a second, redundant hold path. + svc = self._svc(released=False) + with self._enabled(True), patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool"]): + await svc._apply_fail_closed_gate(unrecoverable=True) + svc.docker_control.stop.assert_not_called() + assert svc.fail_closed_held is False + + async def test_holds_when_enabled_and_unrecoverable(self): + svc = self._svc() + with ( + self._enabled(True), + patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool", "xmrig-proxy"]), + ): + await svc._apply_fail_closed_gate(unrecoverable=True) + stopped = {c.args[0] for c in svc.docker_control.stop.await_args_list} + assert stopped == {"p2pool", "xmrig-proxy"} + svc.docker_control.start.assert_not_called() + assert svc.fail_closed_held is True + + async def test_rehold_stops_quietly_after_first_cycle(self): + svc = self._svc() + with self._enabled(True), patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool"]): + await svc._apply_fail_closed_gate(unrecoverable=True) + await svc._apply_fail_closed_gate(unrecoverable=True) + first, second = svc.docker_control.stop.await_args_list + assert first.kwargs.get("quiet") is False + assert second.kwargs.get("quiet") is True + + async def test_releases_once_condition_clears(self): + # Unlike the sync gate's one-way latch, this must release on its own once the + # unrecoverable condition clears (an operator fix + restart, no full stack restart). + svc = self._svc() + svc.fail_closed_held = True + with ( + self._enabled(True), + patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool", "xmrig-proxy"]), + ): + await svc._apply_fail_closed_gate(unrecoverable=False) + started = {c.args[0] for c in svc.docker_control.start.await_args_list} + assert started == {"p2pool", "xmrig-proxy"} + svc.docker_control.stop.assert_not_called() + assert svc.fail_closed_held is False + + async def test_partial_start_failure_stays_held(self): + svc = self._svc() + svc.fail_closed_held = True + svc.docker_control.start = AsyncMock(side_effect=[True, False]) + with ( + self._enabled(True), + patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool", "xmrig-proxy"]), + ): + await svc._apply_fail_closed_gate(unrecoverable=False) + assert svc.fail_closed_held is True # next cycle retries + + async def test_not_a_one_way_latch_can_rehold_after_release(self): + # The defining difference from the #35 sync gate: a later unrecoverable condition (a + # second DB-recovery failure) must be able to hold again after an earlier release. + svc = self._svc() + with self._enabled(True), patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool"]): + await svc._apply_fail_closed_gate(unrecoverable=True) + assert svc.fail_closed_held is True + await svc._apply_fail_closed_gate(unrecoverable=False) + assert svc.fail_closed_held is False + await svc._apply_fail_closed_gate(unrecoverable=True) + assert svc.fail_closed_held is True + + async def test_healthy_and_never_held_is_a_noop(self): + svc = self._svc() + with self._enabled(True), patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool"]): + await svc._apply_fail_closed_gate(unrecoverable=False) + svc.docker_control.start.assert_not_called() + svc.docker_control.stop.assert_not_called() + + async def test_real_trigger_a_transient_unhealthy_dashboard_does_not_gate(self): + # #490 F1/F2: wire the ACTUAL trigger expression the run loop computes + # (is_db_unrecoverable OR containers.is_confirmed_bad("dashboard")) from real objects, and + # prove a first-sighting-unhealthy dashboard (a seed, unvetted by the 120s debounce) does + # NOT hold the fleet. The hardcoded-bool gate tests above cover the mechanism; this covers + # the classification that feeds it. + from mining_dashboard.service.storage_service import StateManager + + svc = self._svc() + sm = StateManager(db_path=":memory:") + try: + svc.state_manager = sm + svc.alert_service.containers.update( + { + "dashboard": { + "running": True, + "restarting": False, + "restart_count": 0, + "health": "unhealthy", + } + } + ) + assert sm.is_db_unrecoverable() is False # healthy DB + trigger = sm.is_db_unrecoverable() or svc.alert_service.containers.is_confirmed_bad( + "dashboard" + ) + assert trigger is False # seed is not confirmed -> no gate + with self._enabled(True), patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool"]): + await svc._apply_fail_closed_gate(trigger) + svc.docker_control.stop.assert_not_called() + assert svc.fail_closed_held is False + finally: + sm.close() + + class TestRunIteration: async def test_single_iteration_aggregates(self): svc, sm, proxy = _make_service() diff --git a/build/dashboard/tests/service/test_storage_service.py b/build/dashboard/tests/service/test_storage_service.py index 7b2fcb7d..5f2d158d 100644 --- a/build/dashboard/tests/service/test_storage_service.py +++ b/build/dashboard/tests/service/test_storage_service.py @@ -220,6 +220,37 @@ def test_transient_write_error_flags_unhealthy_without_reset(self, state_manager state_manager._db_error("History Update Error", sqlite3.OperationalError("disk I/O error")) assert state_manager.is_db_healthy() is False assert state_manager.db_reset_count == 0 # no reset for a transient error + # #490: a transient blip must never read as "unrecoverable" — that's the narrow signal + # dashboard.fail_closed gates the miner hold on, and this is exactly the false positive + # it must not trip on. + assert state_manager.is_db_unrecoverable() is False + + def test_recovery_failure_marks_unrecoverable(self, monkeypatch): + # #490: the auto-heal REBUILD itself failing (disk full, permissions) is the narrow + # "unrecoverable" signal — distinct from db_healthy, which also flips false on an + # ordinary transient write error that must never gate the miner. + sm = StateManager(db_path=":memory:") + try: + monkeypatch.setattr( + sm, "_apply_schema", MagicMock(side_effect=sqlite3.OperationalError("disk full")) + ) + sm._recover_corrupt_db("test: forced recovery failure") + assert sm.is_db_unrecoverable() is True + assert sm.is_db_healthy() is False + finally: + sm.close() + + def test_successful_recovery_clears_unrecoverable_flag(self): + # A later recovery attempt that succeeds (e.g. the disk freed up) clears an earlier + # failure — fail_closed's hold must release once the DB is actually healthy again. + sm = StateManager(db_path=":memory:") + try: + sm.db_unrecoverable = True # simulate a prior failed attempt + sm._recover_corrupt_db("test: retry succeeds") + assert sm.is_db_unrecoverable() is False + assert sm.is_db_healthy() is True + finally: + sm.close() def test_prune_keeps_only_recent_quarantines(self, tmp_path): import os diff --git a/build/dashboard/tests/web/test_views.py b/build/dashboard/tests/web/test_views.py index 33faccec..453c7bb2 100644 --- a/build/dashboard/tests/web/test_views.py +++ b/build/dashboard/tests/web/test_views.py @@ -620,6 +620,16 @@ def test_miner_held(self): out = build_badges({"miner_held": True}, _metrics(global_syncing=True), "ok") assert "Miner held (sync)" in self._texts(out) + def test_fail_closed_held(self): + # #490: distinct badge from the sync-gate hold above — fires post-sync, only with + # dashboard.fail_closed on. + out = build_badges({"fail_closed_held": True}, _metrics(), "ok") + assert any(b["variant"] == "bad" and "Miner held (fail-closed)" in b["text"] for b in out) + + def test_no_fail_closed_badge_by_default(self): + out = build_badges({}, _metrics(), "ok") + assert not any("fail-closed" in b["text"] for b in out) + def test_passive_tari_with_and_without_percent(self): with_pct = build_badges( {"tari_syncing_passive": True}, _metrics(tari=_sync(percent=42)), "ok" diff --git a/config.reference.json b/config.reference.json index 6379f07c..dfef17f3 100644 --- a/config.reference.json +++ b/config.reference.json @@ -75,6 +75,7 @@ "timezone": "auto", "data_dir": "auto", "tari_required": true, + "fail_closed": false, "check_for_updates": true, "hashrate_drop_threshold": 50, "hashrate_drop_minutes": 10, diff --git a/docker-compose.yml b/docker-compose.yml index fc464e19..9d03a22f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -669,6 +669,12 @@ services: # miner waits for Tari's sync, and whether a Tari-only sync takes over the dashboard # (Issue #51). Set dashboard.tari_required:false to make Tari non-blocking. - TARI_REQUIRED=${TARI_REQUIRED:-true} + # Opt-in fail-closed miner hold on an unrecoverable dashboard health failure (#490, + # config.json: dashboard.fail_closed). Default OFF: these failures only alert, mining + # continues. ON reuses the #35 sync-gate hold (stop/start p2pool + xmrig-proxy) but only for + # a DB auto-heal rebuild that itself failed, or the dashboard container crash-looping — + # never a transient blip. + - DASHBOARD_FAIL_CLOSED=${DASHBOARD_FAIL_CLOSED:-false} # Healthchecks.io dead-man's switch (#79). A ping URL is the on/off switch (blank = off); # when set, the dashboard pings it each loop, so if the host dies the pings stop and # Healthchecks.io alerts externally — the one failure mode an in-stack notifier can't diff --git a/docs/architecture.md b/docs/architecture.md index f90114d4..4d534fc0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,7 +18,7 @@ port forwarding is required. | 5 | **Tor** | Provides SOCKS5 proxies and hidden services (onion addresses) for the other containers. | | 6 | **Dashboard** | The web monitoring UI and the algorithmic switching engine. | | 7 | **Docker Proxy** | A **read-only** proxy onto the Docker socket so the dashboard can read container stats/logs — no write access. | -| 8 | **Docker Control** | A second, minimal socket proxy scoped to **only** `start`/`stop` (nothing else — not create/kill/exec/reads), so the dashboard can reject workers when a node is down (Issue #31), hold p2pool + xmrig-proxy until the chains finish syncing (Issue #35), and switch a clearnet-syncing node back to Tor once it's synced (Issue #234). Kept separate so its write grant can't widen the read-only proxy. | +| 8 | **Docker Control** | A second, minimal socket proxy scoped to **only** `start`/`stop` (nothing else — not create/kill/exec/reads), so the dashboard can reject workers when a node is down (Issue #31), hold p2pool + xmrig-proxy until the chains finish syncing (Issue #35), switch a clearnet-syncing node back to Tor once it's synced (Issue #234), and, opt-in via `dashboard.fail_closed`, hold p2pool + xmrig-proxy again on an unrecoverable dashboard health failure (Issue #490). Kept separate so its write grant can't widen the read-only proxy. | | 9 | **Caddy** | A reverse proxy that serves the dashboard over HTTPS (automatic local TLS) on the LAN. | ## High-level diagram diff --git a/docs/configuration.md b/docs/configuration.md index b5580950..c5135e97 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -143,6 +143,7 @@ control channel will commit, are unaffected either way. | `dashboard.hashrate_drop_threshold` | `50` | Percent below the recent normal that counts as a hashrate drop for the `hashrate_loss` alert and its chart marker. `50` = fire when total fleet hashrate falls to half its baseline. Raise it to catch smaller dips, lower it to only flag near-total outages. | | `dashboard.hashrate_drop_minutes` | `10` | How many minutes the hashrate must stay below the threshold before the drop is reported — the debounce that keeps a brief blip from pinging you. | | `dashboard.tari_required` | `true` | How much a Tari problem holds up the rest of the stack. Monero is required to mine, so its behavior isn't configurable: a monerod outage always rejects workers (stops `xmrig-proxy` so miners fail over to their backup pools), and the miner is always held until monerod finishes syncing. Tari is only needed for merge-mining, so this one flag decides how much it blocks. `true` (default): a Tari outage also rejects workers, the miner waits for Tari's initial sync too, and a Tari-only (re)sync shows the full-screen Sync view. `false` (non-blocking): keep mining Monero through a Tari outage, start mining as soon as Monero is synced (Tari finishes in the background), and keep the normal dashboard, with a `Tari syncing` indicator, instead of the takeover screen. | +| `dashboard.fail_closed` | `false` _(off)_ | Whether an **unrecoverable** dashboard health failure holds the miner, not just alerts (#490). The dashboard is an observability layer, not the mining datapath (`xmrig-proxy` → `p2pool` → `monerod` runs independently of it), so the default (`false`) is alert-only: a loud Telegram/Healthchecks alert plus a dashboard badge, and mining keeps running. `true` reuses the same hold the sync gate uses (stops `p2pool` and `xmrig-proxy`) the moment one of two narrow conditions hits, and starts them again once it clears — no restart needed: the dashboard's SQLite database failed to self-heal after its own auto-recovery attempt (disk full, permissions — see `db_reset` in [Dashboard › Node status & failover](dashboard.md#node-status--failover)), or the `dashboard` container itself is crash-looping. A transient write blip, a slow query, or a single failed external fetch never trips it — only these two genuinely non-transient failures do. Set it `true` only if a stale dashboard is a bigger risk to you than a held miner; a false positive here idles the fleet. | | `network.subnet` | `172.28.0.0/24` | The private Docker bridge the stack's containers run on. Change it only if install fails with `Pool overlaps with other one on this address space`, i.e. your host already uses `172.28.0.0/24` for another Docker network or interface. Must be a free `X.Y.Z.0/24` block (e.g. `"172.30.0.0/24"`); the services keep their fixed host octets (`.25`–`.31`) within it, so the structured addressing the dashboard and the worker SSRF guard rely on is preserved. | | `network.tor_egress_firewall` | `true` _(on)_ | Privacy-relevant, default on. Enforces "behind Tor" fail-closed: at `up`/`apply`, `pithead` installs host firewall rules (Docker's `DOCKER-USER` chain) that drop any direct clearnet dial from the mining containers (monerod/p2pool/tari/xmrig-proxy). Only the Tor container reaches the internet, so a misconfigured or buggy daemon can't leak your IP. Needs root (like the GRUB/HugePages steps); removed at `down`. Set `false` to skip it and rely on per-app Tor config only (e.g. a host where you manage egress yourself, or where `iptables` isn't available). Full detail: [Privacy › Enforced fail-closed](privacy.md#enforced-fail-closed-not-just-configured-270). | | `workers.api_auth` | `none` | How the dashboard reads each worker's xmrig API. Beyond what the proxy reports, the dashboard probes every connected miner's own xmrig HTTP API (`/1/summary`) for uptime and per-miner hashrate, one configured way, no auto-detection. `none` (default) expects an open, read-only API (xmrig `http.restricted` with no `access-token`), which is what a stock RigForge worker exposes. `name` sends the worker's stratum name as the Bearer token (for miners whose `access-token` equals their name). `token` sends a single shared `workers.api_token` for every worker. A worker whose probe fails isn't dropped: it keeps its proxy-reported hashrate and is flagged `api ⚠` on the dashboard, with one log line explaining why (so a misconfigured API is distinct from an offline miner). Upgrading from a build whose miners set an `access-token`? Set this to `name` (or reprovision the miners to drop the token), otherwise the default no-auth probe `401`s and every worker shows `api ⚠`. | diff --git a/docs/dashboard.md b/docs/dashboard.md index b1738d4d..8e75b5b5 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -154,6 +154,16 @@ a fresh database, and keeps running. A `db_reset` alert (Telegram and the other history before that point was cleared. Payout and XvB state rebuild from the chain and the live feed; only the historical charts reset. +**Fail-closed miner hold.** By default, every health failure above — DB write failing, DB +corruption, a crash-looping container — only alerts; the dashboard is an observability layer, and +the mining datapath (`xmrig-proxy` → `p2pool` → `monerod`) runs independently of it. Set +[`dashboard.fail_closed`](configuration.md#configuration-reference) to `true` to hold the miner +instead, but only for genuinely **unrecoverable** failures: the DB self-heal above failing on its +own rebuild attempt (not an ordinary write blip, which stays alert-only), or the `dashboard` +container itself crash-looping. A red `Miner held (fail-closed)` badge shows while held, next to +`p2pool` and `xmrig-proxy`, stopped the same way the [Sync Mode hold](#sync-mode) does; unlike that +one-way sync gate, both containers start again on their own once the condition clears — no restart +needed. The database also keeps three smaller series: pool block-found events, hourly monerod-DB-size and host-disk-usage samples, and XvB-credited scalar samples taken roughly every 5 minutes. `/api/state` serves them as `blocks`, `disk_growth`, and `xvb_history`, range-filtered the same way as diff --git a/docs/dev/testing-strategy.md b/docs/dev/testing-strategy.md index e1a520db..bce8c1cf 100644 --- a/docs/dev/testing-strategy.md +++ b/docs/dev/testing-strategy.md @@ -71,6 +71,7 @@ The deploy-time axes — each changes a real runtime path. Full table and assert | Double outage; readmit only when **both** healthy | both down → both up | 1 ✅ (added) · 3 ▶ | | #35 latch × #31 failover coexist after release | down post-release | 1 ✅ (added) · 3 ▶ | | Stop/start fails → retry next cycle (idempotent) | docker error | 1 ✅ | +| `dashboard.fail_closed` (#490): default off never holds on an unrecoverable failure (alert-only); `true` holds (reusing #35's stop/start), releases once it clears (not a one-way latch), no-op before the sync gate releases | `is_db_unrecoverable() ∨ containers.is_confirmed_bad("dashboard")` | 1 ✅ · 3 ▶ | ### D. Container health verdicts (`pithead status`) diff --git a/pithead b/pithead index 90f2d96d..0c7a1622 100755 --- a/pithead +++ b/pithead @@ -3593,6 +3593,15 @@ render_env() { local tari_required tari_required=$(jq -r 'if .dashboard.tari_required != null then .dashboard.tari_required | tostring else "true" end' "$CONFIG_FILE") + # Opt-in fail-closed miner hold on an unrecoverable dashboard health failure (#490), default + # false. The dashboard is an observability layer, not the mining datapath, so the default is + # alert-only (loud Telegram/Healthchecks alert + badge; mining continues). true reuses the #35 + # sync-gate's own hold to stop p2pool+xmrig-proxy until a DB-recovery failure or a + # crash-looping dashboard container clears — see build/dashboard .../service/data_service.py + # DataService._apply_fail_closed_gate for the exact "unrecoverable" set. + local fail_closed + fail_closed=$(normalize_bool "$(config_bool '.dashboard.fail_closed' false)") + # Healthchecks.io dead-man's switch (#79). Optional external liveness monitor: a ping URL is the # on/off switch (blank = off), and the ping always rides Tor. The URL is a capability secret, so # it lives in the owner-only .env (chmod 600 below), never a world-readable file. docs/monitoring.md. @@ -3809,6 +3818,7 @@ XVB_ENABLED=$xvb_enabled XVB_TOR_ENABLED=$xvb_tor XVB_DONATION_LEVEL=$xvb_donation_level TARI_REQUIRED=$tari_required +DASHBOARD_FAIL_CLOSED=$fail_closed DASHBOARD_CHECK_UPDATES=$check_for_updates TARI_MEM_LIMIT=$tari_mem_limit HEALTHCHECKS_PING_URL=$hc_ping_url @@ -4427,6 +4437,13 @@ describe_change() { msg="Tari → non-blocking — keep mining Monero through a Tari outage, start as soon as Monero is synced, and keep the operational dashboard while Tari syncs." fi ;; + DASHBOARD_FAIL_CLOSED) + if [ "$new" == "true" ]; then + msg="Fail-closed ENABLED (#490) — an unrecoverable dashboard health failure (DB recovery itself failing, or the dashboard container crash-looping) now HOLDS p2pool and xmrig-proxy until it clears, instead of only alerting." + else + msg="Fail-closed DISABLED (#490) — an unrecoverable dashboard health failure now only alerts (Telegram/Healthchecks + badge); mining is never held for it." + fi + ;; DASHBOARD_CHECK_UPDATES) if [ "$new" == "true" ]; then msg="Dashboard update check ENABLED (#224) — the dashboard will check GitHub (over Tor) for a newer release and show a link badge; the dashboard container is recreated." @@ -4982,7 +4999,7 @@ run_chain() { # the Telegram channel (the future #338 approval channel), so the dashboard must not silence # them. Space-separated exact env-key names. CONTROL_DASHBOARD_EDITABLE_KEYS='P2POOL_FLAGS P2POOL_PORT - XVB_ENABLED XVB_DONATION_LEVEL TARI_REQUIRED + XVB_ENABLED XVB_DONATION_LEVEL TARI_REQUIRED DASHBOARD_FAIL_CLOSED DASHBOARD_CHECK_UPDATES DASHBOARD_TZ MONERO_MEM_LIMIT TARI_MEM_LIMIT MONERO_PREP_THREADS HASHRATE_DROP_THRESHOLD_PCT HASHRATE_DROP_MINUTES TELEGRAM_DAILY_SUMMARY_TIME diff --git a/tests/stack/run.sh b/tests/stack/run.sh index 468b5983..cf2caf33 100755 --- a/tests/stack/run.sh +++ b/tests/stack/run.sh @@ -411,6 +411,13 @@ esac assert_contains "tor auto-heal enable is INFO" "$(run_sourced "$SANDBOX" describe_change TOR_AUTO_HEAL false true)" "INFO" assert_contains "tor auto-heal enable names the cost" "$(run_sourced "$SANDBOX" describe_change TOR_AUTO_HEAL false true)" "drops ALL Tor circuits" assert_contains "tor auto-heal disable names the manual fix" "$(run_sourced "$SANDBOX" describe_change TOR_AUTO_HEAL true false)" "restart tor" +# Fail-closed miner hold (#490): INFO either way (like TARI_REQUIRED) — it's on the dashboard +# control-channel allowlist, so a DEST flag here would make control_approval_gate refuse every +# commit that touches it, defeating the allowlisting. +assert_contains "fail_closed enable is INFO" "$(run_sourced "$SANDBOX" describe_change DASHBOARD_FAIL_CLOSED false true)" "INFO" +assert_contains "fail_closed enable names the hold" "$(run_sourced "$SANDBOX" describe_change DASHBOARD_FAIL_CLOSED false true)" "HOLDS p2pool and xmrig-proxy" +assert_contains "fail_closed disable is INFO" "$(run_sourced "$SANDBOX" describe_change DASHBOARD_FAIL_CLOSED true false)" "INFO" +assert_contains "fail_closed disable names alert-only" "$(run_sourced "$SANDBOX" describe_change DASHBOARD_FAIL_CLOSED true false)" "only alerts" # Dev-fee donate-level (#173): a brief restart (INFO), shown as a percentage. assert_contains "donate-level is INFO" "$(run_sourced "$SANDBOX" describe_change PROXY_DONATE_LEVEL 0 1)" "INFO" assert_contains "donate-level shows pct" "$(run_sourced "$SANDBOX" describe_change PROXY_DONATE_LEVEL 0 1)" "0% → 1%" @@ -3155,6 +3162,7 @@ assert_eq "P2POOL_URL keeps the internal :3333" "$(run_sourced "$V" env_get_file assert_eq "token preserved" "$(run_sourced "$V" env_get_file "$V/.env" PROXY_AUTH_TOKEN)" "ORIGINALTOKEN" assert_eq "onion preserved" "$(run_sourced "$V" env_get_file "$V/.env" P2POOL_ONION_ADDRESS)" "p2pa.onion" assert_eq "tari_required default" "$(run_sourced "$V" env_get_file "$V/.env" TARI_REQUIRED)" "true" +assert_eq "fail_closed default off (#490)" "$(run_sourced "$V" env_get_file "$V/.env" DASHBOARD_FAIL_CLOSED)" "false" # The new-release check (#224) defaults ON when absent from config — it's Tor-routed, so it leaks # nothing, and an operator who wants zero GitHub contact sets check_for_updates:false to opt out. assert_eq "check_for_updates default on" "$(run_sourced "$V" env_get_file "$V/.env" DASHBOARD_CHECK_UPDATES)" "true" @@ -3205,6 +3213,12 @@ printf '{ "monero": {"mode":"local","wallet_address":"%s","node_username":"u","n out="$(cd "$V" && DOCKER_LOG="$DOCKER_LOG" PATH="$V/bin:$PATH" ./pithead apply -y 2>&1)" assert_eq "tari_required propagated false" "$(run_sourced "$V" env_get_file "$V/.env" TARI_REQUIRED)" "false" +# Opt-in fail-closed (dashboard.fail_closed:true) propagates as DASHBOARD_FAIL_CLOSED=true (#490). +seed_env +printf '{ "monero": {"mode":"local","wallet_address":"%s","node_username":"u","node_password":"p"}, "tari":{"wallet_address":"T"}, "p2pool":{"pool":"mini"}, "dashboard":{"secure":false,"host":"box.lan","fail_closed":true} }\n' "$WALLET" >"$V/config.json" +out="$(cd "$V" && DOCKER_LOG="$DOCKER_LOG" PATH="$V/bin:$PATH" ./pithead apply -y 2>&1)" +assert_eq "fail_closed propagated true" "$(run_sourced "$V" env_get_file "$V/.env" DASHBOARD_FAIL_CLOSED)" "true" + # Opting out (dashboard.check_for_updates:false) propagates as DASHBOARD_CHECK_UPDATES=false (#224) — # only an explicit false disables it (anything else, incl. absent, stays the default-on true). seed_env @@ -5034,6 +5048,7 @@ roundtrip_key() { #