diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f0f8460..cb3ce1f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,11 +24,11 @@ jobs: env: UV_PYTHON_DOWNLOADS: never # use the setup-python 3.11; don't fetch another interpreter steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 # full history so diff-cover can diff the PR against origin/develop (#286) - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: "Install uv (pinned — reproducible, hash-locked installs, #283)" @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -70,7 +70,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Build the dashboard test stage (installs package + runs the suite in-container) @@ -90,7 +90,7 @@ jobs: matrix: service: [monero, p2pool, tor, xmrig-proxy, dashboard] steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: docker build ./build/${{ matrix.service }} @@ -112,7 +112,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: "Install hadolint (pinned + sha256-verified, #286)" @@ -131,10 +131,10 @@ jobs: env: UV_PYTHON_DOWNLOADS: never steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: Install uv (pinned) @@ -152,7 +152,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 # full history so the scan covers every commit, not just the tip @@ -171,10 +171,10 @@ jobs: env: UV_PYTHON_DOWNLOADS: never steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: Install uv (pinned) @@ -190,7 +190,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: "Install shellcheck + shfmt (pinned + sha256-verified, #286)" @@ -227,7 +227,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Validate docker-compose.yml interpolation + hardening invariants (#90) @@ -240,13 +240,13 @@ jobs: env: UV_PYTHON_DOWNLOADS: never steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "20" - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: Install uv (pinned) diff --git a/.github/workflows/integration-mini-stack.yml b/.github/workflows/integration-mini-stack.yml index c2601129..39f6a946 100644 --- a/.github/workflows/integration-mini-stack.yml +++ b/.github/workflows/integration-mini-stack.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # ubuntu-latest ships Docker with the Compose v2 plugin — no setup needed. diff --git a/.github/workflows/lychee.yml b/.github/workflows/lychee.yml index 3c37a27e..5df24b4b 100644 --- a/.github/workflows/lychee.yml +++ b/.github/workflows/lychee.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Check links in Markdown diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index ae25de27..6bedc29c 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -49,7 +49,7 @@ jobs: # box; prefer an ephemeral / just-in-time runner in its own runner group. runs-on: [self-hosted, pithead-release] steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 24822d32..6fc0b966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,95 @@ Pithead ships as **one product, one version** — the version lives in the top-l [`VERSION`](VERSION) file and every released image is tagged with it. Releases are cut per the process in [`docs/dev/releasing.md`](docs/dev/releasing.md). +## [Unreleased] + +## [1.11.0] - 2026-07-21 + +### Added + +- **Confirm-gated config editing for operationally-disruptive settings** (#719). The dashboard + config editor can now commit a small set of disruptive-but-recoverable settings behind a + type-to-confirm, instead of refusing them outright: the four service data directories (a move + re-syncs), the stratum port (rigs must repoint), the Monero/Tari clearnet initial-sync toggles + (the host IP is exposed during IBD, then auto-reverts to Tor), and enabling Monero pruning. Each + renders editable with a "you'll type `APPLY` to confirm" affordance; the typed confirmation rides + to the host approval gate, which requires it before the change proceeds and records the apply in + the control audit log as a distinct `commit-confirmed` action. + +### Changed + +- **The control-channel security perimeter stays host-only** (#719). Type-to-confirm is UX + friction, not a security control — a compromised dashboard that can set a field can also fill the + confirm box — so the perimeter is unchanged: wallets and view keys, the dashboard login and onion + settings, the control channel itself, the Tor egress firewall, the stratum password, node + endpoints and credentials, and the per-rig hosts and tokens all remain refused from the dashboard, + as does the heavier direction of a confirm-gated key (disabling pruning forces a full re-sync). +- **Dashboard-confirmed data-directory moves are allowlisted to the stack's data root** (#728). + #719 made the four `*_DATA_DIR` moves confirmable from the dashboard behind a typed `APPLY`, but + the destination was still checked only by the host-side blocklist (`assert_safe_dir`), which + passes any non-catastrophic absolute path. A confirmed move from the dashboard is now further + held to an allowlist: the new location must sit under the stack's own data root (the install + dir's `data/`) or a parent the stack already keeps data in, else the move is refused even with + the typed confirmation and stays host-CLI only. The host `./pithead apply` path keeps the wider + blocklist — a shell operator already has filesystem-wide reach; only the dashboard-reachable move + is tightened, closing the destination trust-escalation the confirm-gate opened. +- **Opt-in local miner** (#593). A box that runs the stack 24/7 can mine with its spare CPU by + co-locating a RigForge worker on the stack host. `./pithead setup` now asks "Also mine on this + machine with its spare CPU?" (off by default; also the new `local_miner.enabled` config flag), + and setup/apply print the two values a RigForge install needs — the stack's own stratum URL + (loopback `127.0.0.1:3333`, or the configured `p2pool.stratum_bind`/`stratum_port`) and the + stratum secret already in `.env`. The co-located worker self-registers through the proxy like any + other rig. Pithead only declares the intent and hands off those values; RigForge owns all + host-level tuning (HugePages, GRUB, MSR, governor) and the miner service. See + [docs/workers.md](docs/workers.md#mine-on-the-stack-host-itself). +- **Warm XvB donation state on a backup stack (#249).** On a two-host failover pair — same wallet, + workers listing both hosts in `pools[]` — the backup's XvB donation controller used to cold-start + when the fleet failed over to it: the closed-loop split restarted from the feedforward estimate + and re-ramped for hours, over- or under-shooting the credited tier until it reconverged. The + controller's commanded donation fraction is now persisted, so a plain restart resumes the warmed + split instead of re-seeding cold. A backup can also point `xvb.standby.source` at the primary + dashboard's new read-only `/api/xvb-standby` endpoint; it periodically pulls the primary's + controller state and holds it as standby (inspectable in `/api/state`), then adopts it the first + time it actually donates at failover — so the split resumes warm. One-way (backup pulls from + primary), inert unless configured, and never acted on while the primary is authoritative (an idle + backup has no workers, so its controller stays on P2Pool). The pull follows the dashboard's + privacy-safe egress rule: an `.onion` source, a public IP, or any hostname rides the bridge Tor + SOCKS (the primary sees a Tor exit, never the backup's IP); only a provably-private/loopback IP + literal dials direct as a LAN hop — so the pull never opens a clearnet path, and the Security + panel reports its route. +- **Opt-in fail-closed miner hold on an unrecoverable dashboard health failure** (#490). New + `dashboard.fail_closed` toggle, default **off**. The dashboard is an observability layer, not the + mining datapath (`xmrig-proxy` → `p2pool` → `monerod` runs independently of it), so by default an + unhealthy condition only alerts (Telegram/Healthchecks/webhook) and shows a badge while mining + continues. Set it `true` and a genuinely unrecoverable failure — the SQLite database failing to + rebuild after its own auto-heal attempt (disk full, permissions), or the `dashboard` container + itself crash-looping past the #337 debounce — holds `p2pool` and `xmrig-proxy` using the same + #35 sync-gate stop/start mechanism, with a `Miner held (fail-closed)` badge. Unlike the sync + gate's one-way latch it re-checks every cycle and releases on its own once the condition clears. + A transient write blip, a slow query, or a single failed external fetch never trips it — those + still only alert. Gated by the #33 control-approval path like other `dashboard.*` toggles. +- **`/api/state` exposure for three of the #196 telemetry-backbone series (Tier-1).** The backbone + PR (#600) added five persisted SQLite tables with capture, storage, and retention, but shipped + without surfacing them to the client. This slice exposes three — `blocks` (pool block-found + events), `disk_growth` (hourly monerod-DB-size + host-disk-usage samples), and `xvb_history` + (~5-min XvB-credited scalar samples) — as range-filtered arrays under those same keys, bounded + at the existing 700-point chart cap for the two higher-cadence series. `network_history` and + `worker_history` are a separate (Tier-2) slice, not touched here. No chart renders any of these + series yet — that's a further follow-up. + +### Security + +- **The out-of-band audit trail is now flood-capped per worker (#724).** The `rig-edit` audit + source (#530) reads a worker's reported change id off the unauthenticated LAN worker feed. The + #530 deterministic row id collapses *repeats* of one change id to a single row, but not *distinct* + ones — so a malicious or compromised device presenting as a worker could report a fresh random + change id every poll, writing a new permanent `audit_events` row each ~30s cycle and slowly + filling the dashboard database (the table has no pruning). New `rig-edit` rows are now capped per + worker per rolling hour; beyond the cap the extra rows are dropped behind a single `rate-limited` + marker row and a logged warning, so the flood stays visible instead of growing the table without + limit. A genuine occasional rig change still records normally, and the non-attacker-controllable + `host-edit` and mirrored `control.log` rows are unaffected. + ## [1.10.2] - 2026-07-21 ### Fixed diff --git a/SECURITY.md b/SECURITY.md index 206cf560..315e35a7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -65,14 +65,21 @@ The stack's defaults: API (over the stack's Tor SOCKS), refuses any mismatch or non-release tag, and limits attempts to one per 10 minutes — the container cannot choose an image, tag, or registry. Enabling the channel without a dashboard password is a validation error, on a published onion it additionally - requires Tor client authorization, and every mutation is audited host-side. Commits are default-denied against an explicit allowlist of - operational settings: a commit that changes any env key off that list — in every direction - (enabling, changing, or disabling) — is refused, as is anything the change preview flags - destructive. Wallets, dashboard auth and onion exposure, the control channel itself, the Tor - egress firewall, clearnet toggles, node endpoints, binds, and every credential are off the - list, and a key added in the future stays un-committable until deliberately listed. Those - edits must be applied from the host CLI; out-of-band approval is tracked in - [#338](https://github.com/p2pool-starter-stack/pithead/issues/338). + requires Tor client authorization, and every mutation is audited host-side. Commits are default-denied against an explicit allowlist. Low-risk + operational settings commit directly; a small set of operationally-disruptive ones — data-directory + moves, the stratum port, enabling clearnet initial sync, and enabling pruning — commit only behind + a typed confirmation in the dashboard, and only in that direction. A dashboard-confirmed + data-directory move is further held to an **allowlist** (#728): the new location must sit under the + stack's own data root (the install dir's `data/`) or a parent the stack already keeps data in; + a move to any other absolute path is refused even with the typed confirmation and stays host-CLI + only. The host CLI keeps its wider blocklist check — a shell operator already has filesystem-wide + reach. Everything else is refused in + every direction, as is anything the change preview flags destructive (including the heavy direction + of a confirm-gated key, e.g. disabling pruning, which forces a full re-sync). The security + perimeter — wallets and view keys, dashboard auth and onion exposure, the control channel itself, + the Tor egress firewall, node endpoints, binds, every credential, and the per-rig hosts and tokens — + is never dashboard-committable, with or without the typed confirmation. A key added in the + future stays un-committable until deliberately listed. Those edits must be applied from the host CLI. - Attack visibility (#349): Caddy writes a JSON access log for every dashboard vhost (LAN and onion), and the control channel's host-side audit log records who changed what (setting names only, never values). The dashboard surfaces both read-only — a burst of 401s is the @@ -82,6 +89,18 @@ The stack's defaults: anonymous prober stored XSS against the operator. Both logs are size-bounded (Caddy's native rolling; a trim-before-append cap in the audit writer). Neither ever records a secret: Caddy redacts credential headers by default, and the audit writer logs key names only. +- Out-of-band change detection (#530): the audit trail above only sees requests the dashboard + itself handled. Its poll loop separately watches for a `config.json` change with no matching + control-channel commit, and a worker control-API report for a change the dashboard never sent, + and appends both — `host-edit` / `rig-edit` — to the same trail, keys or worker names only. The + persisted trail (mirrored `control.log` rows plus these two out-of-band kinds) lives in the + dashboard's own database, not just the log tail, so the Security panel's hour/day/month grouping + covers more than `control.log`'s own trimmed window. The `rig-edit` source reads off the + unauthenticated worker feed, so it is rate-capped per worker (#724): a rig reporting distinct + change_ids on every poll can add at most a bounded number of rows per hour before the rest are + dropped behind a single `rate-limited` marker — one LAN device can't grow the database without + limit. The `host-edit` and mirrored `control.log` rows are not attacker-controllable and are not + capped. ### Telegram control commands (#338) diff --git a/VERSION b/VERSION index 5ad2491c..1cac385c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.2 +1.11.0 diff --git a/build/dashboard/Dockerfile b/build/dashboard/Dockerfile index feedab0b..0301345a 100644 --- a/build/dashboard/Dockerfile +++ b/build/dashboard/Dockerfile @@ -4,7 +4,7 @@ # uv's own CVEs into image scans, #282). uv lives only in the build/test stages. # ========================================================================== # Pinned by digest (#135) so the python:3.11-slim tag can't be silently re-pointed. -FROM python:3.11-slim@sha256:baf89808ec37adeaab83cec287adb4a2afa4a11c1d51e961c7ec737877e61af6 AS base +FROM python:3.11-slim@sha256:db3ff2e1800a8581e2c48a27c3995339d47bdf046da21c7627accd3d51053a93 AS base # Run from a project venv on PATH (so entrypoint.sh's `python3` resolves to it); use the # digest-pinned base interpreter (never let uv download a different Python); compile bytecode and diff --git a/build/dashboard/mining_dashboard/config/config.py b/build/dashboard/mining_dashboard/config/config.py index 1408024b..aba598b8 100644 --- a/build/dashboard/mining_dashboard/config/config.py +++ b/build/dashboard/mining_dashboard/config/config.py @@ -429,6 +429,15 @@ def _nonneg(v): if c.strip() ] +# Opt-in fail-closed miner hold on an UNRECOVERABLE health failure (#490), `dashboard.fail_closed`, +# default false. The dashboard is an observability layer, not the mining datapath (xmrig-proxy -> +# p2pool -> monerod is independent of it), so the default posture is alert-only: a cosmetic +# dashboard fault must never idle the fleet. `true` reuses the #35 sync-gate's own mechanism +# (stop/start SYNC_GATE_CONTAINERS) to hold the miner until the failure clears — see +# DataService._apply_fail_closed_gate for what counts as "unrecoverable" (narrowly: DB recovery +# itself failing, or the dashboard container crash-looping — never a transient blip). +DASHBOARD_FAIL_CLOSED = os.environ.get("DASHBOARD_FAIL_CLOSED", "false").strip().lower() == "true" + # Debounce: a node must be unreachable this long before it's declared DOWN, and reachable # this long before recovery — so a single transient timeout or a brief restart doesn't # kick every miner to their backups (and back) on a blip. @@ -689,6 +698,21 @@ def _telegram_event_enabled(name, default=True): # Switching overhead (ms) to account for connection ramp-up time XVB_SWITCH_OVERHEAD_MS = 5000 +# --- XvB warm-standby, backup stack (#249, config.json: xvb.standby.source) --- +# On a two-host failover pair (same wallet, workers list both in pools[]), the BACKUP points this at +# the PRIMARY's dashboard so it can pull the primary's XvB controller state and hold it as standby — +# so when workers fail over, the backup resumes the donation split warm instead of cold-ramping for +# hours. Blank (the default) = off: an ordinary single stack never pulls. The value is the full URL +# of the primary's read-only endpoint (e.g. https:///api/xvb-standby, or a LAN +# http://host:port/api/xvb-standby); an .onion source rides the bridge Tor SOCKS, a LAN source dials +# direct — no new clearnet egress either way. It is a capability URL (it can carry the primary's +# dashboard basic-auth as userinfo), so pithead renders it into the owner-only .env and masks it in +# the control-channel config, exactly like the Healthchecks ping URL. +XVB_STANDBY_SOURCE = os.environ.get("XVB_STANDBY_SOURCE", "").strip() +# How often the backup re-pulls the primary's state (seconds). A code-level constant, not a knob: +# the state moves slowly (the closed loop steps once per 10-min cycle), so a minute is ample. +XVB_STANDBY_INTERVAL_S = 60 + # --- Data Retention Policies --- HISTORY_RETENTION_SEC = 30 * 24 * 3600 # 30 Days # How long an offline worker lingers in the live "Workers Alive" table before it falls off (#182). diff --git a/build/dashboard/mining_dashboard/main.py b/build/dashboard/mining_dashboard/main.py index 27fc0cb3..960f1061 100644 --- a/build/dashboard/mining_dashboard/main.py +++ b/build/dashboard/mining_dashboard/main.py @@ -15,6 +15,7 @@ from mining_dashboard.service.data_service import DataService from mining_dashboard.service.storage_service import StateManager from mining_dashboard.service.telegram_commands import TelegramCommandBot +from mining_dashboard.service.xvb_standby import XvbStandbyPuller from mining_dashboard.web.server import create_app logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") @@ -37,20 +38,29 @@ def build_app() -> web.Application: # On-demand Telegram command interface (#45). Reads the snapshot data_service already collects; # a no-op unless telegram.enabled + telegram.commands.enabled + bot_token + chat_id are set. telegram_bot = TelegramCommandBot(data_service) + # Backup-stack warm-standby puller (#249): pulls the primary's XvB controller state so a + # failover resumes warm. Inert unless xvb.standby.source is configured. + xvb_standby = XvbStandbyPuller(state_manager) async def start_background_tasks(app): """Initializes background services upon web application startup.""" app["data_task"] = asyncio.create_task(data_service.run()) app["algo_task"] = asyncio.create_task(algo_service.run()) app["telegram_task"] = asyncio.create_task(telegram_bot.run()) + app["xvb_standby_task"] = asyncio.create_task(xvb_standby.run()) async def cleanup_background_tasks(app): """Stops background tasks and closes resources on shutdown.""" app["data_task"].cancel() app["algo_task"].cancel() app["telegram_task"].cancel() + app["xvb_standby_task"].cancel() await asyncio.gather( - app["data_task"], app["algo_task"], app["telegram_task"], return_exceptions=True + app["data_task"], + app["algo_task"], + app["telegram_task"], + app["xvb_standby_task"], + return_exceptions=True, ) if "state_manager" in app: app["state_manager"].close() diff --git a/build/dashboard/mining_dashboard/service/alert_service.py b/build/dashboard/mining_dashboard/service/alert_service.py index 9b100c5a..a40a68b1 100644 --- a/build/dashboard/mining_dashboard/service/alert_service.py +++ b/build/dashboard/mining_dashboard/service/alert_service.py @@ -765,6 +765,16 @@ async def process(self, **signals): self._wallet_edges(signals.get("observed_wallet", "")) except Exception as exc: # never let the tripwire break the data loop logger.debug("Wallet baseline update failed (%s)", type(exc).__name__) + # Keep the container-health debounce state current even with every sink off (#490): + # `dashboard.fail_closed` reads `self.containers.is_confirmed_bad("dashboard")` off + # the same tracker this alerting path would otherwise be the only feeder for. No + # alert fires here — just the state update the alerting branch below does anyway. + containers = signals.get("containers") + if containers is not None: + try: + self.containers.update(containers, now=signals.get("now")) + except Exception as exc: # never let a tracker bug break the data loop + logger.debug("Container-health update failed (%s)", type(exc).__name__) return [] try: alerts = self.evaluate(**signals) diff --git a/build/dashboard/mining_dashboard/service/algo_service.py b/build/dashboard/mining_dashboard/service/algo_service.py index ab3ba7a2..6971b6e7 100644 --- a/build/dashboard/mining_dashboard/service/algo_service.py +++ b/build/dashboard/mining_dashboard/service/algo_service.py @@ -297,15 +297,53 @@ def _advance_controller(self, current_hr, target_hr, avg_1h, max_fraction): if current_hr <= 0: return - # Seed from the feedforward estimate so we converge from a sane point. + # Seed the closed-loop state on the first real cycle — warm if we can, cold otherwise. if self.donation_fraction is None: - self.donation_fraction = min(self._reference_hr(target_hr) / current_hr, max_fraction) + self.donation_fraction = self._seed_donation_fraction( + target_hr, current_hr, max_fraction + ) return error = self._reference_hr(target_hr) - avg_1h self.donation_fraction += self.control_gain * error / current_hr self.donation_fraction = max(0.0, min(self.donation_fraction, max_fraction)) + def _seed_donation_fraction(self, target_hr, current_hr, max_fraction): + """The starting donated fraction for the closed loop, warm when we can prove it (#249). + + Precedence, highest first — each clamped to the VIP reserve (``max_fraction``): + + 1. **This host's own persisted commanded fraction.** Once the controller has steered and + persisted a non-zero fraction, a plain process restart resumes from it rather than + re-ramping cold. It also wins over standby: a stack that has been authoritative owns its + state, so a stale standby from a since-departed primary can't override it. + 2. **Standby state pulled from the primary** (backup failover). The first time a backup + actually donates — its workers just failed over — it adopts the primary's last-known + commanded fraction so the split resumes warm instead of restarting from zero (the whole + point of #249). While idle the backup never steers, so its own persisted fraction stays + 0.0 and this branch is what fires at handover. + 3. **Feedforward estimate** (cold start). A fresh install with no history and no standby — + the original behaviour, converging from a sane point via the closed loop. + """ + feedforward = min(self._reference_hr(target_hr) / current_hr, max_fraction) + + own = (self.state_manager.get_xvb_stats() or {}).get("commanded_fraction", 0.0) or 0.0 + if own > 0: + return min(own, max_fraction) + + standby = self.state_manager.get_xvb_standby() or {} + standby_fraction = standby.get("commanded_fraction", 0.0) or 0.0 + if standby_fraction > 0: + logger.info( + "Warm-resume: adopting primary's standby donation fraction %.3f on failover " + "(#249) instead of cold-seeding %.3f", + standby_fraction, + feedforward, + ) + return min(standby_fraction, max_fraction) + + return feedforward + def _fraction_to_ms(self, fraction): """Convert a donated fraction of the cycle to a slice length (ms), adding the fixed switch/ramp-up overhead. Returns 0 for a non-positive fraction.""" @@ -447,10 +485,13 @@ async def run(self): ) # Record the fraction of this cycle actually routed to XvB so the - # dashboard can show routed-vs-credited (the live credit factor). + # dashboard can show routed-vs-credited (the live credit factor), and persist + # the controller's own commanded fraction so a restart / backup failover resumes + # warm rather than re-seeding cold (#249). await asyncio.to_thread( self.state_manager.update_xvb_stats, donation_fraction=self._routed_fraction(decision, xvb_duration), + commanded_fraction=self.donation_fraction or 0.0, ) if decision == "P2POOL": diff --git a/build/dashboard/mining_dashboard/service/container_health.py b/build/dashboard/mining_dashboard/service/container_health.py index deeca415..63a56f73 100644 --- a/build/dashboard/mining_dashboard/service/container_health.py +++ b/build/dashboard/mining_dashboard/service/container_health.py @@ -69,6 +69,17 @@ def __init__( # ok_since : when the current continuous-clean streak began self._containers = {} + def is_confirmed_bad(self, name): + """True only if `name`'s bad state was CONFIRMED by the debounce — a crash loop or a + continuous-unhealthy streak past ``unhealthy_after`` — i.e. it produced an alert edge + (``alerted``). A first-sighting silently-seeded baseline (already unhealthy/restarting at + the monitor's first look) is deliberately NOT confirmed: it skipped the debounce a KNOWN + container must pass, exactly as it skips the alert. `dashboard.fail_closed`'s miner hold + (#490) reads this rather than the raw level, so it holds the fleet only on a confirmed, + non-transient failure. Unknown/never-seen container reads as not bad.""" + c = self._containers.get(name) + return bool(c and c["state"] == "bad" and c["alerted"]) + def update(self, states, now=None): """Feed this cycle's ``{name: state}`` snapshot; return the debounced edges.""" now = self._clock() if now is None else now diff --git a/build/dashboard/mining_dashboard/service/control_service.py b/build/dashboard/mining_dashboard/service/control_service.py index 2b9e4147..9311be6f 100644 --- a/build/dashboard/mining_dashboard/service/control_service.py +++ b/build/dashboard/mining_dashboard/service/control_service.py @@ -38,6 +38,9 @@ # A capability secret: pithead's describe_change already refuses to echo it, but read_config # was serving it in cleartext to the browser. Mask it too (#33 hardening). ("healthchecks", "ping_url"), + # The backup's primary-dashboard URL (#249) can carry the primary's dashboard basic-auth as + # userinfo — a capability secret, masked like the ping URL above. + ("xvb", "standby", "source"), ] SECRET_SENTINEL = {"__secret__": True} @@ -105,6 +108,7 @@ def _deep_merge(base, override): "XVB_ENABLED": ("xvb.enabled",), "XVB_DONATION_LEVEL": ("xvb.donation_level",), "TARI_REQUIRED": ("dashboard.tari_required",), + "DASHBOARD_FAIL_CLOSED": ("dashboard.fail_closed",), "DASHBOARD_CHECK_UPDATES": ("dashboard.check_for_updates",), "DASHBOARD_TZ": ("dashboard.timezone",), "MONERO_MEM_LIMIT": ("monero.mem_limit",), @@ -171,6 +175,48 @@ def _editable_paths(): return sorted(paths) +# Env-var -> config-path map for the CONFIRM-gated set (#719), mirroring pithead's +# CONTROL_DASHBOARD_CONFIRM_KEYS the same way EDITABLE_ENV_KEY_PATHS mirrors the editable allowlist +# (drift-guarded by test_confirm_keys_have_no_intra_repo_drift). These are operationally-disruptive +# but NOT the security perimeter: the dashboard MAY commit them, but only behind a type-to-confirm. +# Surfaced to the browser as ``_confirm_keys`` so the Configuration view renders them editable with +# a "confirm to proceed" affordance instead of greying them out as host-only. The gate is still the +# authority: describe_change decides per-DIRECTION whether a change is CONFIRM (a data-dir move, a +# stratum-port repoint, a clearnet-sync ENABLE, a prune ENABLE) or stays a host-only DEST (prune +# DISABLE, a TOR data-dir move), so a field here can still be refused at commit in its heavy +# direction — the same edit-then-maybe-refuse tradeoff the issue accepts for MONERO_PRUNE. +CONFIRM_ENV_KEY_PATHS = { + "MONERO_DATA_DIR": ("monero.data_dir",), + "TARI_DATA_DIR": ("tari.data_dir",), + "P2POOL_DATA_DIR": ("p2pool.data_dir",), + "DASHBOARD_DATA_DIR": ("dashboard.data_dir",), + "STRATUM_PORT": ("p2pool.stratum_port",), + "MONERO_CLEARNET_SYNC": ("monero.clearnet_initial_sync",), + "TARI_CLEARNET_SYNC": ("tari.clearnet_initial_sync",), + "MONERO_PRUNE": ("monero.prune",), +} + + +def _confirm_paths(): + """Every config path the control gate will commit behind a type-to-confirm (#719).""" + return sorted({p for target in CONFIRM_ENV_KEY_PATHS.values() for p in target}) + + +def env_key_config_paths(env_key): + """The config-path prefixes a committed audit ``keys`` env-var name covers (#530). + + The #33 audit log records a commit's WHAT-changed as env-var NAMES (control_approval_gate's + ``porcelain_keys``), while the out-of-band host-edit watcher diffs config.json PATHS — so + correlating "did a commit explain this changed key" needs this env->path bridge. Mirrors the + commit gate's own derivation: an allowlisted var maps through EDITABLE_ENV_KEY_PATHS, and the + synthetic ``DASHBOARD_ENERGY`` name (which the gate folds in for a dashboard.energy-only + commit, a config.json-only block that never renders to .env, #504) covers the whole energy + block by prefix. An unrecognised name maps to nothing, so it can never explain a diffed key.""" + if env_key == "DASHBOARD_ENERGY": + return ("dashboard.energy",) + return EDITABLE_ENV_KEY_PATHS.get(env_key, ()) + + def _load_core_keys(): """The wizard's core-key shortlist (#502/#529), read from the SAME file ``./pithead setup`` reads — the one shared artifact, not a second hand-maintained list. Degrades to an empty list @@ -211,29 +257,38 @@ def read_config(): _set(cfg, path, dict(SECRET_SENTINEL)) cfg["_core_keys"] = _load_core_keys() cfg["_editable_keys"] = _editable_paths() + cfg["_confirm_keys"] = _confirm_paths() return cfg -def submit(action, cfg=None, actor="", intent_id=None, version=None): +def submit(action, cfg=None, actor="", intent_id=None, version=None, confirm=None): """Write one intent into the requests spool (atomic: temp + rename, so the runner never reads a half-written file). Returns the request id — always a UUID, because the id becomes a host-side filename and the runner rejects anything else. ``version`` rides only on the upgrade intent (#59): the version the operator confirmed, which the host re-verifies - against the GitHub release API — a proposal, never a target the container picks.""" + against the GitHub release API — a proposal, never a target the container picks. ``confirm`` + rides only on a commit intent (#719): the operator's typed confirmation for an in-scope + disruptive change, which the host gate requires before it lets a CONFIRM row proceed.""" rid = str(uuid.UUID(intent_id)) if intent_id else str(uuid.uuid4()) request = {"id": rid, "action": action, "actor": actor} if cfg is not None: - # read_config's own metadata injections (#529/#613) ride back with the editor's POST — + # read_config's own metadata injections (#529/#613/#719) ride back with the editor's POST — # both modes round-trip the fetched doc wholesale — and the host gate's closed-schema # check would refuse a commit carrying them (#679). Shed them at the one choke point # every config intent passes through; everything else unknown still fails closed host-side. # A non-dict cfg passes through untouched: the host runner already rejects it with its # own "config must be a JSON object" result, which the UI knows how to surface. if isinstance(cfg, dict): - cfg = {k: v for k, v in cfg.items() if k not in ("_core_keys", "_editable_keys")} + cfg = { + k: v + for k, v in cfg.items() + if k not in ("_core_keys", "_editable_keys", "_confirm_keys") + } request["config"] = cfg if version is not None: request["version"] = version + if confirm is not None: + request["confirm"] = confirm tmp = os.path.join(config.CONTROL_REQUESTS_DIR, f".{rid}.tmp") with open(tmp, "w") as f: json.dump(request, f) diff --git a/build/dashboard/mining_dashboard/service/data_service.py b/build/dashboard/mining_dashboard/service/data_service.py index 498a9c3d..b01270da 100644 --- a/build/dashboard/mining_dashboard/service/data_service.py +++ b/build/dashboard/mining_dashboard/service/data_service.py @@ -1,7 +1,10 @@ import asyncio +import json import logging import os import time +import uuid +from datetime import UTC, datetime from aiohttp import ClientSession @@ -35,10 +38,12 @@ get_load_average, get_memory_usage, ) +from mining_dashboard.config import config from mining_dashboard.config.config import ( CHECK_FOR_UPDATES, CLEARNET_STATE_DIR, DASHBOARD_ENERGY, + DASHBOARD_FAIL_CLOSED, ENABLE_XVB, GITHUB_RELEASES_API, GITHUB_RIGFORGE_RELEASES_API, @@ -67,8 +72,16 @@ pplns_block_time, shares_in_pplns_window, ) +from mining_dashboard.service import audit_service from mining_dashboard.service.alert_service import AlertService from mining_dashboard.service.clearnet_sync import ClearnetSyncSupervisor +from mining_dashboard.service.control_service import ( + SECRET_PATHS, + SECRET_SENTINEL, + _get, + _set, + env_key_config_paths, +) from mining_dashboard.service.degradation import DegradationMonitor from mining_dashboard.service.healthchecks import HealthchecksClient from mining_dashboard.service.metrics import build_metrics, share_reject_pct @@ -117,6 +130,16 @@ # like the capture cadences above. _XVB_WINNERS_SYNC_SEC = 1800 +# Per-worker flood cap on NEW rig-edit audit rows (#724). The enriched worker feed is +# unauthenticated LAN input, so a rogue device presenting as a worker can report a fresh random +# change_id every poll — each a distinct, permanent audit_events row (#530's deterministic id only +# collapses REPEATS of one change_id, never distinct ones). At most _RIG_EDIT_CAP_PER_HOUR genuine +# rig-edit rows per worker per rolling hour; beyond that, rows are dropped and a single +# rate-limited marker is recorded + logged. A real fleet edits a rig a handful of times an hour at +# most, so a legitimate cadence never trips it — only a flood does. +_RIG_EDIT_CAP_PER_HOUR = 12 +_RIG_EDIT_WINDOW_SEC = 3600 + def _parse_proxy_list_worker(w): """Parse one xmrig-proxy 6.x positional row into a worker dict. @@ -344,6 +367,79 @@ def _shares_to_record(last_known_total, current_total): return 0, last_known_total +def _iso_now(): + """UTC now, formatted to match the #33 audit writer's own ``ts`` (``control_audit`` in + ``pithead``) — same string shape both sources write, so the audit_events table sorts and + groups by hour/day/month with a plain string-prefix slice, no parsing needed at read time.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _flatten_config_keys(cfg, out, prefix=""): + """Fill ``out`` with ``{dotted.path: leaf_value}`` for every leaf in nested dict ``cfg``. Only + ever used to compare/NAME keys (see ``_diff_config_keys``) — a leaf value is compared for + equality, never rendered; the source, ``config.HOST_CONFIG_PATH``, is already the host's + pre-masked copy (#440), so no secret is present to leak even here. + + The masked secret sentinel (``control_service.SECRET_SENTINEL``, ``{"__secret__": True}``) is + treated as an opaque LEAF, not descended into — otherwise a secret being set/cleared would + name a synthetic ``...password.__secret__`` path instead of the real setting.""" + if not isinstance(cfg, dict): + return + for k, v in cfg.items(): + path = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict) and v != SECRET_SENTINEL: + _flatten_config_keys(v, out, path) + else: + out[path] = v + + +def _diff_config_keys(old, new): + """Dotted config-key paths added, removed, or changed between two config snapshots (#530), + sorted. Names only — the values feed only an equality check and are never returned, matching + the #33 audit contract (key names, never values).""" + old_flat, new_flat = {}, {} + _flatten_config_keys(old, old_flat) + _flatten_config_keys(new, new_flat) + changed = set(old_flat) ^ set(new_flat) # added or removed entirely + changed |= {k for k in old_flat.keys() & new_flat.keys() if old_flat[k] != new_flat[k]} + return sorted(changed) + + +def _parse_audit_ts(ts): + """Parse a #33 audit-log ``ts`` string (``%Y-%m-%dT%H:%M:%SZ``) to epoch seconds, or None for + anything else — a malformed/garbage ts (already length-capped and charset-stripped by + ``audit_service._clean``) must never crash the out-of-band watcher, just fail to "explain" a + change (the safe direction: an unparsable commit ts causes a spurious host-edit row, not a + swallowed one).""" + try: + return datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=UTC).timestamp() + except (TypeError, ValueError): + return None + + +def _read_host_config(): + """The masked config.json (``config.HOST_CONFIG_PATH``, #440), or None if the mount isn't + ready / isn't valid JSON yet. A plain blocking function — ``_watch_host_config`` runs it via + ``asyncio.to_thread`` rather than opening the file directly in an ``async def``. + + The mount is the host's PRE-MASKED copy already (docker-compose bind-mounts + ``control/masked/config.json``; the raw config.json never enters the container). We still + re-apply the SECRET_PATHS mask here — exactly the defense-in-depth pass ``control_service. + read_config`` runs — so a host-side masking regression can never leave a raw secret VALUE + resident in ``self._last_host_config`` across polls. The diff only ever compares/names keys, + but this keeps the one long-lived config dict secret-free regardless.""" + try: + with open(config.HOST_CONFIG_PATH) as f: + cfg = json.load(f) + except (OSError, ValueError): + return None + for path in SECRET_PATHS: + found, value = _get(cfg, path) + if found and value: + _set(cfg, path, dict(SECRET_SENTINEL)) + return cfg + + class WorkerLifecycle: """Dashboard-side per-worker connection tracking for the "Workers Alive" table (#169 / #182). @@ -447,6 +543,27 @@ def __init__(self, state_manager, proxy_client, xvb_client): self._last_xvb_history_write = 0.0 self._last_hourly_capture = 0.0 self._last_worker_capture = 0.0 + # Out-of-band audit watcher (#530): the last config.json snapshot this poll loop read + # (None until the first poll baselines it — never diff against nothing, same "re-baseline, + # never backfill" contract as every other watcher here) and the wall-clock of that read, so + # a later change can be checked against control.log entries that landed AFTER it. + self._last_host_config = None + self._last_host_check = 0.0 + # (worker, change_id) pairs already recorded as a rig-edit this run, so a rig that keeps + # reporting the same terminal change_id in its /status mirror every poll is flagged ONCE, + # not on every ~30s cycle. In-memory only: the deterministic audit-row id below is what + # actually bounds the table across restarts (INSERT OR IGNORE); this just skips the + # redundant DB work in the steady state. Bounded by the count of distinct real rig edits. + self._flagged_rig_changes = set() + # Per-worker fixed-window flood cap on NEW rig-edit rows (#724): {worker: (window_start, + # count)}. Distinct change_ids clear #530's deterministic-id dedup, so a rogue rig can + # spam a permanent audit row every poll; this bounds them to _RIG_EDIT_CAP_PER_HOUR per + # worker per hour. In-memory like _flagged_rig_changes — a restart resets the window, which + # at worst grants one extra window's budget, still bounded per wall-hour. + # ponytail: a rogue device rotating the worker NAME each poll sidesteps a per-worker cap; + # that's the broader unauth-feed vector (#235), out of scope here — cap keyed on worker per + # the issue, so one rogue rig can't crowd genuine rig-edit history out. + self._rig_edit_window = {} # XvB raffle-winners mirror: wall-clock of the last successful winners-file read. Starts # at 0.0 so the first eligible poll reads it; NOT stamped on a failed fetch, so a failure # retries on the next 10th poll instead of waiting out the 30-min gate. @@ -477,6 +594,7 @@ def __init__(self, state_manager, proxy_client, xvb_client): "workers_rejected": False, "miner_released": False, "miner_held": False, + "fail_closed_held": False, "timestamp": 0, } @@ -545,6 +663,11 @@ def __init__(self, state_manager, proxy_client, xvb_client): self.miner_released = False self.miner_held = False + # Opt-in fail-closed miner hold on an UNRECOVERABLE health failure (#490), dashboard. + # fail_closed, default false — see `_apply_fail_closed_gate`. Transient like `miner_held`, + # not persisted: a restart re-derives it from the current health signals. + self.fail_closed_held = False + # Restore persistent state from DB to prevent empty dashboard on service restart loaded_snapshot = self.state_manager.load_snapshot() if loaded_snapshot and isinstance(loaded_snapshot, dict): @@ -595,6 +718,19 @@ async def _apply_worker_rejection(self, monero_down, tari_down): if await self.docker_control.start(REJECT_WORKERS_CONTAINER): self.workers_rejected = False + async def _stop_gate_containers(self, quiet): + """Stop every ``SYNC_GATE_CONTAINERS`` container; shared by the #35 sync gate and the + #490 fail-closed gate, the two holds that stop the same container set.""" + for container in SYNC_GATE_CONTAINERS: + await self.docker_control.stop(container, quiet=quiet) + + async def _start_gate_containers(self): + """Start every ``SYNC_GATE_CONTAINERS`` container; True only if every start succeeded.""" + ok = True + for container in SYNC_GATE_CONTAINERS: + ok = (await self.docker_control.start(container)) and ok + return ok + async def _apply_sync_gate(self, gate_satisfied): """ Hold p2pool + xmrig-proxy stopped until the required chain(s) have fully synced once, @@ -619,10 +755,7 @@ async def _apply_sync_gate(self, gate_satisfied): return if gate_satisfied: - ok = True - for container in SYNC_GATE_CONTAINERS: - ok = (await self.docker_control.start(container)) and ok - if ok: + if await self._start_gate_containers(): self.miner_released = True self.miner_held = False logger.info( @@ -633,8 +766,7 @@ async def _apply_sync_gate(self, gate_satisfied): # Still syncing: keep the miner held. Log the human-facing notice only on the first # cycle of a hold; the per-cycle re-assert stops are quiet to avoid flooding the log. - for container in SYNC_GATE_CONTAINERS: - await self.docker_control.stop(container, quiet=self.miner_held) + await self._stop_gate_containers(quiet=self.miner_held) if not self.miner_held: self.miner_held = True logger.info( @@ -642,6 +774,53 @@ async def _apply_sync_gate(self, gate_satisfied): f"until synced." ) + async def _apply_fail_closed_gate(self, unrecoverable): + """ + Opt-in (`dashboard.fail_closed`, default False) miner hold on an UNRECOVERABLE health + failure (#490) — reuses the #35 sync gate's own mechanism (stop/start + ``SYNC_GATE_CONTAINERS`` through ``docker_control``) rather than a new hold path. + + "Unrecoverable" is scoped narrowly by the caller to genuine, non-transient failures: a DB + whose auto-heal rebuild itself failed (``StateManager.is_db_unrecoverable``), or the + dashboard container itself crash-looping / stuck unhealthy past the #337 debounce + (``AlertService.containers.is_confirmed_bad("dashboard")`` — a debounce-CONFIRMED verdict, + never a first-sighting seed). A transient write blip, a slow query, a single failed + external fetch, or a container merely reported unhealthy on one poll is never + "unrecoverable" — those already alert (#131/#337) and must never gate; a false positive + here idles the fleet and costs revenue. + + Unlike the sync gate's one-way latch, this re-checks every cycle and releases once + ``unrecoverable`` clears — the failures it watches (disk full, a crash-looping container) + are the kind an operator fixes without a full stack restart, and the miner should resume + on its own once they do. Only engages once the sync gate has actually released the miner; + holding before that is already #35's job. + + Default False is alert-only: `dashboard.fail_closed` off means these same signals keep + alerting (unchanged) but this method is a no-op, so a cosmetic dashboard fault never idles + the fleet — the mining datapath (xmrig-proxy -> p2pool -> monerod) is independent of the + dashboard by design. + """ + if not DASHBOARD_FAIL_CLOSED or not self.miner_released: + return + + if unrecoverable: + await self._stop_gate_containers(quiet=self.fail_closed_held) + if not self.fail_closed_held: + self.fail_closed_held = True + logger.error( + f"Unrecoverable health failure with dashboard.fail_closed enabled — holding " + f"{', '.join(SYNC_GATE_CONTAINERS)} until it clears." + ) + return + + if self.fail_closed_held and await self._start_gate_containers(): + self.fail_closed_held = False + logger.info( + f"Unrecoverable health failure cleared — starting " + f"{', '.join(SYNC_GATE_CONTAINERS)}; mining can resume." + ) + # On a partial-start failure stay held so the next cycle retries. + async def _sync_xvb_stats(self): """ Fetch XvB's reported averages (avg_1h/avg_24h/fail_count) over Tor and persist them. @@ -858,25 +1037,219 @@ async def _sync_tari_payouts(self): ) await self.alert_service.payout_confirmed_alert(chain, r["amount_atomic"], r["txid"]) - async def _reconcile_worker_config(self, worker_results): + async def _record_audit_event(self, source, actor, action, status, keys, event_id=None): + """Write one out-of-band audit row (#530), through the SAME sanitizer #33's own audit + trail is served through (``audit_service._clean``) — defense in depth: ``actor``/``keys`` + here are already schema-shaped (a validated worker name, dotted config-key paths), but + every field the Security panel serves gets the identical whitelist treatment regardless of + source, so a future caller can't accidentally skip it. + + ``event_id`` lets a caller supply a DETERMINISTIC row id so ``INSERT OR IGNORE`` collapses + repeat reports of the SAME event to one row (rig-edit: a rig re-reports its last change_id + every poll). A host-edit passes None — each detection is a genuinely distinct event, so a + random id is right there.""" + await asyncio.to_thread( + self.state_manager.add_audit_event, + id=event_id or f"{source}-{uuid.uuid4()}", + ts=_iso_now(), + source=audit_service._clean(source, 16), + actor=audit_service._clean(actor, 64), + action=audit_service._clean(action, 16), + status=audit_service._clean(status, 32), + keys=audit_service._clean(keys, 400), + ) + + async def _watch_host_config(self): + """Out-of-band HOST-EDIT detection (#530): config.json changed without a matching + control-channel commit. + + Reads the same pre-masked copy the control channel itself prefills from + (``config.HOST_CONFIG_PATH``, #440 — already secret-free) each poll and diffs it against + the previous poll's snapshot. A changed key is "explained" — and stays quiet — only when + the #33 audit trail shows a ``commit``/``applied`` entry that both landed AFTER the last + time this watcher looked AND actually touched that key (the entry's env-var names are + bridged to config paths via ``env_key_config_paths``). Correlating by key, not merely by + time, is what stops a legit dashboard commit of key A from swallowing a concurrent + host-side hand-edit of key B. Any changed key no fresh commit covers (a hand-edit, a + `pithead apply` run outside the dashboard) is recorded as a ``host-edit`` audit row naming + the unexplained keys. First poll only baselines (no control + log exists yet to compare against, and every other watcher in this loop shares that + never-backfill contract). No-op with the control channel off — there is neither a masked + config mount nor an audit trail to compare against.""" + if not config.DASHBOARD_CONTROL_ENABLED: + return + current = await asyncio.to_thread(_read_host_config) + if current is None: + return # mount not ready yet — quiet no-op, the next poll retries + now = time.time() + if self._last_host_config is None: + self._last_host_config, self._last_host_check = current, now + return + changed_keys = _diff_config_keys(self._last_host_config, current) + if changed_keys: + # The audit log's ts is whole-second (_iso_now/control_audit both write + # "%Y-%m-%dT%H:%M:%SZ"), while `_last_host_check` is a sub-second time.time() — a + # commit landed in the SAME wall-clock second as the baseline poll would otherwise + # floor below it and be missed. One second of grace absorbs that truncation. + # ponytail: ≤1s correlation window — a control-channel commit up to 1s before the last + # poll could "explain" (suppress) an unrelated hand-edit detected in this poll. The + # honest ceiling of a timestamp correlation; tighten to id-based ("commits seen since + # last poll") only if a real false-negative shows up. Pinned by + # test_explained_window_is_at_most_one_second. + since = self._last_host_check - 1 + # Correlate BY KEY, not just by time: a commit only "explains" the keys it actually + # touched. Fold every fresh commit's env-var names into the config paths they cover + # (env_key_config_paths bridges the audit log's env names to config.json paths), then + # record only the changed keys NO recent commit covers — the genuine out-of-band edits + # this watcher exists to catch. A concurrent dashboard commit of key A + a host + # hand-edit of key B no longer swallows B. + explained_paths = set() + for e in audit_service.recent_changes(): + if ( + e.get("action") == "commit" + and e.get("status") == "applied" + and (ts := _parse_audit_ts(e.get("ts"))) is not None + and ts >= since + ): + for env_key in (e.get("keys") or "").split(): + explained_paths.update(env_key_config_paths(env_key)) + # ponytail: env-var granularity — a var fed by >1 config path (e.g. P2POOL_FLAGS <- + # p2pool.pool + p2pool.clearnet) explains ALL its paths, so a commit touching one could + # still suppress a concurrent hand-edit of its sibling. Inherent to a name-only audit + # log; fix only if per-path audit keys ever land. + unexplained = [ + k + for k in changed_keys + if not any(k == p or k.startswith(p + ".") for p in explained_paths) + ] + if unexplained: + await self._record_audit_event( + "host-edit", "", "host-edit", "detected", " ".join(unexplained) + ) + self._last_host_config, self._last_host_check = current, now + + async def _mirror_control_audit(self): + """Copy the #33 control.log's recent entries into the durable ``audit_events`` table + (#530), so the Security panel's time-grouped view can drill deeper than the log's own + trimmed tail. ``audit_service.recent_changes()`` output is already sanitized (it's the SAME + read the panel used before this table existed); ``add_audit_event``'s ``INSERT OR IGNORE`` + on the log's own ``id`` makes re-mirroring the same tail every poll a no-op. Entries with no + id (a handful of pre-auth "invalid"/"refused" rows, #33) are skipped — they're visible only + while still in the log tail, same as before this feature.""" + if not config.DASHBOARD_CONTROL_ENABLED: + return + for e in audit_service.recent_changes(): + if not e.get("id"): + continue + await asyncio.to_thread( + self.state_manager.add_audit_event, + id=e["id"], + ts=e.get("ts", ""), + source="control", + actor=e.get("actor", ""), + action=e.get("action", ""), + status=e.get("status", ""), + keys=e.get("keys", ""), + ) + + def _rig_edit_within_cap(self, worker, now): + """Per-worker fixed-window cap on NEW rig-edit audit rows (#724). Counts this rig-edit + against the worker's current hour window and returns ``(allowed, first_over)``: ``allowed`` + is True while the worker is under ``_RIG_EDIT_CAP_PER_HOUR`` this window; ``first_over`` is + True only on the single call that tips it over, so the caller logs + records the + rate-limited marker exactly once per window rather than every poll. A handful of real edits + an hour never trips it; a rig spamming distinct change_ids does.""" + start, count = self._rig_edit_window.get(worker, (now, 0)) + if now - start >= _RIG_EDIT_WINDOW_SEC: + start, count = now, 0 + self._rig_edit_window[worker] = (start, count + 1) + return count < _RIG_EDIT_CAP_PER_HOUR, count == _RIG_EDIT_CAP_PER_HOUR + + async def _reconcile_worker_config(self, workers, worker_results): """Catch up any still-``accepted`` #185 worker-config history row whose change_id the rig - now reports terminal (#579). + now reports terminal (#579), and flag an out-of-band RIG-EDIT (#530). A rollback slower than the host runner's 20s status-poll deadline (#517/#543) is honestly recorded ``accepted`` and never revisited — this rides THIS poll's already-fetched enriched - bodies (``worker_results``, positionally aligned with the worker probes in ``run()``), so - there's no new dial and no host-runner change. A plain-xmrig rig, a rig still mid-change, or - an unreachable/offline rig (``{}``) all parse to ``None`` via ``parse_worker_control_status`` - and are a quiet no-op — the row simply stays ``accepted`` until a later poll catches it.""" - for extra_stats in worker_results: + bodies (``worker_results``, positionally aligned with ``workers`` and with the worker probes + in ``run()``), so there's no new dial and no host-runner change. A plain-xmrig rig, a rig + still mid-change, or an unreachable/offline rig (``{}``) all parse to ``None`` via + ``parse_worker_control_status`` and are a quiet no-op. + + A TERMINAL report whose ``change_id`` this dashboard never spooled (``worker_config`` has no + row for it — checked via ``worker_config_change_known``) is a change the RIG applied on its + own: reconciling it would be a silent no-op anyway (the ``WHERE status='accepted'`` UPDATE + matches nothing), so instead it's recorded as a ``rig-edit`` audit row naming the worker. + RigForge's ``/status`` mirror carries only the outcome of a change, not a per-key diff, so + unlike host-edit's ``keys`` this can only name the change_id — a real limitation, not an + oversight; see the #530 PR notes. + + A rig keeps reporting its last terminal change_id every poll, so this fires ONCE per + (worker, change_id): an in-memory guard skips the redundant work in the steady state, and + the audit row's deterministic id makes the write itself idempotent even across a restart + (when the guard is empty but a repeat report must still not duplicate the row). Both matter + — the id is the correctness bound (a rogue rig can't flood the permanent table with repeats + of one bogus change_id), the guard is the optimisation. + + DISTINCT change_ids each clear that dedup, though, so a rogue rig on the unauthenticated + feed can still write one permanent row per poll (#724). ``_rig_edit_within_cap`` bounds NEW + rig-edit rows to ``_RIG_EDIT_CAP_PER_HOUR`` per worker per hour; beyond that the row is + dropped, but never silently — a single ``rate-limited`` marker is logged and recorded so the + flood stays visible in the Security panel. host-edit rows are unaffected (a different, + non-attacker-controlled path).""" + for w, extra_stats in zip(workers, worker_results, strict=False): ctrl = parse_worker_control_status(extra_stats) if extra_stats else None - if ctrl: + if not ctrl: + continue + known = await asyncio.to_thread( + self.state_manager.worker_config_change_known, ctrl["change_id"] + ) + if known: await asyncio.to_thread( self.state_manager.reconcile_worker_config_status, ctrl["change_id"], ctrl["status"], ctrl["reason"], ) + else: + worker = w.get("name", "") + guard_key = (worker, ctrl["change_id"]) + if guard_key in self._flagged_rig_changes: + continue + allowed, first_over = self._rig_edit_within_cap(worker, time.time()) + if not allowed: + # Over cap this window — drop the row (don't add to the guard set, so its size + # stays bounded by what we actually record, not by the flood). Surface the cap + # once per window: a warning plus one marker row, its deterministic id keyed to + # this worker's window start so it's idempotent even if a restart re-trips + # `first_over`, and a fresh window later gets its own distinct marker. + if first_over: + window_start = self._rig_edit_window[worker][0] + logger.warning( + "Worker %s exceeded %d rig-edit audit rows this hour (#724) — a rig " + "reporting distinct change_ids on the unauthenticated feed; further " + "rig-edit rows are dropped until the window resets.", + worker, + _RIG_EDIT_CAP_PER_HOUR, + ) + await self._record_audit_event( + "rig-edit", + worker, + "rate-limited", + "dropped", + f"rig-edit rows capped at {_RIG_EDIT_CAP_PER_HOUR}/hour", + event_id=f"rig-edit-ratelimited-{worker}-{int(window_start)}", + ) + continue + self._flagged_rig_changes.add(guard_key) + await self._record_audit_event( + "rig-edit", + worker, + "rig-edit", + ctrl["status"], + f"change_id={ctrl['change_id']}", + event_id=f"rig-edit-{worker}-{ctrl['change_id']}", + ) def _on_clearnet_transition(self, name, ok): """Called by the supervisor after a clearnet→Tor flip attempt (#234).""" @@ -947,8 +1320,16 @@ async def run(self): worker_results = await asyncio.gather(*tasks) # 3a. Reconcile any #185 history row a slow rig rollback left stuck 'accepted' - # (#579) — rides this same poll's results, no new dial. - await self._reconcile_worker_config(worker_results) + # (#579), and flag a rig-side out-of-band edit (#530) — rides this same poll's + # results, no new dial. + await self._reconcile_worker_config(proxy_workers, worker_results) + + # 3a-2. Out-of-band audit (#530): a config.json change not made through the + # control channel, plus mirroring the #33 log into the durable audit_events + # table so the Security panel can group by hour/day/month. Both are no-ops with + # the control channel off. + await self._watch_host_config() + await self._mirror_control_audit() current_mode = self.state_manager.get_xvb_stats().get("current_mode", "P2POOL") # Determine active pool port for UI badges based on current Algo mode @@ -1111,9 +1492,14 @@ async def run(self): ) # Per-container restart/health snapshot for the crash-loop/unhealthy alert # (#337) — 9 inspect calls against the read-only docker-proxy, skipped - # entirely while Telegram is off (same cost discipline as alert_metrics). + # entirely while Telegram is off AND dashboard.fail_closed is off (same cost + # discipline as alert_metrics). fail_closed needs it even with Telegram off: + # it's the only source for "is the dashboard container itself crash-looping" + # (#490). container_states = ( - await get_container_health() if self.alert_service.enabled else {} + await get_container_health() + if (self.alert_service.enabled or DASHBOARD_FAIL_CLOSED) + else {} ) await self.alert_service.process( monero_down=monero_down, @@ -1164,6 +1550,14 @@ async def run(self): # the read-only docker-proxy. containers=container_states, ) + # 5b. Fail-closed miner hold (#490), opt-in via dashboard.fail_closed. Reads + # the DB auto-heal outcome and the dashboard's OWN crash-loop state — both + # narrow, non-transient "unrecoverable" signals — off the trackers `process` + # above just fed (see `_apply_fail_closed_gate` for what counts and why). + await self._apply_fail_closed_gate( + self.state_manager.is_db_unrecoverable() + or self.alert_service.containers.is_confirmed_bad("dashboard") + ) # Once-daily status digest, reusing the metrics built above (only when the bot # is on, which is also the only time maybe_daily_summary would send). await self.alert_service.maybe_daily_summary( @@ -1212,6 +1606,7 @@ async def run(self): "workers_rejected": self.workers_rejected, "miner_released": self.miner_released, "miner_held": self.miner_held, + "fail_closed_held": self.fail_closed_held, "clearnet_sync": self.clearnet_sync_state, "system": { "disk": disk_usage, diff --git a/build/dashboard/mining_dashboard/service/egress.py b/build/dashboard/mining_dashboard/service/egress.py index ab171600..89fc998d 100644 --- a/build/dashboard/mining_dashboard/service/egress.py +++ b/build/dashboard/mining_dashboard/service/egress.py @@ -10,7 +10,7 @@ subnet — so a container's clearnet route can't actually leave while it's on. * It does **not** cover the **host-networked dashboard** (``network_mode: host``), whose own egress (XvB stats fetch, update check, Healthchecks ping, Telegram bot, price feed, webhook/ntfy alert - sinks) bypasses ``DOCKER-USER`` entirely. Those rely solely on their SOCKS config — a clearnet + sinks, #249 XvB standby pull) bypasses ``DOCKER-USER`` entirely. Those rely solely on their SOCKS config — a clearnet route there is a real leak regardless of the firewall. (All are Tor-routed by default, so none leak.) @@ -48,6 +48,28 @@ def _notify_route(enabled, tor, private): return LOCAL if private else CLEARNET +def _xvb_standby_route(source): + """Route of the #249 backup→primary standby pull, derived from ``xvb.standby.source`` alone. + + Same #160 reasoning as ``_sinks_all_private``: only an IP literal can be *proven* to stay on your + network without a DNS lookup. So an ``.onion`` or any public/non-private source rides Tor (the + puller's ``_proxies`` sends it socks5h, like every other dashboard read); a private/loopback IP + literal is a LAN hop (``local``); an unset source is ``inactive``. A hostname can't be proven + private, so it routes over Tor — never a silent clearnet beacon. The puller reads this exact + route (``XvbStandbyPuller._proxies``), so the panel can't disagree with where the pull goes.""" + source = (source or "").strip() + if not source: + return INACTIVE + host = (urlsplit(source).hostname or "").lower() + if host.endswith(".onion"): + return TOR + try: + ip = ipaddress.ip_address(host) + except ValueError: # not an IP literal (a hostname) — unprovable, route over Tor + return TOR + return LOCAL if (ip.is_private or ip.is_loopback or ip.is_link_local) else TOR + + def _sinks_all_private(urls): """True when every configured sink URL targets a private/loopback IP literal — the LAN carve-out proof. A hostname can't be verified without a DNS lookup (which a pure config @@ -81,10 +103,12 @@ def compute_egress_posture( notify_sinks_enabled=False, notify_tor=True, notify_sinks_private=False, + xvb_standby_source="", ): """Pure derivation of the egress posture from config knobs. Returns ``{components, summary}``.""" xvb = _xvb_route(xvb_enabled, xvb_tor) sinks = _notify_route(notify_sinks_enabled, notify_tor, notify_sinks_private) + standby = _xvb_standby_route(xvb_standby_source) # ``firewalled``: is this component's egress on the container subnet the #270 firewall guards? # The dashboard is host-networked, so its own outbound traffic is NOT covered. @@ -153,6 +177,10 @@ def compute_egress_posture( # Webhook/ntfy alert sinks (#380) — Tor by default; ``notifications.tor: false`` # to an all-private-IP endpoint set is the LAN carve-out (local, not a leak). {"to": "alert sinks (webhook / ntfy)", "route": sinks}, + # XvB standby pull (#249) — a backup pulls the primary's controller state. onion or + # any non-private source rides Tor (like every read above); only a private-IP-literal + # primary is a LAN hop (local). Never clearnet, so it can't leak the backup's IP. + {"to": "XvB standby pull (backup ← primary)", "route": standby}, ], }, { @@ -207,6 +235,7 @@ def egress_posture_from_config(): healthchecks_enabled=bool(config.HEALTHCHECKS_PING_URL), telegram_enabled=config.TELEGRAM_ENABLED, price_feed_enabled=config.DASHBOARD_ENERGY["price_feed"], + xvb_standby_source=config.XVB_STANDBY_SOURCE, **_notify_knobs(), ) @@ -274,6 +303,7 @@ def compute_topology( notify_sinks_enabled=False, notify_tor=True, notify_sinks_private=False, + xvb_standby_source="", ): """Pure derivation of the stack topology. Returns ``{nodes, edges, summary}``. @@ -295,9 +325,11 @@ def compute_topology( notify_sinks_enabled=notify_sinks_enabled, notify_tor=notify_tor, notify_sinks_private=notify_sinks_private, + xvb_standby_source=xvb_standby_source, ) xvb = _xvb_route(xvb_enabled, xvb_tor) sinks = _notify_route(notify_sinks_enabled, notify_tor, notify_sinks_private) + standby = _xvb_standby_route(xvb_standby_source) sidechain = CLEARNET if p2pool_clearnet else TOR rpc = CLEARNET if remote_monero else LOCAL @@ -346,6 +378,14 @@ def compute_topology( if sinks != LOCAL else [] ), + # XvB standby pull (#249) — onion/public source rides the tor hub; a private-IP primary is a + # LAN hop with no placeable node (like the alert-sink LAN carve-out), so it draws no edge. + # The route is never clearnet, so it can never bypass the hub to the internet node. + *( + [_edge("dashboard", _ext(standby), standby, "XvB standby", "egress")] + if standby != LOCAL + else [] + ), # The Tor hub to the network: SOCKS egress for every daemon + onion-service ingress. _edge("tor", "internet", TOR, "SOCKS + onion circuits", "p2p"), # Internal mesh (hidden until expanded). @@ -390,5 +430,6 @@ def topology_from_config(): healthchecks_enabled=bool(config.HEALTHCHECKS_PING_URL), telegram_enabled=config.TELEGRAM_ENABLED, price_feed_enabled=config.DASHBOARD_ENERGY["price_feed"], + xvb_standby_source=config.XVB_STANDBY_SOURCE, **_notify_knobs(), ) diff --git a/build/dashboard/mining_dashboard/service/storage_service.py b/build/dashboard/mining_dashboard/service/storage_service.py index c2abf2a7..289a585d 100644 --- a/build/dashboard/mining_dashboard/service/storage_service.py +++ b/build/dashboard/mining_dashboard/service/storage_service.py @@ -89,6 +89,12 @@ def __init__(self, db_path: str = None): # (routed) next to what XvB *credits* (avg_1h/24h) — the live # credit-factor signal (Issue #70). "donation_fraction": 0.0, + # The controller's own last-COMMANDED donation fraction — the closed-loop + # integrator state (AlgoService.donation_fraction), distinct from the routed + # fraction above. Persisted so a restart resumes the warmed-up split instead of + # re-seeding cold from the feedforward estimate, and so a backup stack can hand it + # off on failover (#249). 0.0 until the controller first steers. + "commanded_fraction": 0.0, }, # Initialize state with default values from configuration "tiers": TIER_DEFAULTS.copy(), @@ -117,6 +123,14 @@ def __init__(self, db_path: str = None): None # {"ts", "reason", "quarantine"} of the most recent reset, for the alert ) + # True only when the auto-heal RECOVERY ITSELF just failed (disk full, permissions) — + # distinct from ``db_healthy``, which also flips false on an ordinary transient write + # error (a locked DB, a momentary I/O hiccup) that must never be treated as unrecoverable. + # This is the narrow signal `dashboard.fail_closed` (#490) gates on: a DB that a corruption + # was DETECTED for and whose rebuild then failed, not merely "a write failed once". Cleared + # on the next recovery attempt that succeeds. + self.db_unrecoverable = False + # Per-table "last successful write" health signal for the v1.7 telemetry backbone (#196 # Wave-0), mirroring db_healthy above but per table: DataService's whole poll loop is one # big try/except, so a capture hook that starts silently raising would otherwise stop @@ -219,6 +233,7 @@ def _recover_corrupt_db(self, reason: str): self._conn.row_factory = sqlite3.Row self._apply_schema() self.db_healthy = True + self.db_unrecoverable = False # a later successful attempt clears an earlier failure self.db_reset_count += 1 self.last_db_reset = {"ts": time.time(), "reason": reason, "quarantine": quarantine} self.logger.error( @@ -228,7 +243,9 @@ def _recover_corrupt_db(self, reason: str): quarantine or "(in-memory, nothing to quarantine)", ) except (sqlite3.Error, OSError) as e: - # Recovery itself failed (disk full, permissions) — leave persistence flagged unhealthy. + # Recovery itself failed (disk full, permissions) — this is the unrecoverable case + # #490's fail-closed gate watches for, distinct from an ordinary transient write error. + self.db_unrecoverable = True self._db_error("DB Recovery Error", e) def _prune_quarantined(self): @@ -260,6 +277,12 @@ def is_db_healthy(self) -> bool: """True unless a DB init or write has failed — drives the dashboard persistence badge (#131).""" return self.db_healthy + def is_db_unrecoverable(self) -> bool: + """True only when the auto-heal rebuild itself just failed (#489/#490) — narrower than + ``is_db_healthy() is False``, which also covers an ordinary transient write error. Feeds + `dashboard.fail_closed`'s miner hold; a transient blip must never trip it.""" + return self.db_unrecoverable + def _create_tables(self): """Creates necessary tables if they don't exist.""" # Per-window hashrate columns (#168) are appended so a fresh DB starts with them; existing @@ -353,6 +376,21 @@ def _create_tables(self): "CREATE TABLE IF NOT EXISTS worker_history " "(ts REAL, name TEXT, h15 REAL, accepted INTEGER, rejected INTEGER)" ) + # audit_events (#530): the durable backing store for the Security panel's audit trail. The + # #33 control.log is host-owned, read-only from this container, and the writers already trim + # it — so it can't back a month-level drill-down on its own. This table mirrors each + # control.log row (source="control", `id` reused as the primary key — INSERT OR IGNORE makes + # the mirror idempotent) AND records the two kinds this dashboard detects itself: + # source="host-edit" (config.json changed without a matching control-channel commit) and + # source="rig-edit" (a rig's control-apply outcome carries a change_id this dashboard never + # issued). `keys` is names only — never a value — the same contract as control.log itself. + # Permanent, no pruning, like blocks/payouts/disk_growth: these are human-paced admin events, + # not a hot metrics series. + self._conn.execute( + "CREATE TABLE IF NOT EXISTS audit_events " + "(id TEXT PRIMARY KEY, ts TEXT, source TEXT, actor TEXT, action TEXT, status TEXT, " + "keys TEXT)" + ) def _create_indexes(self): """Creates indexes. Called after migrations so the indexed columns are guaranteed to @@ -370,6 +408,7 @@ def _create_indexes(self): ) self._conn.execute("CREATE INDEX IF NOT EXISTS idx_disk_growth_ts ON disk_growth(ts)") self._conn.execute("CREATE INDEX IF NOT EXISTS idx_worker_history_ts ON worker_history(ts)") + self._conn.execute("CREATE INDEX IF NOT EXISTS idx_audit_events_ts ON audit_events(ts)") def _migrate_db(self): """Handles schema migrations for existing databases.""" @@ -864,6 +903,65 @@ def reconcile_worker_config_status( except sqlite3.Error as e: self._db_error("Worker Config Reconcile Error", e) + def worker_config_change_known(self, change_id: str) -> bool: + """Whether ``change_id`` was ever spooled by THIS dashboard (#530): a row exists in + ``worker_config`` — the table only ``add_worker_config_version`` writes to, one row per + change the dashboard itself sent. A rig reporting a terminal outcome for a change_id NOT + found here is reporting something it applied on its own — an out-of-band rig edit.""" + if not change_id: + return False + try: + with self._db_lock: + if not self._conn: + return False + cursor = self._conn.cursor() + cursor.execute( + "SELECT 1 FROM worker_config WHERE change_id = ? LIMIT 1", (change_id,) + ) + return cursor.fetchone() is not None + except sqlite3.Error as e: + self.logger.error(f"Worker Config Lookup Error: {e}") + return True # fail toward NOT flagging a false rig-edit on a DB read hiccup + + def add_audit_event( + self, id: str, ts: str, source: str, actor: str, action: str, status: str, keys: str + ) -> None: + """Record one audit-trail row (#530) — mirrored from the #33 control.log (``source`` = + "control", the log's own ``id`` reused as the primary key) or detected out-of-band + ("host-edit" / "rig-edit"). ``INSERT OR IGNORE`` makes both idempotent: a re-mirrored + control.log row and a re-detected out-of-band event are no-ops. ``keys`` is names only — + the caller is responsible for the same no-values contract the log itself holds to.""" + try: + with self._db_lock: + if not self._conn: + return + self._conn.execute( + "INSERT OR IGNORE INTO audit_events " + "(id, ts, source, actor, action, status, keys) VALUES (?, ?, ?, ?, ?, ?, ?)", + (id, ts, source, actor, action, status, keys), + ) + self._conn.commit() + except sqlite3.Error as e: + self._db_error("Audit Event Write Error", e) + + def get_audit_events(self, limit: int = 1000) -> list[dict[str, Any]]: + """Every persisted audit row (#530), newest first by ``ts`` — the merged, durable + backing store for the Security panel's time-grouped view.""" + try: + with self._db_lock: + if not self._conn: + return [] + cursor = self._conn.cursor() + cursor.execute( + "SELECT id, ts, source, actor, action, status, keys FROM audit_events " + "ORDER BY ts DESC LIMIT ?", + (limit,), + ) + return [dict(row) for row in cursor.fetchall()] + except sqlite3.Error as e: + self.logger.error(f"Audit Event Read Error: {e}") + return [] + def get_last_applied_worker_config(self, worker: str) -> dict[str, Any]: """The merged writable config the dashboard last successfully applied to ``worker`` — the best prefill for the editor, since the rig's enriched feed does not expose the writable config @@ -1239,6 +1337,30 @@ def get_xvb_stats(self) -> dict[str, Any]: with self._lock: return self.state["xvb"].copy() + def set_xvb_standby(self, standby: dict[str, Any]): + """Store the XvB controller state last pulled from the PRIMARY stack (#249). Held as + standby only — never folded into the live controller until this host takes over on + failover, and never acted on while the primary is authoritative (this host has no workers + then, so the controller stays on P2Pool regardless). Persisted (kv_store) so the standby + survives a backup restart. A JSON blob, mirroring ``save_snapshot``.""" + try: + self.set_kv("xvb_standby", json.dumps(standby)) + except (TypeError, ValueError) as e: + self._db_error("XvB Standby Serialization Error", e) + + def get_xvb_standby(self) -> dict[str, Any] | None: + """The last-pulled primary XvB controller state (#249), or None if a backup source was + never configured / has not fetched yet. Inspectable via ``/api/state`` so an operator can + confirm the backup is warm before a failover.""" + raw = self.get_kv("xvb_standby") + if not raw: + return None + try: + val = json.loads(raw) + return val if isinstance(val, dict) else None + except (json.JSONDecodeError, TypeError): + return None + def get_xvb_reward_estimates(self) -> dict[str, Any]: """The cached XvB per-tier reward estimates (#118): ``{"estimates": {...}, "last_update": ts}``.""" with self._lock: diff --git a/build/dashboard/mining_dashboard/service/xvb_standby.py b/build/dashboard/mining_dashboard/service/xvb_standby.py new file mode 100644 index 00000000..1d59412f --- /dev/null +++ b/build/dashboard/mining_dashboard/service/xvb_standby.py @@ -0,0 +1,138 @@ +"""Backup-stack XvB warm-standby puller (#249). + +Two full Pithead hosts sharing one Monero/Tari wallet run as a failover pair: workers list both in +``pools[]`` (primary first), and on a primary outage they fail over to the backup. The backup's XvB +donation controller would otherwise cold-start — it restarts the donation split from the feedforward +estimate and re-ramps for hours — so its credited tier over/under-shoots until the closed loop +reconverges. + +This service closes that gap on the **backup**: when ``xvb.standby.source`` names the primary's +dashboard, it periodically pulls the primary's read-only ``/api/xvb-standby`` and holds the result as +*standby* state (``StateManager.set_xvb_standby``). It is never acted on while the primary is +authoritative — the backup has no workers then, so its controller stays on P2Pool regardless. The +adoption happens only at failover, inside ``AlgoService._seed_donation_fraction``, when the backup +first actually donates. + +One-way, backup-pulls-from-primary; inert unless configured (blank source = off). The pull follows +the dashboard's #160-safe egress rule (``_proxies`` / ``egress._xvb_standby_route``): an ``.onion`` +source, a public IP, or any hostname rides the bridge Tor SOCKS — so the primary sees a Tor exit, +never the backup's real IP — and only a provably-private/loopback IP *literal* dials direct as a LAN +hop. It never opens a clearnet path. Every failure is silent — a missed pull just keeps the +last-held standby, and a backup with no standby yet simply cold-starts as before. +""" + +import asyncio +import logging +import time + +import requests + +from mining_dashboard.config.config import ( + TOR_SOCKS_PROXY, + XVB_STANDBY_INTERVAL_S, + XVB_STANDBY_SOURCE, +) +from mining_dashboard.helper.http import bounded_get +from mining_dashboard.service.egress import LOCAL, _xvb_standby_route + +logger = logging.getLogger("XvbStandby") + + +def parse_standby(payload): + """The minimal warm state from a primary's ``/api/xvb-standby`` body, or ``None`` when the + payload is unusable. Pure + unit-tested. Coerces the numeric fields defensively — the source is + another stack's API, trusted but still validated so a malformed body degrades to "no pull" + rather than poisoning the held standby.""" + if not isinstance(payload, dict): + return None + try: + commanded = float(payload.get("commanded_fraction", 0.0) or 0.0) + avg_1h = float(payload.get("avg_1h", 0.0) or 0.0) + avg_24h = float(payload.get("avg_24h", 0.0) or 0.0) + except (TypeError, ValueError): + return None + return { + "commanded_fraction": commanded, + "avg_1h": avg_1h, + "avg_24h": avg_24h, + "donation_level": str(payload.get("donation_level", "")), + "mode": str(payload.get("mode", "")), + } + + +class XvbStandbyPuller: + """Periodically pulls the primary's XvB controller state into standby (#249). + + Inert unless ``source`` is set. ``fetch_once`` does one blocking HTTP read (call via + ``asyncio.to_thread`` or straight in a test); ``run`` is the throttled loop wired into main.""" + + def __init__(self, state_manager, source=XVB_STANDBY_SOURCE, interval=XVB_STANDBY_INTERVAL_S): + self.state_manager = state_manager + self.source = (source or "").strip() + self.interval = interval + + @property + def enabled(self): + return bool(self.source) + + def _proxies(self): + """Route the pull the way the whole dashboard reads clearnet: an ``.onion`` OR any public / + not-provably-private source rides the bridge Tor SOCKS (DNS resolved proxy-side), so the + primary sees a Tor exit — never the backup's real IP (#160/#249). Only a provably-private, + loopback, or link-local IP *literal* dials direct, as a LAN hop. A hostname can't be proven + private without DNS, so it goes over Tor. This mirrors ``egress._xvb_standby_route`` exactly + (``local`` there == direct here), so the Security panel reports where this pull truly goes.""" + return ( + None + if _xvb_standby_route(self.source) == LOCAL + else { + "http": TOR_SOCKS_PROXY, + "https": TOR_SOCKS_PROXY, + } + ) + + def fetch_once(self): + """Pull the primary's ``/api/xvb-standby`` once and store it as standby. Returns the stored + blob, or ``None`` on any failure (kept silent — the last-held standby stands).""" + if not self.enabled: + return None + try: + resp = bounded_get( + self.source, + timeout=15, + proxies=self._proxies(), + headers={"User-Agent": "pithead-dashboard"}, + ) + if resp.status_code != 200: + logger.debug("Standby pull got HTTP %s (kept silent)", resp.status_code) + return None + standby = parse_standby(resp.json()) + except (requests.RequestException, ValueError) as e: + logger.debug("Standby pull failed (kept silent): %s", e) + return None + if standby is None: + return None + standby["pulled_at"] = time.time() + self.state_manager.set_xvb_standby(standby) + logger.info( + "Pulled primary XvB standby: commanded %.3f, 1h %.0f / 24h %.0f, tier %s", + standby["commanded_fraction"], + standby["avg_1h"], + standby["avg_24h"], + standby["donation_level"] or "auto", + ) + return standby + + async def run(self): + """Poll the primary on ``interval`` while configured. A no-op forever when the source is + blank, so an ordinary single-stack install pays nothing.""" + if not self.enabled: + logger.info("XvB standby puller idle (no xvb.standby.source configured).") + return + # Log only the enabled state, never the source URL — it can embed basic-auth userinfo, a + # capability secret that must not land in stdout/docker logs (mirrors healthchecks.py, which + # logs "enabled" and never the ping_url). + logger.info("Service Started: XvB standby puller (enabled).") + while True: + await asyncio.to_thread(self.fetch_once) + await asyncio.sleep(self.interval) diff --git a/build/dashboard/mining_dashboard/sim/donation_model.py b/build/dashboard/mining_dashboard/sim/donation_model.py index 49c859ed..e7bc5965 100644 --- a/build/dashboard/mining_dashboard/sim/donation_model.py +++ b/build/dashboard/mining_dashboard/sim/donation_model.py @@ -54,7 +54,9 @@ class _FixedTiers: - """Minimal StateManager stand-in: the controller only needs `get_tiers()`.""" + """Minimal StateManager stand-in for the closed-loop sim: `get_tiers()`, plus the cold + warm-resume reads (#249) so the controller seeds from feedforward — the simulator models a + fresh cold start, never a restart or failover, so both return the "no warm state" values.""" def __init__(self, tiers=None): self._tiers = dict(tiers or TIER_DEFAULTS) @@ -62,6 +64,12 @@ def __init__(self, tiers=None): def get_tiers(self): return dict(self._tiers) + def get_xvb_stats(self): + return {"commanded_fraction": 0.0} + + def get_xvb_standby(self): + return None + def make_algo_controller(algo, p2pool_difficulty=0) -> Controller: """Adapt a real `AlgoService` into a `decide(...) -> fraction` callable. diff --git a/build/dashboard/mining_dashboard/web/server.py b/build/dashboard/mining_dashboard/web/server.py index a5b3158c..04a18559 100644 --- a/build/dashboard/mining_dashboard/web/server.py +++ b/build/dashboard/mining_dashboard/web/server.py @@ -55,6 +55,31 @@ async def handle_state(request): return web.json_response({"error": "Failed to build dashboard state."}, status=500) +async def handle_xvb_standby(request): + """The XvB controller state a BACKUP stack pulls to warm its donation split (#249). + + Read-only, behind the same Caddy auth + loopback bind as ``/api/state`` — no new trust boundary + or egress class. The payload is the minimal warm state: the controller's commanded donation + fraction, XvB's credited 1h/24h averages (already public per-wallet), the target tier, and the + current mode. Always registered; on a stack with no backup nothing ever pulls it.""" + state_mgr = request.app["state_manager"] + try: + xvb = state_mgr.get_xvb_stats() + return web.json_response( + { + "commanded_fraction": xvb.get("commanded_fraction", 0.0), + "avg_1h": xvb.get("avg_1h", 0.0), + "avg_24h": xvb.get("avg_24h", 0.0), + "mode": xvb.get("current_mode", ""), + "donation_level": config.XVB_DONATION_LEVEL, + "ts": xvb.get("last_update", 0.0), + } + ) + except Exception: + logger.exception("Error building XvB standby state") + return web.json_response({"error": "Failed to build XvB standby state."}, status=500) + + async def handle_metrics(request): """Prometheus text exposition (#379), rendered from the same ``build_metrics`` snapshot ``/api/state`` uses — live gauges only, no history. Same trust boundary as the state API: @@ -148,7 +173,12 @@ async def handle_control_commit(request): try: body = await request.json() rid = control_service.submit( - "commit", actor=request.headers.get("X-Auth-User", ""), intent_id=body.get("id") + "commit", + actor=request.headers.get("X-Auth-User", ""), + intent_id=body.get("id"), + # #719: the operator's typed confirmation for an in-scope disruptive change. It is + # friction, not a secret — the host gate requires it before a CONFIRM row proceeds. + confirm=body.get("confirm"), ) except Exception: raise web.HTTPBadRequest(text="Body must be JSON with a valid intent 'id'.") from None @@ -308,11 +338,35 @@ async def handle_control_result(request): # charset) before it reaches the browser — log content is attacker-influenceable input. +def _merged_audit_entries(state_mgr): + """The Security panel's full audit feed (#530): the #33 log's live tail (read directly, so a + commit that landed since the last poll cycle shows immediately — no mirror lag) UNION the + durable ``audit_events`` table (the mirrored log history PLUS the out-of-band host-edit/rig-edit + detections, which never appear in control.log at all). Deduplicated by ``id`` — a control.log + row already mirrored to the DB is identical either way, so the DB copy wins and the direct log + read is skipped for it. Entries with no ``id`` (a handful of pre-auth "invalid"/"refused" rows, + #33) are never mirrored and so appear only while still in the log's own tail — a disclosed, minor + gap, not a bug. Sorted newest first by ``ts`` (both sources share one string format, so this is a + plain lexical sort, no parsing).""" + merged = {e["id"]: e for e in state_mgr.get_audit_events() if e.get("id")} + for e in audit_service.recent_changes(): + eid = e.get("id") + if eid and eid not in merged: + merged[eid] = {**e, "source": "control"} + elif not eid: + # No stable id to dedupe on — always shown live from the log tail (never mirrored). + merged[f"log-{id(e)}"] = {**e, "source": "control"} + return sorted(merged.values(), key=lambda e: e.get("ts", ""), reverse=True) + + async def handle_audit_log(request): - """Recent config-change audit entries, from the read-only /control/audit mount. Registered - only alongside the control channel — the log is a #33 artifact.""" + """Config-change audit entries — the #33 control-channel log plus the out-of-band host-edit / + rig-edit detections (#530), merged and persisted so the Security panel can group by hour/day/ + month deeper than the log's own trimmed tail. Registered only alongside the control channel — + the log is a #33 artifact and the out-of-band watchers only run when it's on.""" try: - return web.json_response({"entries": audit_service.recent_changes()}) + state_mgr = request.app["state_manager"] + return web.json_response({"entries": _merged_audit_entries(state_mgr)}) except Exception: logger.exception("Error reading the control audit log") return web.json_response({"error": "Failed to read the audit log."}, status=500) @@ -371,6 +425,9 @@ def create_app(state_manager, latest_data_ref): [ web.get("/", handle_index), web.get("/api/state", handle_state), + # Warm-standby state a backup stack pulls on failover (#249). Read-only, same auth as + # /api/state; harmless (and unread) on a stack with no backup. + web.get("/api/xvb-standby", handle_xvb_standby), web.get("/metrics", handle_metrics), web.get("/api/access", handle_access_log), ] diff --git a/build/dashboard/mining_dashboard/web/static/configlogic.mjs b/build/dashboard/mining_dashboard/web/static/configlogic.mjs index de4707c9..19c938fa 100644 --- a/build/dashboard/mining_dashboard/web/static/configlogic.mjs +++ b/build/dashboard/mining_dashboard/web/static/configlogic.mjs @@ -127,7 +127,9 @@ export const LOGICAL_GROUPS = [ "p2pool.clearnet", "proxy", "xvb", + "local_miner", "dashboard.tari_required", + "dashboard.fail_closed", ], }, { name: "Workers", prefixes: ["workers"] }, @@ -286,11 +288,20 @@ export function nestSection(section) { // CONTROL_DASHBOARD_EDITABLE_KEYS). A missing/empty set fails CLOSED — nothing is marked // editable — rather than defaulting to "everything editable" and silently reintroducing the // edit-then-reject problem this feature exists to remove. -export function markEditable(sections, editableKeys) { +// +// `confirmKeys` is `_confirm_keys` (#719): the operationally-disruptive paths the gate WILL commit, +// but only behind a type-to-confirm. They render editable (not greyed) and carry `confirm: true` so +// the field can show a "confirm to proceed" affordance instead of the "host-only" one. A key on +// both lists is treated as freely editable (editable wins); the gate is still the authority. +export function markEditable(sections, editableKeys, confirmKeys) { const editable = new Set(editableKeys || []); + const confirm = new Set(confirmKeys || []); return sections.map((s) => ({ name: s.name, - fields: s.fields.map((f) => ({ ...f, editable: editable.has(f.key) })), + fields: s.fields.map((f) => { + const isConfirm = !editable.has(f.key) && confirm.has(f.key); + return { ...f, editable: editable.has(f.key) || isConfirm, confirm: isConfirm }; + }), })); } diff --git a/build/dashboard/mining_dashboard/web/static/configview.mjs b/build/dashboard/mining_dashboard/web/static/configview.mjs index a34ef4d0..1501cb25 100644 --- a/build/dashboard/mining_dashboard/web/static/configview.mjs +++ b/build/dashboard/mining_dashboard/web/static/configview.mjs @@ -73,6 +73,9 @@ async function pollResult(id, skip, max = POLL_MAX) { } const HOST_ONLY_TITLE = "Host-only — edit config.json and run ./pithead apply"; +// #719: an in-scope confirm-gated field IS editable, but committing it is disruptive — the review +// modal makes you type APPLY. The tooltip sets that expectation up front. +const CONFIRM_TITLE = "Editable — this change is disruptive; you'll type APPLY to confirm at Save"; // `full` (#529): the pinned Core card mixes fields from several sections, so its rows need the // FULL dotted key ("monero.wallet_address") to stay unambiguous. A natural section keeps the @@ -90,7 +93,7 @@ const Field = ({ field, edits, onEdit, full }) => { const editable = field.editable !== false; const value = field.key in edits ? edits[field.key] : field.value; const label = full ? field.key : field.path.slice(1).join(".") || field.path[0]; - const title = editable ? undefined : HOST_ONLY_TITLE; + const title = !editable ? HOST_ONLY_TITLE : field.confirm ? CONFIRM_TITLE : undefined; const change = editable ? (e) => onEdit(field.key, e.target.value) : undefined; let input; if (field.type === "boolean") { @@ -133,10 +136,13 @@ export const PreviewModal = ({ changes.length === 0 ? html`

No configuration changes detected.

` : html`` } ${ @@ -166,6 +172,7 @@ export class ConfigView extends Component { sections: [], coreKeys: [], editableKeys: [], // #613: config paths the control gate will actually commit + confirmKeys: [], // #719: config paths the gate commits behind a type-to-confirm edits: {}, mode: loadPref("dashboardConfigMode", ["form", "json"], "form"), // form | json (#529, persisted #658) editText: "", @@ -201,6 +208,7 @@ export class ConfigView extends Component { sections: buildSections(cfg), coreKeys: cfg._core_keys || [], editableKeys: cfg._editable_keys || [], + confirmKeys: cfg._confirm_keys || [], edits: {}, editText: JSON.stringify(cfg, null, 2), jsonError: null, @@ -276,10 +284,15 @@ export class ConfigView extends Component { const id = this.state.preview.id; this.setState({ phase: "committing" }); try { + // #719: an in-scope disruptive change (preview.destructive) rides its typed confirmation to + // the host gate, which requires it before a CONFIRM row proceeds. Friction, not a secret. + const body = this.state.preview.destructive + ? { id, confirm: this.state.confirmText } + : { id }; const res = await fetch("/api/control/commit", { method: "POST", headers: CONTROL_HEADERS, - body: JSON.stringify({ id }), + body: JSON.stringify(body), }); if (!res.ok && res.status !== 202) throw new Error(`HTTP ${res.status}`); let out = await res.json(); @@ -351,6 +364,7 @@ export class ConfigView extends Component { sections, coreKeys, editableKeys, + confirmKeys, edits, mode, editText, @@ -394,7 +408,10 @@ export class ConfigView extends Component { const busy = phase === "previewing" || phase === "committing"; const dirty = Object.keys(edits).length > 0; const canSave = mode === "json" ? !jsonError : dirty; - const { core, sections: groups } = regroupCore(markEditable(sections, editableKeys), coreKeys); + const { core, sections: groups } = regroupCore( + markEditable(sections, editableKeys, confirmKeys), + coreKeys, + ); return html`
${error ? html`

${error}

` : null}
diff --git a/build/dashboard/mining_dashboard/web/static/dashboard.css b/build/dashboard/mining_dashboard/web/static/dashboard.css index 97e59c1c..7aabc07a 100644 --- a/build/dashboard/mining_dashboard/web/static/dashboard.css +++ b/build/dashboard/mining_dashboard/web/static/dashboard.css @@ -419,6 +419,26 @@ tr:last-child td { white-space: nowrap; } +/* The audit trail's title + its hour/day/month grouping select (#530), side by side. */ +.card-header-row { + display: flex; + justify-content: space-between; + align-items: center; + gap: 12px; + margin-bottom: 8px; +} +.card-header-row h3 { + margin: 0; +} + +/* Audit-trail time-bucket header row (#530), between groups when the operator picks + * hour/day/month grouping. A raised, muted divider — readable, not another data row. */ +.audit-group-header td { + background: var(--elevated); + color: var(--text-muted); + font-weight: 600; +} + /* Components */ .status-ok { color: var(--ok); diff --git a/build/dashboard/mining_dashboard/web/static/securityview.mjs b/build/dashboard/mining_dashboard/web/static/securityview.mjs index 992bc753..b79cad92 100644 --- a/build/dashboard/mining_dashboard/web/static/securityview.mjs +++ b/build/dashboard/mining_dashboard/web/static/securityview.mjs @@ -1,8 +1,9 @@ // Security panel (#349): recent dashboard accesses (Caddy's access log, read-only) and the -// config-change audit trail (the #33 host-side audit log, read-only). Both APIs serve -// server-sanitized fields — the backend whitelists every character before it leaves the host -// logs — and everything here renders through Preact text nodes, never markup, so a hostile log -// line stays inert even if the server-side filter regressed. +// config-change audit trail (the #33 host-side audit log, read-only, plus the #530 out-of-band +// host-edit/rig-edit detections and their persisted history). Both APIs serve server-sanitized +// fields — the backend whitelists every character before it leaves the host logs — and everything +// here renders through Preact text nodes, never markup, so a hostile log line stays inert even if +// the server-side filter regressed. // // The operator story: over Tor there is no source IP, so the attack signal is the RATE of 401s. // A burst of failed logins shows a rotate nudge — change the password, or mint a fresh onion @@ -12,6 +13,36 @@ import { Component, html } from "./preact.mjs"; const ACCESS_LIMIT_SHOWN = 20; +// Audit entries share one ts format across every source (control.log's own writer and the #530 +// watchers both emit "YYYY-MM-DDTHH:MM:SSZ", see data_service._iso_now), so a bucket key is a +// plain string slice — no date parsing, no timezone math. +export function bucketKey(ts, granularity) { + if (typeof ts !== "string") return ""; + if (granularity === "hour") return ts.slice(0, 13); + if (granularity === "month") return ts.slice(0, 7); + return ts.slice(0, 10); // "day" +} + +// Group already newest-first ``entries`` into contiguous {bucket, entries} runs for +// ``granularity`` ("hour"|"day"|"month"), or one ungrouped run for "flat"/anything else — the +// drill: pick "month" to scan a year at a glance, "hour" to pin down one incident. +export function groupAuditEntries(entries, granularity) { + if (granularity !== "hour" && granularity !== "day" && granularity !== "month") { + return [{ bucket: null, entries }]; + } + const groups = []; + let current = null; + for (const e of entries) { + const key = bucketKey(e.ts, granularity); + if (!current || current.bucket !== key) { + current = { bucket: key, entries: [] }; + groups.push(current); + } + current.entries.push(e); + } + return groups; +} + // Epoch seconds -> local "YYYY-MM-DD HH:MM:SS"-style string; blank for a missing/zero ts. export function fmtEpoch(ts) { if (!Number.isFinite(ts) || ts <= 0) return ""; @@ -63,11 +94,33 @@ const AccessCard = ({ access }) => {
`; }; -const AuditCard = ({ audit }) => { +// Outcome values a "detected" (never applied/rejected) out-of-band row never has, so it keeps its +// own neutral styling instead of picking up the "applied" green. +const AuditRow = (e) => html` + ${e.ts} + ${e.actor} + ${e.action} + ${e.status} + ${e.keys} +`; + +const AuditCard = ({ audit, group, onGroupChange }) => { // null = control channel off (the /api/audit route 404s) — no card at all. if (!audit) return null; return html`
-

Recent config changes

+
+

Recent config changes

+ ${ + audit.length > 0 + ? html`` + : null + } +
${ audit.length === 0 ? html`

No config changes have gone through the dashboard yet.

` @@ -75,15 +128,14 @@ const AuditCard = ({ audit }) => { - ${audit.map( - (e) => html` - - - - - - `, - )} + ${groupAuditEntries(audit, group).flatMap((g) => [ + g.bucket !== null + ? html` + + ` + : null, + ...g.entries.map(AuditRow), + ])}
Time (UTC)UserActionOutcomeSettings
${e.ts}${e.actor}${e.action}${e.status}${e.keys}
${g.bucket} (${g.entries.length})
` @@ -94,7 +146,10 @@ const AuditCard = ({ audit }) => { export class SecurityPanel extends Component { constructor(props) { super(props); - this.state = { access: null, audit: null, error: null }; + // auditGroup: "flat" (today's plain newest-first list) is the default so existing behavior + // doesn't change until the operator opts into grouping (#530). + this.state = { access: null, audit: null, auditGroup: "flat", error: null }; + this.setAuditGroup = (group) => this.setState({ auditGroup: group }); } async componentDidMount() { @@ -110,11 +165,11 @@ export class SecurityPanel extends Component { } render() { - const { access, audit, error } = this.state; + const { access, audit, auditGroup, error } = this.state; if (error) return html`

${error}

`; return html`
<${AccessCard} access=${access} /> - <${AuditCard} audit=${audit} /> + <${AuditCard} audit=${audit} group=${auditGroup} onGroupChange=${this.setAuditGroup} />
`; } } diff --git a/build/dashboard/mining_dashboard/web/views.py b/build/dashboard/mining_dashboard/web/views.py index d3921f8b..4acd31a9 100644 --- a/build/dashboard/mining_dashboard/web/views.py +++ b/build/dashboard/mining_dashboard/web/views.py @@ -335,6 +335,77 @@ def _window_reject_pct(rows, seconds): return "—" if pct is None else f"{pct:.2f}%" +# -------------------------------------------------------------------------------------- +# #196 Tier-1 telemetry backbone: blocks / disk_growth / xvb_history surfaced on /api/state. +# The backbone (capture + storage + retention, PR #600) shipped without this exposure step — +# these three formatters are it. network_history and worker_history are Tier-2 (a separate +# slice of the epic) and are not exposed here. +# -------------------------------------------------------------------------------------- + + +def _downsample_gauge_rows(rows, value_cols, target=_MAX_CHART_POINTS): + """Bucket-average arbitrary point-in-time (gauge) columns down to ``target`` points. + + Mirrors ``_downsample_share_stats``, but averages instead of summing: these rows are + periodic READINGS (disk size, XvB credited averages), not per-interval deltas, so summing + them would inflate the series instead of thinning it. A no-op when already at/under target.""" + if len(rows) <= target: + return rows + chunk_size = len(rows) / target + out = [] + for i in range(target): + chunk = rows[int(i * chunk_size) : int((i + 1) * chunk_size)] + if not chunk: + continue + bucket = {"ts": chunk[len(chunk) // 2]["ts"]} + for col in value_cols: + vals = [r.get(col, 0) or 0 for r in chunk] + bucket[col] = round(sum(vals) / len(vals), 2) + out.append(bucket) + return out + + +def build_blocks(blocks, range_arg, window=None): + """Persisted P2Pool block-found events (#196) as chart-ready points, restricted to the + selected range/window (``_filter_events`` bounds any ts-keyed list, so this table reuses + it as-is). A handful of rows a week — no downsampling needed.""" + return [ + { + "x": int(b["ts"] * 1000), + "height": b.get("height", 0), + "difficulty": b.get("difficulty", 0), + } + for b in _filter_events(blocks, range_arg, window) + ] + + +def _gauge_series(rows, range_arg, window, value_cols): + """Shared shape for a persisted gauge series (#196): filter to the selected range/window, + bucket-average past ``_MAX_CHART_POINTS`` like ``share_stats``, and key each row's own + ``value_cols`` under ``x`` (ms epoch). ``build_disk_growth``/``build_xvb_history`` are this + with their own column set — the only thing that differs between them.""" + filtered = _downsample_gauge_rows(_filter_events(rows, range_arg, window), value_cols) + return [{"x": int(r["ts"] * 1000), **{c: r.get(c, 0) for c in value_cols}} for r in filtered] + + +def build_disk_growth(rows, range_arg, window=None): + """Persisted hourly monerod-DB-size + host-disk-usage samples (#196) as chart-ready points — + the table keeps every row (no retention prune), so a long-lived install can otherwise pass + the chart-point cap ``_gauge_series`` bounds it at.""" + return _gauge_series( + rows, range_arg, window, ("monero_db_bytes", "disk_used_gb", "disk_total_gb") + ) + + +def build_xvb_history(rows, range_arg, window=None): + """Persisted ~5-minute XvB-credited scalar samples (#196) as chart-ready points — the 30-day + retention at this cadence is ~8.6k rows, well past the chart-point cap ``_gauge_series`` + bounds it at.""" + return _gauge_series( + rows, range_arg, window, ("avg_1h", "avg_24h", "fail_count", "donation_fraction") + ) + + def _window_duration(filtered_history, range_arg, window): """Seconds the chart currently spans — drives adaptive resolution/smoothing. From the window if zoomed, else the preset length, else (``all``/unknown) the actual data extent.""" @@ -1224,6 +1295,17 @@ def build_badges(data, metrics, mode_variant, db_healthy=True, wallet_change=Non "title": "p2pool and xmrig-proxy are held until the required chains finish syncing", } ) + # Fail-closed miner hold on an unrecoverable health failure (Issue #490), opt-in via + # dashboard.fail_closed. Distinct from the sync-gate badge above — this fires post-sync. + if data.get("fail_closed_held"): + badges.append( + { + "text": "Miner held (fail-closed)", + "variant": "bad", + "title": "dashboard.fail_closed is on and an unrecoverable health failure is " + "holding p2pool and xmrig-proxy until it clears", + } + ) # Non-blocking Tari (Issue #51): stay operational, surface a top-bar badge with the live # percentage once known (omitted early so it isn't a stale "0%"). if data.get("tari_syncing_passive"): @@ -1703,6 +1785,10 @@ def build_state(data, state_mgr, range_arg, window=None, avg_window=DEFAULT_HASH xvb_day=xvb_current_tier_reward_day(metrics, state_mgr), ), "xvb_calc": build_xvb_calc(metrics, state_mgr), + # On a backup stack, the XvB controller state last pulled from the primary (#249) — held as + # standby, adopted only at failover. None on a single stack (nothing pulls). Inspectable so + # an operator can confirm the backup is warm before it takes over. + "xvb_standby": state_mgr.get_xvb_standby(), "tari": build_tari(data), "workers": build_workers(data.get("workers", []), data.get("rigforge_release")), # Fleet power draw / efficiency and (once a price is set) net profit after power (#260), @@ -1713,6 +1799,12 @@ def build_state(data, state_mgr, range_arg, window=None, avg_window=DEFAULT_HASH # proxy_summary so its (cumulative) shape stays unchanged for existing clients. "share_stats": build_share_stats(share_stats, range_arg, window), "reject_pct_24h": _window_reject_pct(share_stats, 24 * 3600), + # #196 Tier-1 telemetry backbone exposure: block-found events, hourly disk-growth + # samples, and ~5-min XvB-credited samples. No chart renders these yet — that's the + # deliberate next slice — the payload just carries the persisted series. + "blocks": build_blocks(state_mgr.get_blocks(), range_arg, window), + "disk_growth": build_disk_growth(state_mgr.get_disk_growth(), range_arg, window), + "xvb_history": build_xvb_history(state_mgr.get_xvb_history(), range_arg, window), "egress": egress, "topology": topology, "chart": build_chart( diff --git a/build/dashboard/pyproject.toml b/build/dashboard/pyproject.toml index f56d129d..0a89cadb 100644 --- a/build/dashboard/pyproject.toml +++ b/build/dashboard/pyproject.toml @@ -7,13 +7,13 @@ name = "mining-dashboard" # Keep in lockstep with the top-level VERSION file — the single source of truth for the stack version # (#44). A shell test (tests/stack/run.sh) fails if these drift; the dashboard *displays* the version # from VERSION (baked in as PITHEAD_VERSION, #58), so this is packaging metadata only. -version = "1.10.2" +version = "1.11.0" description = "Monitoring dashboard and XvB switching engine for Pithead" readme = "README.md" requires-python = ">=3.11" license = { text = "MIT" } dependencies = [ - "aiohttp>=3.10.11", + "aiohttp>=3.14.2", # [socks] pulls in PySocks so the XvB stats fetch can route over the Tor SOCKS proxy (#163). "requests[socks]>=2.34.2", # grpcio/protobuf floors are dictated by the checked-in Tari gRPC stubs @@ -34,11 +34,11 @@ test = [ # diff-cover (#286): patch-coverage gate — new/changed lines must be >=90% covered. "diff-cover>=9", # hypothesis (#284): property-based tests asserting invariants on the money/numeric logic. - "hypothesis>=6.156.6", + "hypothesis>=6.158.0", ] # Developer tooling (Wave 7, #280). Pinned so local, pre-commit, and CI all run the SAME ruff # — lint output is version-sensitive, so a floor would let CI and a contributor disagree. -dev = ["ruff==0.15.21", "pre-commit>=4"] +dev = ["ruff==0.15.22", "pre-commit>=4"] [tool.setuptools.packages.find] include = ["mining_dashboard*"] diff --git a/build/dashboard/tests/config/test_config.py b/build/dashboard/tests/config/test_config.py index 34e59d8d..75aebdc0 100644 --- a/build/dashboard/tests/config/test_config.py +++ b/build/dashboard/tests/config/test_config.py @@ -41,6 +41,19 @@ def test_monero_prune_accepts_falsy_forms(self): with patch.dict(os.environ, {"MONERO_PRUNE": v}): assert _reload_config().MONERO_PRUNE is False, f"{v!r} should be full" + def test_fail_closed_defaults_off(self): + # #490: the dashboard is an observability layer, so a cosmetic fault must never idle the + # fleet by default — absent DASHBOARD_FAIL_CLOSED must read False. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DASHBOARD_FAIL_CLOSED", None) + assert _reload_config().DASHBOARD_FAIL_CLOSED is False + + def test_fail_closed_env_override(self): + with patch.dict(os.environ, {"DASHBOARD_FAIL_CLOSED": "true"}): + assert _reload_config().DASHBOARD_FAIL_CLOSED is True + with patch.dict(os.environ, {"DASHBOARD_FAIL_CLOSED": "false"}): + assert _reload_config().DASHBOARD_FAIL_CLOSED is False + def test_update_interval_tolerates_bad_values(self): # A malformed override must fall back to the default, not crash the dashboard at import. for v, expected in [("2", 2), ("2.5", 2), ("", 30), ("nonsense", 30)]: diff --git a/build/dashboard/tests/frontend/configlogic.test.mjs b/build/dashboard/tests/frontend/configlogic.test.mjs index 47ea74d0..44eebfdb 100644 --- a/build/dashboard/tests/frontend/configlogic.test.mjs +++ b/build/dashboard/tests/frontend/configlogic.test.mjs @@ -314,6 +314,25 @@ test("markEditable: host-only fields (e.g. dashboard.auth.password, a security/s assert.equal(password.editable, false); }); +// --- Confirm-gated set (#719) ------------------------------------------------------------------- + +const fieldsByKey = (sections) => + Object.fromEntries(sections.flatMap((s) => s.fields).map((f) => [f.key, f])); + +test("markEditable: a confirm-gated key is editable AND flagged confirm; host-only stays neither", () => { + const byKey = fieldsByKey(markEditable(buildSections(CFG), [], ["monero.prune"])); + assert.equal(byKey["monero.prune"].editable, true); // editable... + assert.equal(byKey["monero.prune"].confirm, true); // ...but confirm-to-proceed + assert.equal(byKey["monero.wallet_address"].editable, false); // host-only untouched + assert.equal(byKey["monero.wallet_address"].confirm, false); +}); + +test("markEditable: editable wins over confirm — a key on both lists is freely editable, not gated", () => { + const byKey = fieldsByKey(markEditable(buildSections(CFG), ["monero.prune"], ["monero.prune"])); + assert.equal(byKey["monero.prune"].editable, true); + assert.equal(byKey["monero.prune"].confirm, false); // no needless friction on a freely-editable key +}); + // --- JSON mode's whole-config parse (#529) ---------------------------------------------------- test("parseConfigJson: valid JSON builds the same staged config shape applyEdits does", () => { diff --git a/build/dashboard/tests/frontend/configview.test.mjs b/build/dashboard/tests/frontend/configview.test.mjs index d6b85715..4f6e238c 100644 --- a/build/dashboard/tests/frontend/configview.test.mjs +++ b/build/dashboard/tests/frontend/configview.test.mjs @@ -192,6 +192,37 @@ test("an empty preview leaves Confirm disabled", () => { assert.match(out, /disabled/); // nothing to commit }); +// --- Confirm-gated disruptive change in the modal (#719) -------------------------------------- +// +// An in-scope disruptive change previews as destructive (a CONFIRM row). The modal must warn (⚠), +// show the type-APPLY box, and keep Confirm disabled until the operator types the literal APPLY. +const CONFIRM_PREVIEW = { + changes: [ + { flag: "CONFIRM", key: "monero.clearnet_initial_sync", msg: "Clearnet initial sync ENABLED — host IP exposed during IBD." }, + ], + destructive: true, +}; + +test("a CONFIRM change warns and gates Confirm behind the typed APPLY (#719)", () => { + const notYet = renderToString( + PreviewModal({ preview: CONFIRM_PREVIEW, confirmText: "", busy: false }), + ); + assert.match(notYet, /⚠/); // the disruptive row is warned + assert.match(notYet, /host IP exposed during IBD/); + assert.match(notYet, /Type APPLY<\/code> to confirm/); // the type-to-confirm box is shown + // Confirm is still disabled — the confirm-apply button carries `disabled` until APPLY is typed. + const btnNotYet = notYet.match(/