From c11c138ad00c53db7d5f1a16be4cc3c74a5a9a74 Mon Sep 17 00:00:00 2001 From: agriev Date: Thu, 7 May 2026 22:19:20 +0200 Subject: [PATCH 1/2] ci(supply-chain): pin Dockerfile base images by digest + extend Rust CI to ha-main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin debian:bookworm-slim and cubejs/rust-builder:bookworm-llvm-18 by sha256 digest so a tag rewrite upstream can't silently change what we build. Add scripts/pin-base-images.sh as a tooled refresh path — intentional roll-forward becomes a reviewable diff. Also fire the Rust master workflow on ha-main pushes so the HA fork catches Cargo.lock drift on every merge instead of only at the next upstream sync. --- .github/workflows/rust-cubestore-master.yml | 3 + rust/cubestore/Dockerfile | 7 ++- rust/cubestore/scripts/pin-base-images.sh | 61 +++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100755 rust/cubestore/scripts/pin-base-images.sh diff --git a/.github/workflows/rust-cubestore-master.yml b/.github/workflows/rust-cubestore-master.yml index 003509248e6c5..6611fb28d013e 100644 --- a/.github/workflows/rust-cubestore-master.yml +++ b/.github/workflows/rust-cubestore-master.yml @@ -6,6 +6,9 @@ on: - 'rust/cubestore/**' branches: - master + # PR-S1: also gate ha-main so the HA fork's Cargo.lock drift is + # caught on every push, not just the next upstream merge. + - ha-main env: CARGO_INCREMENTAL: 0 diff --git a/rust/cubestore/Dockerfile b/rust/cubestore/Dockerfile index f8c9bef78976d..6d1959d3bde64 100644 --- a/rust/cubestore/Dockerfile +++ b/rust/cubestore/Dockerfile @@ -1,4 +1,7 @@ -FROM cubejs/rust-builder:bookworm-llvm-18 AS builder +# Base images are pinned by digest to defend against tag rewrite (PR-S1). +# Refresh both via `make pin-base-images` (resolves current digests from the +# upstream registry) when intentionally rolling forward. +FROM cubejs/rust-builder:bookworm-llvm-18@sha256:9545a70ec86854a9f2aa5cd24f1b86b4efec8d46ba37450bb09ac7f25c1e2d3a AS builder WORKDIR /build/cubestore @@ -28,7 +31,7 @@ COPY cubestore/cubestore cubestore RUN [ "$WITH_AVX2" -eq "1" ] && export RUSTFLAGS="-C target-feature=+avx2"; \ cargo build --release -p cubestore -FROM debian:bookworm-slim +FROM debian:bookworm-slim@sha256:f9c6a2fd2ddbc23e336b6257a5245e31f996953ef06cd13a59fa0a1df2d5c252 WORKDIR /cube diff --git a/rust/cubestore/scripts/pin-base-images.sh b/rust/cubestore/scripts/pin-base-images.sh new file mode 100755 index 0000000000000..620964fbf37ea --- /dev/null +++ b/rust/cubestore/scripts/pin-base-images.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# pin-base-images.sh +# +# Resolves current upstream digests for the Cube Store Dockerfile's base +# images and rewrites the FROM lines in-place. Use this when intentionally +# rolling forward (e.g. picking up a debian:bookworm-slim CVE patch). +# +# Refuses to run if the working tree is dirty under rust/cubestore/Dockerfile. +# Prints a diff and exits 0 even if there is nothing to update. +# +# Requires: bash, git, docker (with `docker buildx imagetools`). + +set -euo pipefail + +cd "$(dirname "$0")/.." # rust/cubestore + +DOCKERFILE="Dockerfile" + +if ! command -v docker >/dev/null; then + echo "error: docker not found in PATH" >&2 + exit 1 +fi + +if ! git diff --quiet -- "$DOCKERFILE"; then + echo "error: $DOCKERFILE has uncommitted changes; commit or stash first" >&2 + exit 1 +fi + +resolve_digest() { + local image="$1" + docker buildx imagetools inspect "$image" 2>/dev/null \ + | awk '/^Digest: / {print $2; exit}' +} + +# Pin pairs: :: +pairs=( + "cubejs/rust-builder:bookworm-llvm-18::FROM cubejs/rust-builder:bookworm-llvm-18" + "debian:bookworm-slim::FROM debian:bookworm-slim" +) + +for entry in "${pairs[@]}"; do + image="${entry%%::*}" + anchor="${entry##*::}" + digest=$(resolve_digest "$image") + if [ -z "$digest" ]; then + echo "error: could not resolve digest for $image" >&2 + exit 1 + fi + printf 'pinning %s -> %s\n' "$image" "$digest" + # Replace the entire @sha256:... suffix (or absent suffix) with the new + # digest. Single-line, no extended regex needed. + sed -i.bak -E \ + "s|(${anchor//\//\\/})(@sha256:[a-f0-9]+)?|\1@${digest}|" \ + "$DOCKERFILE" +done + +rm -f "${DOCKERFILE}.bak" +git diff -- "$DOCKERFILE" || true +echo +echo "Done. Review the diff, then commit:" +echo " git add $DOCKERFILE && git commit -m 'chore(docker): refresh base image digests'" From d4c76c09a02e477f13c256c56a9d133b51340000 Mon Sep 17 00:00:00 2001 From: agriev Date: Thu, 7 May 2026 22:44:18 +0200 Subject: [PATCH 2/2] =?UTF-8?q?feat(cubestore/raft):=20production=20harden?= =?UTF-8?q?ing=20=E2=80=94=20propose=20timeout,=20parse-fail=20drop,=20dis?= =?UTF-8?q?k-full=20runbook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-S2 of the production hardening series. State machine: - propose() now bounded by CUBESTORE_RAFT_PROPOSE_TIMEOUT_SECS (default 30s; 0 disables). A stalled apply path or a partition where the local replica isn't actually leader anymore used to block the caller — typically an HTTP handler — forever. On timeout the propose returns a CubeError so the caller can return 503, retry against the current leader, or fail the request. Transport: - The recv loop's per-frame protobuf decode used to `continue` forever on malformed input. A peer flooding garbage frames could pin the task indefinitely. New MAX_CONSECUTIVE_PARSE_FAILS=16 trips the loop into Err on the 17th consecutive failure, which closes the TCP connection and lets the next message reconnect from a clean state. Counter resets on a successful frame so transient corruption (one bad packet, no flood) is still tolerated. Docs: - New docs/ha/RAFT-DISK-FULL-RUNBOOK.md covers the three drive_ready panic sites — they're correct by design (panic > silent commit loss) but operators need a clear recovery script. Includes PVC sizing math, single-pod recovery, all-routers wedged recovery, and the slow-fsync (vs hard-fail) symptom guide. Tests: full raft:: suite still 101/101 green. Build: protobuf@21 toolchain on macOS, debian:bookworm-slim on CI. --- docs/ha/RAFT-DISK-FULL-RUNBOOK.md | 150 ++++++++++++++++++ .../cubestore/src/raft/state_machine.rs | 39 ++++- .../cubestore/cubestore/src/raft/transport.rs | 27 +++- 3 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 docs/ha/RAFT-DISK-FULL-RUNBOOK.md diff --git a/docs/ha/RAFT-DISK-FULL-RUNBOOK.md b/docs/ha/RAFT-DISK-FULL-RUNBOOK.md new file mode 100644 index 0000000000000..52ff62e409e85 --- /dev/null +++ b/docs/ha/RAFT-DISK-FULL-RUNBOOK.md @@ -0,0 +1,150 @@ +# Raft disk-full / fsync failure — runbook + +Scope: `RaftMetaStore` running on the agriev/cube fork (`cubestored` +HA image). Three call sites in `raft/state_machine.rs::drive_ready` +panic on persistent storage failure: + +- `panic!("raft storage apply_snapshot failed: …")` at ~line 556 +- `panic!("raft storage append failed: …")` at ~line 570 +- `panic!("raft storage set_hard_state failed: …")` at ~line 577 + +These panics are **correct by design** — they prevent committing +log entries that haven't reached durable storage. A panic crashes the +tokio task, which crashes the cube process, which trips the k8s +liveness probe and gets the pod restarted. No data loss; no silent +corruption. + +When you see one of these crashes in a pod log, this runbook walks +through the recovery. + +## 1. Detection + +The chart's PrometheusRule (`cube-stack-ha`) ships an alert +`CubeStorePVCNearFull` that fires at 15% remaining. If you see that +alert OR a pod is in `CrashLoopBackOff` and the last log line is one +of the three panic strings above, you're in this runbook. + +Quick triage: + +```bash +NS=cube +SEL=app=cubestore-router + +# How full are the PVCs? +kubectl -n $NS get pvc | grep cubestore-router + +# Recent restarts + crash reason +kubectl -n $NS get pods -l $SEL -o wide +kubectl -n $NS logs --previous --tail 50 | grep -E 'panic|append failed|set_hard_state' +``` + +## 2. PVC sizing math + +Cube Store's Raft log keeps every uncompacted entry. Compaction runs +every `M5.4_SNAPSHOT_INTERVAL` applied entries (default 10_000). At +typical write rates (50 ops/sec), the log is ~3 minutes deep before +the snapshot rolls. + +Rough sizing: + +``` +PVC_size_GiB = max( + 20, # floor for RocksDB SST + WAL + snapshot_size_GiB * 3, # last + previous + headroom + log_entry_size_KiB * 10_000 * 1.5 / 1_048_576 +) +``` + +Default snapshot size on a busy metastore (~5 M chunks) is ~2 GiB, so +the headroom default of `50Gi` for the router PVC is comfortable. If +your alert fires anyway, the snapshot interval has slipped — see §4 +below for forcing one. + +## 3. Recovery — disk full, single pod + +If only one router is full and the cluster still has quorum: + +```bash +# 3a. Drain it (if not already CrashLoopBackOff'ing). +kubectl -n $NS cordon # only if the node itself is full +kubectl -n $NS delete pod --grace-period=30 + +# 3b. Resize the PVC. StorageClass must allow expansion (most do). +kubectl -n $NS patch pvc --type='json' \ + -p='[{"op":"replace","path":"/spec/resources/requests/storage","value":"100Gi"}]' + +# 3c. The StatefulSet recreates the pod with the larger volume. +# Raft re-fetches the snapshot from the leader; cluster re-converges. +kubectl -n $NS rollout status statefulset/-cubestore-router --timeout=10m + +# 3d. Verify the cluster. +make -C charts/cube-stack-ha ha-verify # one leader, term ≥ N +``` + +## 4. Recovery — all routers full, cluster wedged + +If every router PVC is full simultaneously, Raft can't form a +quorum because the leader can't append the heartbeat. This is the +worst case. + +```bash +# 4a. Suspend traffic so we don't pile on more writes. +kubectl -n $NS scale deploy/-api --replicas=0 +kubectl -n $NS scale deploy/-refresh-worker --replicas=0 + +# 4b. Resize ALL router PVCs in parallel. +for i in 0 1 2; do + kubectl -n $NS patch pvc data--cubestore-router-${i} \ + --type='json' \ + -p='[{"op":"replace","path":"/spec/resources/requests/storage","value":"100Gi"}]' +done + +# 4c. Bounce the whole router StatefulSet — the underlying volumes +# resize on remount. +kubectl -n $NS delete pod -l app=cubestore-router + +# 4d. Wait for quorum + leader. +kubectl -n $NS rollout status statefulset/-cubestore-router --timeout=15m +make -C charts/cube-stack-ha ha-verify + +# 4e. Resume traffic. +kubectl -n $NS scale deploy/-api --replicas=2 +kubectl -n $NS scale deploy/-refresh-worker --replicas=1 +``` + +If quorum **still won't form** after the resize, the snapshot file +itself may be corrupt mid-write (panic during `apply_snapshot`). +Restore from the most recent backup — see `docs/RESTORE.md`. + +## 5. Prevention + +- Provision PVCs at `2× expected snapshot size + 10 GiB`. +- Enable `CubeStorePVCNearFull` alert + page at 15%. +- Run the daily backup CronJob (chart's `cubestore.backup.enabled`). +- Set the kubelet eviction threshold so the kubelet doesn't OOM-kill + the pod *before* Raft notices the disk-full and panics — panicking + is preferable to a silent data race. +- Never disable fsync. The `CUBESTORE_NO_FSYNC=true` knob exists for + benchmarks; it's correctness-unsafe in production. + +## 6. fsync timeout (slow disk) + +A different failure mode: fsync is **slow but not failing**. The +storage RwLock holds the guard across the rocksdb `write_opt`, so a +40-second fsync wedges the raft tick loop. Symptoms: + +- `up{component="cubestore-router"}` stays 1, `cs_raft_is_leader` + flaps between 0 and 1 every few seconds, term advances rapidly. +- `kubectl exec router-0 -- iostat -x 1` shows `await > 5000` (ms). + +Fix: replace the disk class. The fsync wait is unavoidable as long as +the underlying volume is slow; PR-S2 splits the in-memory copy from +the persistent write so cube reads aren't blocked, but raft's +`set_hard_state` itself must complete before the leader can ack +commits. + +## See also + +- `docs/RESTORE.md` — full backup/restore procedure. +- `docs/ha/M3-NOTES.md` — determinism / leader-stamp design. +- `charts/cube-stack-ha/UPGRADE.md` — running upgrades through this. diff --git a/rust/cubestore/cubestore/src/raft/state_machine.rs b/rust/cubestore/cubestore/src/raft/state_machine.rs index 2669a9019aee4..e8090bbc304c2 100644 --- a/rust/cubestore/cubestore/src/raft/state_machine.rs +++ b/rust/cubestore/cubestore/src/raft/state_machine.rs @@ -313,6 +313,13 @@ impl RaftNode { } /// Propose a command and resolve the future once it has applied. + /// + /// PR-S2: bounded by `CUBESTORE_RAFT_PROPOSE_TIMEOUT_SECS` (default 30s). + /// A slow leader, a stalled apply path (RocksDB write stall, large + /// blob encode), or a partition where the local replica isn't actually + /// leader anymore would otherwise block the caller (typically an HTTP + /// handler) forever. Timeout returns `RaftError::ProposeTimeout` -> + /// `CubeError`, which the caller surfaces as a 503. pub async fn propose(&self, command: MetaCommand) -> Result { let (tx, rx) = oneshot::channel(); self.proposals @@ -321,9 +328,17 @@ impl RaftNode { respond_to: tx, }) .map_err(|_| CubeError::internal("raft task is not running".to_string()))?; - rx.await.map_err(|_| { - CubeError::internal("raft task dropped the response channel".to_string()) - })? + let timeout = propose_timeout(); + match tokio::time::timeout(timeout, rx).await { + Ok(Ok(result)) => result, + Ok(Err(_)) => Err(CubeError::internal( + "raft task dropped the response channel".to_string(), + )), + Err(_) => Err(CubeError::internal(format!( + "raft propose timed out after {}s; check leader stability and apply queue depth", + timeout.as_secs() + ))), + } } /// M5.4 — handle to the underlying storage so the snapshot @@ -361,6 +376,24 @@ impl RaftNode { } } +/// Resolve the configured propose timeout (PR-S2). Reads +/// `CUBESTORE_RAFT_PROPOSE_TIMEOUT_SECS`, falls back to 30s. A value of +/// "0" disables the timeout (only useful in tests). The variable is +/// re-read on every call — cheap and lets operators tune at runtime +/// via `kubectl set env` without restarting the cube process. +fn propose_timeout() -> Duration { + let secs = std::env::var("CUBESTORE_RAFT_PROPOSE_TIMEOUT_SECS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(30); + if secs == 0 { + // Effectively unbounded; matches pre-PR-S2 behavior. + Duration::from_secs(60 * 60 * 24 * 365) + } else { + Duration::from_secs(secs) + } +} + /// Single-node mode has no peers; install a `Transport` that drops /// every message. Outbound messages still arise (heartbeats addressed /// to self filter out at the raft-rs layer; the rare stray gets diff --git a/rust/cubestore/cubestore/src/raft/transport.rs b/rust/cubestore/cubestore/src/raft/transport.rs index 6129d2381c44b..a72cb1219c13e 100644 --- a/rust/cubestore/cubestore/src/raft/transport.rs +++ b/rust/cubestore/cubestore/src/raft/transport.rs @@ -262,7 +262,15 @@ pub async fn spawn_listener( Ok(handle) } +// PR-S2: tolerated parse-failure rate. We accept the occasional flexbuffer- +// or protobuf-decode hiccup, but a peer flooding malformed frames would +// keep the recv loop pinned indefinitely on `continue` without this cap. +// After this many consecutive parse failures on the same connection we +// drop the connection and let the next message reconnect. +const MAX_CONSECUTIVE_PARSE_FAILS: u32 = 16; + async fn run_recv_loop(mut sock: TcpStream, inbound: Inbound) -> std::io::Result<()> { + let mut consecutive_parse_fails: u32 = 0; loop { // Drop the per-frame magic+version+len header. EOF here is the // peer cleanly closing — return Ok and let the spawn task end. @@ -298,9 +306,26 @@ async fn run_recv_loop(mut sock: TcpStream, inbound: Inbound) -> std::io::Result sock.read_exact(&mut buf).await?; let mut msg = Message::default(); if let Err(e) = msg.merge_from_bytes(&buf) { - log::warn!("raft transport: dropping malformed frame: {}", e); + consecutive_parse_fails += 1; + log::warn!( + "raft transport: dropping malformed frame ({}/{}): {}", + consecutive_parse_fails, + MAX_CONSECUTIVE_PARSE_FAILS, + e + ); + if consecutive_parse_fails >= MAX_CONSECUTIVE_PARSE_FAILS { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "raft transport: peer exceeded {} consecutive malformed frames; \ + dropping connection (last error: {})", + MAX_CONSECUTIVE_PARSE_FAILS, e + ), + )); + } continue; } + consecutive_parse_fails = 0; inbound.feed(msg); } }